00000400:00000200:19.0F:1616513067.871234:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:19.0:1616513067.871244:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:7.2F:1616513067.880005:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0F:1616513067.880063:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513067.880076:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513067.880087:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.76@tcp1 : PUT - for me 00000400:00000200:2.0:1616513067.880098:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.76@tcp1 of length 224 into portal 26 MB=0x601ad7c08de00 00000400:00000200:2.0:1616513067.880105:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.76@tcp1 of length 224/224 into md 0xf97c3189 [1] + 0 00000400:00000200:2.0:1616513067.880110:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcebdaaa0 00000400:00000200:2.0:1616513067.880114:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513067.880117:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0F:1616513067.880122:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513067.880122:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcebdaaa0 00000800:00000200:2.0:1616513067.880126:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513067.880130:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513067.880131:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0F:1616513067.880194:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690694482189824 00010000:00000200:20.0:1616513067.880210:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1690694482189824/t0(0) o400->4df55e66-cb7c-297b-5956-0320bb2c340a@10.31.140.76@tcp1:24/0 lens 224/224 e 0 to 0 dl 1616513159 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513067.880226:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690694482189824, offset 192 00000400:00000200:20.0:1616513067.880233:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.76@tcp1 00000400:00000200:20.0:1616513067.880241:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.76@tcp1 routed destination 00000400:00000200:20.0:1616513067.880258:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.76@tcp1(10.31.140.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513067.880264:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513067.880271:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513067.880274:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513067.880276:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513067.880283:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513067.880291:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2F:1616513067.880293:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0F:1616513067.880351:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513067.880361:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513067.880364:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513067.880371:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513067.880377:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23cb0 00000800:00000200:24.0:1616513067.880380:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513067.880383:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513067.880562:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0F:1616513067.880617:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513067.880629:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513067.880638:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.76@tcp1 : PUT - for me 00000400:00000200:18.0:1616513067.880647:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.76@tcp1 of length 224 into portal 12 MB=0x601ad7c08e940 00000400:00000200:18.0:1616513067.880654:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.76@tcp1 of length 224/224 into md 0xf963e789 [1] + 92832 00000400:00000200:18.0:1616513067.880659:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513067.880662:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513067.880669:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513067.880672:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513067.880674:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513067.880677:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0F:1616513067.880688:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690694482192704 00010000:00000200:10.0:1616513067.880700:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d0900 x1690694482192704/t0(0) o400->9a6fca0a-cb5e-6cd6-3773-68cf989244e9@10.31.140.76@tcp1:693/0 lens 224/224 e 0 to 0 dl 1616513073 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513067.880713:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690694482192704, offset 192 00000400:00000200:10.0:1616513067.880718:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.76@tcp1 00000400:00000200:10.0:1616513067.880725:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.76@tcp1 routed destination 00000400:00000200:10.0:1616513067.880737:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.76@tcp1(10.31.140.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513067.880743:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513067.880748:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513067.880751:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513067.880752:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513067.880757:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513067.880761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513067.880772:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513067.880778:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513067.880780:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513067.880783:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513067.880787:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513067.880791:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82b28 00000800:00000200:26.0:1616513067.880795:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513067.894518:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0F:1616513067.894532:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513067.894544:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513067.894553:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.109@tcp1 : PUT - for me 00000400:00000200:4.0:1616513067.894562:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.109@tcp1 of length 224 into portal 26 MB=0x5f91ac86a0580 00000400:00000200:4.0:1616513067.894569:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.109@tcp1 of length 224/224 into md 0xf97c3199 [1] + 0 00000400:00000200:4.0:1616513067.894573:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb540 00000400:00000200:4.0:1616513067.894576:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513067.894579:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513067.894583:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb540 00000800:00000200:4.0:1616513067.894587:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513067.894589:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0F:1616513067.894592:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513067.894601:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513067.894604:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513067.894605:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513067.894614:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.109@tcp1 : PUT - for me 00000400:00000200:16.0:1616513067.894623:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.109@tcp1 of length 224 into portal 12 MB=0x5f91ac86a0600 00000400:00000200:16.0:1616513067.894629:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.109@tcp1 of length 224/224 into md 0x865b9f31 [1] + 21552 00000400:00000200:16.0:1616513067.894635:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513067.894638:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513067.894645:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513067.894648:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513067.894649:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681268310410624 00000800:00000200:16.0:1616513067.894650:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0F:1616513067.894660:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681268310410752 00010000:00000200:20.0:1616513067.894661:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1681268310410624/t0(0) o400->c608d7be-6df9-48bb-d54d-198a90f83d8b@10.31.143.109@tcp1:24/0 lens 224/224 e 0 to 0 dl 1616513159 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:8.0:1616513067.894670:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e4800 x1681268310410752/t0(0) o400->84b38476-d982-253e-7217-750adb18d6f8@10.31.143.109@tcp1:693/0 lens 224/224 e 0 to 0 dl 1616513073 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513067.894675:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681268310410624, offset 192 00000400:00000200:20.0:1616513067.894681:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.109@tcp1 00000100:00000200:8.0:1616513067.894684:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681268310410752, offset 192 00000400:00000200:20.0:1616513067.894689:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.109@tcp1 routed destination 00000400:00000200:8.0:1616513067.894689:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.109@tcp1 00000400:00000200:20.0:1616513067.894704:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.109@tcp1(10.31.143.109@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513067.894710:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:8.0:1616513067.894711:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.109@tcp1 routed destination 00000800:00000200:20.0:1616513067.894718:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513067.894721:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:8.0:1616513067.894722:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.109@tcp1(10.31.143.109@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513067.894723:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513067.894727:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513067.894728:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513067.894731:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:11.2:1616513067.894733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513067.894733:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513067.894734:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513067.894735:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513067.894738:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513067.894745:0:48142:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0F:1616513067.894784:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513067.894793:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513067.894796:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513067.894796:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513067.894802:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0F:1616513067.894806:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513067.894807:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22cc0 00000800:00000200:12.0:1616513067.894811:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513067.894816:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513067.894819:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513067.894825:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513067.894829:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70d48 00000800:00000200:22.0:1616513067.894833:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513067.894836:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513067.909003:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513067.909058:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513067.909067:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513067.909076:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.7@tcp1 : PUT - for me 00000400:00000200:2.0:1616513067.909086:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.7@tcp1 of length 224 into portal 26 MB=0x60006c253e880 00000400:00000200:2.0:1616513067.909092:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.7@tcp1 of length 224/224 into md 0xf97c31a9 [1] + 0 00000400:00000200:2.0:1616513067.909097:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba220 00000400:00000200:2.0:1616513067.909100:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513067.909102:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513067.909107:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba220 00000800:00000200:2.0:1616513067.909110:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513067.909113:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513067.909113:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513067.909117:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513067.909175:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688878890346624 00010000:00000200:20.0:1616513067.909186:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688878890346624/t0(0) o400->afa66ea3-c5aa-348c-c6d7-878e1c8eb9d1@10.31.133.7@tcp1:24/0 lens 224/224 e 0 to 0 dl 1616513159 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513067.909201:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688878890346624, offset 192 00000400:00000200:20.0:1616513067.909207:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.7@tcp1 00000400:00000200:20.0:1616513067.909217:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.7@tcp1 routed destination 00000400:00000200:20.0:1616513067.909232:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.7@tcp1(10.31.133.7@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513067.909238:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513067.909244:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513067.909246:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513067.909249:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513067.909253:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513067.909260:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513067.909260:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513067.909298:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513067.909325:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513067.909332:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513067.909335:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513067.909341:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513067.909347:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22198 00000800:00000200:24.0:1616513067.909350:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513067.909353:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513067.909355:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513067.909364:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513067.909368:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513067.909373:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.7@tcp1 : PUT - for me 00000400:00000200:22.0:1616513067.909382:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.7@tcp1 of length 224 into portal 12 MB=0x60006c253e980 00000400:00000200:22.0:1616513067.909389:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.7@tcp1 of length 224/224 into md 0xf963e789 [1] + 93056 00000400:00000200:22.0:1616513067.909394:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513067.909396:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513067.909404:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513067.909407:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513067.909409:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513067.909421:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688878890346880 00010000:00000200:10.0:1616513067.909431:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08ba80 x1688878890346880/t0(0) o400->8cf3d54c-8b99-5411-1e2a-b18e32d32b94@10.31.133.7@tcp1:693/0 lens 224/224 e 0 to 0 dl 1616513073 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513067.909453:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688878890346880, offset 192 00000400:00000200:10.0:1616513067.909459:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.7@tcp1 00000400:00000200:10.0:1616513067.909465:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.7@tcp1 routed destination 00000400:00000200:10.0:1616513067.909479:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.7@tcp1(10.31.133.7@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513067.909485:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513067.909490:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513067.909492:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513067.909494:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513067.909498:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513067.909503:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513067.909513:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513067.909518:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513067.909521:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513067.909523:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513067.909528:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513067.909532:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e833a8 00000800:00000200:26.0:1616513067.909536:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513067.930589:0:48146:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513067.930602:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513067.930610:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513067.930619:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.37@tcp1 : PUT - for me 00000400:00000200:18.0:1616513067.930630:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.37@tcp1 of length 224 into portal 26 MB=0x5ffddd7460040 00000400:00000200:18.0:1616513067.930636:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.37@tcp1 of length 224/224 into md 0xf97c31c9 [1] + 0 00000400:00000200:18.0:1616513067.930641:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bad48 00000400:00000200:18.0:1616513067.930643:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513067.930646:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513067.930651:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bad48 00000800:00000200:18.0:1616513067.930653:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513067.930656:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513067.930656:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513067.930660:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513067.930714:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703148097600 00010000:00000200:20.0:1616513067.930725:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688703148097600/t0(0) o400->93947cdc-0ff3-b891-4283-ca9dc7da003c@10.31.140.37@tcp1:24/0 lens 224/224 e 0 to 0 dl 1616513159 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513067.930740:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703148097600, offset 192 00000400:00000200:20.0:1616513067.930746:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.37@tcp1 00000400:00000200:20.0:1616513067.930754:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.37@tcp1 routed destination 00000400:00000200:20.0:1616513067.930769:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.37@tcp1(10.31.140.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513067.930776:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513067.930781:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513067.930784:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513067.930786:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513067.930791:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513067.930798:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513067.930799:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513067.930809:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513067.930812:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513067.930815:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513067.930820:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513067.930824:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22770 00000800:00000200:4.0:1616513067.930827:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513067.930829:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513067.930834:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513067.930861:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513067.930872:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513067.930882:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.37@tcp1 : PUT - for me 00000800:00000200:24.0:1616513067.930891:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513067.930891:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.37@tcp1 of length 224 into portal 12 MB=0x5ffddd7460340 00000400:00000200:12.0:1616513067.930897:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.37@tcp1 of length 224/224 into md 0x865b9f31 [1] + 21776 00000400:00000200:12.0:1616513067.930902:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513067.930905:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513067.930912:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513067.930915:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513067.930917:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513067.930924:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703148098368 00010000:00000200:8.0:1616513067.930932:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa47080 x1688703148098368/t0(0) o400->e31725e8-15c5-eba1-7059-8b16968c3cd9@10.31.140.37@tcp1:693/0 lens 224/224 e 0 to 0 dl 1616513073 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513067.930941:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703148098368, offset 192 00000400:00000200:8.0:1616513067.930945:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.37@tcp1 00000400:00000200:8.0:1616513067.930950:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.37@tcp1 routed destination 00000400:00000200:8.0:1616513067.930959:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.37@tcp1(10.31.140.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513067.930964:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513067.930967:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513067.930969:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513067.930970:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513067.930973:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513067.930978:0:48146:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513067.931031:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513067.931038:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513067.931041:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513067.931048:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513067.931052:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f713a8 00000800:00000200:2.0:1616513067.931056:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513067.931058:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513067.939990:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513067.940046:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513067.940055:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513067.940064:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.67@tcp1 : PUT - for me 00000400:00000200:22.0:1616513067.940072:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.67@tcp1 of length 224 into portal 12 MB=0x603ff6c15c100 00000800:00000200:11.2:1616513067.940073:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513067.940079:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.67@tcp1 of length 224/224 into md 0xf963e789 [1] + 93280 00000400:00000200:22.0:1616513067.940084:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513067.940086:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513067.940089:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513067.940093:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513067.940096:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:4.0:1616513067.940096:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513067.940097:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513067.940100:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513067.940104:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.67@tcp1 : PUT - for me 00000400:00000200:4.0:1616513067.940112:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.67@tcp1 of length 224 into portal 26 MB=0x603ff6c15c080 00000400:00000200:4.0:1616513067.940118:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.67@tcp1 of length 224/224 into md 0xf97c31d9 [1] + 0 00000400:00000200:4.0:1616513067.940122:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bbcb0 00000400:00000200:4.0:1616513067.940124:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513067.940126:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513067.940130:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbcb0 00000800:00000200:4.0:1616513067.940132:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513067.940135:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513067.940136:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513067.940139:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513067.940150:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245425172736 00010000:00000200:10.0:1616513067.940161:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b088d80 x1693245425172736/t0(0) o400->fa796449-551b-6b7f-dc0c-1d922fe7d0bd@10.31.132.67@tcp1:693/0 lens 224/224 e 0 to 0 dl 1616513073 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513067.940173:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245425172736, offset 192 00000400:00000200:10.0:1616513067.940179:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.67@tcp1 00000400:00000200:10.0:1616513067.940185:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.67@tcp1 routed destination 00000100:00000200:20.0:1616513067.940193:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245425172608 00000400:00000200:10.0:1616513067.940198:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.67@tcp1(10.31.132.67@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:20.0:1616513067.940203:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1693245425172608/t0(0) o400->22740cb1-b3f2-4812-be7f-294a1852af03@10.31.132.67@tcp1:24/0 lens 224/224 e 0 to 0 dl 1616513159 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:10.0:1616513067.940205:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513067.940210:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513067.940212:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513067.940214:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513067.940216:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245425172608, offset 192 00000800:00000200:10.0:1616513067.940218:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513067.940222:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.67@tcp1 00000800:00000200:11.2:1616513067.940224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513067.940228:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.67@tcp1 routed destination 00000400:00000200:20.0:1616513067.940242:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.67@tcp1(10.31.132.67@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513067.940248:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513067.940255:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513067.940258:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513067.940260:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513067.940265:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513067.940270:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513067.940271:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513067.940276:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513067.940283:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513067.940286:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513067.940292:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513067.940296:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83980 00000800:00000200:12.0:1616513067.940300:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513067.940303:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513067.940323:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513067.940332:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513067.940335:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513067.940341:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513067.940347:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee225d8 00000800:00000200:2.0:1616513067.940350:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513067.940353:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513067.948888:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513067.948899:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513067.948907:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513067.948915:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.48@tcp1 : PUT - for me 00000400:00000200:4.0:1616513067.948923:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.48@tcp1 of length 224 into portal 12 MB=0x603d045d14500 00000400:00000200:4.0:1616513067.948929:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.48@tcp1 of length 224/224 into md 0x865b9f31 [1] + 22000 00000400:00000200:4.0:1616513067.948934:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513067.948936:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513067.948942:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513067.948945:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513067.948946:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513067.948949:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513067.948955:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042919687424 00010000:00000200:8.0:1616513067.948962:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8542ee9200 x1693042919687424/t0(0) o400->5898e81a-3c0d-afb8-a73d-89b5bcae2c96@10.31.140.48@tcp1:693/0 lens 224/224 e 0 to 0 dl 1616513073 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513067.948970:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693042919687424, offset 192 00000400:00000200:8.0:1616513067.948974:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.48@tcp1 00000400:00000200:8.0:1616513067.948979:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.48@tcp1 routed destination 00000400:00000200:8.0:1616513067.948988:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.48@tcp1(10.31.140.48@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513067.948992:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513067.948996:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513067.948997:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513067.948998:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513067.949002:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513067.949006:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513067.949043:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513067.949058:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513067.949065:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513067.949068:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513067.949074:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513067.949078:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71cb0 00000800:00000200:12.0:1616513067.949082:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513067.949085:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513067.949096:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513067.949105:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513067.949114:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.48@tcp1 : PUT - for me 00000400:00000200:22.0:1616513067.949123:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.48@tcp1 of length 224 into portal 26 MB=0x603d045d142c0 00000400:00000200:22.0:1616513067.949129:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.48@tcp1 of length 224/224 into md 0xf97c31e9 [1] + 0 00000400:00000200:22.0:1616513067.949136:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb320 00000400:00000200:22.0:1616513067.949139:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513067.949142:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513067.949145:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb320 00000800:00000200:22.0:1616513067.949148:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513067.949148:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513067.949151:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513067.949153:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513067.949206:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042919686848 00010000:00000200:20.0:1616513067.949216:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1693042919686848/t0(0) o400->820cc64e-8e2e-429e-c1bc-9047d6d37047@10.31.140.48@tcp1:24/0 lens 224/224 e 0 to 0 dl 1616513159 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513067.949230:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693042919686848, offset 192 00000400:00000200:20.0:1616513067.949236:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.48@tcp1 00000400:00000200:20.0:1616513067.949243:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.48@tcp1 routed destination 00000400:00000200:20.0:1616513067.949257:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.48@tcp1(10.31.140.48@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513067.949263:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513067.949268:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513067.949271:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513067.949273:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513067.949277:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513067.949282:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513067.949283:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513067.949335:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513067.949342:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513067.949345:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513067.949351:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513067.949357:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee225d8 00000800:00000200:2.0:1616513067.949360:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513067.949363:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513067.954106:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513067.954124:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513067.954134:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513067.954143:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.110@tcp1 : PUT - for me 00000800:00000200:11.2:1616513067.954150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513067.954153:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.110@tcp1 of length 224 into portal 26 MB=0x6000849d68080 00000400:00000200:18.0:1616513067.954159:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.110@tcp1 of length 224/224 into md 0xf97c31f9 [1] + 0 00000400:00000200:18.0:1616513067.954164:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6baee0 00000400:00000200:18.0:1616513067.954167:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513067.954171:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513067.954175:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baee0 00000800:00000200:18.0:1616513067.954179:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513067.954180:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513067.954181:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513067.954184:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513067.954205:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513067.954214:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513067.954214:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513067.954224:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.110@tcp1 : PUT - for me 00000400:00000200:24.0:1616513067.954233:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.110@tcp1 of length 224 into portal 12 MB=0x6000849d68180 00000100:00000200:20.0:1616513067.954236:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688885458796672 00000400:00000200:24.0:1616513067.954239:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.110@tcp1 of length 224/224 into md 0x865b9f31 [1] + 22224 00000400:00000200:24.0:1616513067.954244:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:20.0:1616513067.954245:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688885458796672/t0(0) o400->d258bcec-f3cb-451e-a457-5685bd5b89fd@10.31.138.110@tcp1:24/0 lens 224/224 e 0 to 0 dl 1616513159 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513067.954247:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513067.954254:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513067.954257:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513067.954257:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688885458796672, offset 192 00000800:00000200:24.0:1616513067.954259:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:20.0:1616513067.954263:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.110@tcp1 00000100:00000200:8.0:1616513067.954263:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688885458796928 00010000:00000200:8.0:1616513067.954268:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc0480 x1688885458796928/t0(0) o400->caf0d9ea-7b49-b677-e6ce-69c43e0af531@10.31.138.110@tcp1:693/0 lens 224/224 e 0 to 0 dl 1616513073 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513067.954269:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.110@tcp1 routed destination 00000100:00000200:8.0:1616513067.954285:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688885458796928, offset 192 00000400:00000200:20.0:1616513067.954288:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.110@tcp1(10.31.138.110@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513067.954288:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.110@tcp1 00000800:00000200:20.0:1616513067.954294:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:8.0:1616513067.954294:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.110@tcp1 routed destination 00000800:00000200:20.0:1616513067.954299:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513067.954302:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:8.0:1616513067.954302:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.110@tcp1(10.31.138.110@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513067.954303:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513067.954305:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513067.954308:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513067.954309:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513067.954310:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513067.954311:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513067.954313:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513067.954313:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513067.954319:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513067.954319:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513067.954368:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513067.954375:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513067.954377:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513067.954384:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513067.954387:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513067.954389:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22198 00000800:00000200:12.0:1616513067.954395:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513067.954395:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513067.954397:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513067.954398:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513067.954404:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513067.954408:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71d38 00000800:00000200:2.0:1616513067.954412:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513067.954415:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513067.968858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513067.968913:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513067.968923:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513067.968932:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.240@tcp1 : PUT - for me 00000400:00000200:22.0:1616513067.968942:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.240@tcp1 of length 224 into portal 26 MB=0x60233ff940140 00000400:00000200:22.0:1616513067.968949:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.240@tcp1 of length 224/224 into md 0xf97c3219 [1] + 0 00000400:00000200:22.0:1616513067.968954:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb760 00000400:00000200:22.0:1616513067.968956:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513067.968959:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513067.968963:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb760 00000800:00000200:22.0:1616513067.968966:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513067.968967:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513067.968969:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513067.968973:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513067.968976:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513067.968977:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513067.968986:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.240@tcp1 : PUT - for me 00000400:00000200:18.0:1616513067.968995:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.240@tcp1 of length 224 into portal 12 MB=0x60233ff940380 00000400:00000200:18.0:1616513067.969002:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.240@tcp1 of length 224/224 into md 0xf963e789 [1] + 93504 00000400:00000200:18.0:1616513067.969007:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513067.969009:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513067.969016:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513067.969019:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513067.969021:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513067.969022:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691272214741312 00010000:00000200:20.0:1616513067.969032:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1691272214741312/t0(0) o400->489eacf0-f0bd-497e-4436-2c6147ed5ed4@10.31.142.240@tcp1:24/0 lens 224/224 e 0 to 0 dl 1616513159 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513067.969044:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691272214741312, offset 192 00000400:00000200:20.0:1616513067.969049:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.240@tcp1 00000400:00000200:20.0:1616513067.969056:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.240@tcp1 routed destination 00000400:00000200:20.0:1616513067.969069:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.240@tcp1(10.31.142.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513067.969074:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513067.969079:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:10.0:1616513067.969080:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691272214741888 00000800:00000200:20.0:1616513067.969081:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513067.969083:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513067.969087:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:10.0:1616513067.969092:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d6780 x1691272214741888/t0(0) o400->3ea63e59-9d07-2bd6-4dc3-93acb861270d@10.31.142.240@tcp1:693/0 lens 224/224 e 0 to 0 dl 1616513073 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513067.969093:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513067.969094:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:10.0:1616513067.969107:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691272214741888, offset 192 00000400:00000200:10.0:1616513067.969113:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.240@tcp1 00000400:00000200:10.0:1616513067.969121:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.240@tcp1 routed destination 00000400:00000200:10.0:1616513067.969135:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.240@tcp1(10.31.142.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513067.969142:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513067.969147:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513067.969147:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513067.969150:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513067.969152:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513067.969154:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513067.969156:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513067.969157:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513067.969161:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513067.969162:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513067.969167:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23cb0 00000800:00000200:4.0:1616513067.969170:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513067.969173:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513067.969221:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513067.969228:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513067.969231:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513067.969232:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513067.969237:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513067.969241:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82198 00000800:00000200:26.0:1616513067.969245:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513067.978352:0:48151:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513067.978406:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513067.978414:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513067.978423:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.186@tcp1 : PUT - for me 00000400:00000200:18.0:1616513067.978431:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.186@tcp1 of length 224 into portal 26 MB=0x5ffde95f81dc0 00000400:00000200:18.0:1616513067.978449:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.186@tcp1 of length 224/224 into md 0xf97c3229 [1] + 0 00000400:00000200:18.0:1616513067.978454:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba198 00000400:00000200:18.0:1616513067.978456:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513067.978459:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513067.978459:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513067.978462:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba198 00000800:00000200:18.0:1616513067.978465:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513067.978468:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513067.978470:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513067.978523:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706347441600 00010000:00000200:20.0:1616513067.978532:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688706347441600/t0(0) o400->9731353a-a51f-e821-d202-e2ba0d42a085@10.31.128.186@tcp1:24/0 lens 224/224 e 0 to 0 dl 1616513159 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513067.978543:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706347441600, offset 192 00000400:00000200:20.0:1616513067.978548:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.186@tcp1 00000400:00000200:20.0:1616513067.978554:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.186@tcp1 routed destination 00000800:00000200:7.2:1616513067.978555:0:48151:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513067.978566:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.186@tcp1(10.31.128.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513067.978571:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513067.978576:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513067.978578:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513067.978580:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513067.978584:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513067.978589:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513067.978590:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513067.978613:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513067.978622:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513067.978631:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.186@tcp1 : PUT - for me 00000400:00000200:22.0:1616513067.978639:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.186@tcp1 of length 224 into portal 12 MB=0x5ffde95f82280 00000800:00000200:12.0:1616513067.978643:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513067.978645:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.186@tcp1 of length 224/224 into md 0xf963e789 [1] + 93728 00000400:00000200:22.0:1616513067.978651:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513067.978652:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:22.0:1616513067.978653:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513067.978654:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513067.978656:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513067.978660:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513067.978661:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513067.978663:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513067.978665:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513067.978666:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23298 00000800:00000200:22.0:1616513067.978667:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513067.978670:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513067.978722:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706347442816 00010000:00000200:10.0:1616513067.978734:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b089f80 x1688706347442816/t0(0) o400->a742e05d-9433-3718-f2a4-cdee49404d95@10.31.128.186@tcp1:693/0 lens 224/224 e 0 to 0 dl 1616513073 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513067.978749:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706347442816, offset 192 00000400:00000200:10.0:1616513067.978755:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.186@tcp1 00000400:00000200:10.0:1616513067.978762:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.186@tcp1 routed destination 00000400:00000200:10.0:1616513067.978777:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.186@tcp1(10.31.128.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513067.978783:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513067.978789:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513067.978792:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513067.978794:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513067.978799:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513067.978803:0:48151:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513067.978813:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513067.978818:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513067.978821:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513067.978822:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513067.978827:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513067.978831:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82c38 00000800:00000200:2.0:1616513067.978835:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:7.0F:1616513068.026560:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923040, portal 4 00000100:00000200:7.0:1616513068.026565:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923040, offset 0 00000400:00000200:7.0:1616513068.026570:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:7.0:1616513068.026575:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:7.0:1616513068.026584:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513068.026588:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:7.0:1616513068.026593:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513068.026596:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:7.0:1616513068.026597:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:7.0:1616513068.026603:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2F:1616513068.026612:0:48159:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513068.026664:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513068.026671:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513068.026674:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513068.026680:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513068.026686:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456ec800 x1693067725923040/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513068.026696:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f188 00000800:00000200:4.0:1616513068.026700:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513068.026702:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513068.026767:0:48159:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513068.026821:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513068.026830:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513068.026839:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513068.026849:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6262e0 00000400:00000200:20.0:1616513068.026856:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d1e71 [1] + 192 00000400:00000200:20.0:1616513068.026861:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.026866:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456ec800 x1693067725923040/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513068.026875:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513068.026881:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513068.026885:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513068.026886:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:7.0:1616513068.026887:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456ec800 x1693067725923040/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513068.026894:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456ec800 x1693067725923040/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513068.026901:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f5c8 00000800:00000200:7.2:1616513068.040260:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513068.040316:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513068.040325:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513068.040334:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.19@tcp1 : PUT - for me 00000400:00000200:26.0:1616513068.040343:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.19@tcp1 of length 224 into portal 26 MB=0x604f9ccf6fe80 00000400:00000200:26.0:1616513068.040350:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.19@tcp1 of length 224/224 into md 0xf97c3249 [1] + 0 00000400:00000200:26.0:1616513068.040354:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb3a8 00000400:00000200:26.0:1616513068.040357:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513068.040360:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513068.040364:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb3a8 00000800:00000200:26.0:1616513068.040367:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.040368:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513068.040370:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513068.040372:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513068.040452:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694320792370816 00010000:00000200:20.0:1616513068.040463:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1694320792370816/t0(0) o400->5eda2b9f-5d34-4d14-488e-a8c299b20339@10.31.141.19@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513068.040476:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694320792370816, offset 192 00000400:00000200:20.0:1616513068.040482:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.19@tcp1 00000400:00000200:20.0:1616513068.040490:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.19@tcp1 routed destination 00000400:00000200:20.0:1616513068.040503:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.19@tcp1(10.31.141.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513068.040509:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:11.2:1616513068.040511:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513068.040515:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513068.040518:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513068.040520:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.040525:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:20.0:1616513068.040531:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513068.040580:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.040590:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513068.040599:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.19@tcp1 : PUT - for me 00000400:00000200:4.0:1616513068.040608:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.19@tcp1 of length 224 into portal 12 MB=0x604f9ccf70140 00000400:00000200:4.0:1616513068.040614:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.19@tcp1 of length 224/224 into md 0x865b9f31 [1] + 22448 00000400:00000200:4.0:1616513068.040620:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513068.040623:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513068.040630:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513068.040633:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513068.040635:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513068.040637:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:8.0:1616513068.040640:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694320792371520 00000800:00000200:4.0:1616513068.040641:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513068.040643:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513068.040644:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:8.0:1616513068.040646:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8542eede80 x1694320792371520/t0(0) o400->1b948c47-a6bc-3007-ff95-2d9ed32b822c@10.31.141.19@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:4.0:1616513068.040649:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513068.040653:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee237e8 00000100:00000200:8.0:1616513068.040656:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694320792371520, offset 192 00000800:00000200:4.0:1616513068.040656:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513068.040659:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.19@tcp1 00000400:00000200:8.0:1616513068.040663:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.19@tcp1 routed destination 00000400:00000200:8.0:1616513068.040670:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.19@tcp1(10.31.141.19@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.040673:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513068.040676:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.040678:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513068.040679:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513068.040681:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.040686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513068.040737:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.040744:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513068.040746:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513068.040752:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.040756:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70550 00000800:00000200:18.0:1616513068.040761:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513068.040764:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.045204:0:48161:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513068.045259:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.045268:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513068.045277:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.133@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.045286:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.133@tcp1 of length 224 into portal 26 MB=0x5ffdf3995c500 00000400:00000200:22.0:1616513068.045293:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.133@tcp1 of length 224/224 into md 0xf97c3259 [1] + 0 00000400:00000200:22.0:1616513068.045297:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcebdbb18 00000400:00000200:22.0:1616513068.045300:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513068.045303:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513068.045307:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcebdbb18 00000800:00000200:22.0:1616513068.045310:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513068.045312:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.045313:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.045319:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.045322:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513068.045323:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513068.045332:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.133@tcp1 : PUT - for me 00000400:00000200:26.0:1616513068.045342:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.133@tcp1 of length 224 into portal 12 MB=0x5ffdf3995c580 00000400:00000200:26.0:1616513068.045349:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.133@tcp1 of length 224/224 into md 0xf963e789 [1] + 93952 00000400:00000200:26.0:1616513068.045354:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513068.045356:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513068.045364:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513068.045367:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513068.045368:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709092459776 00000800:00000200:26.0:1616513068.045369:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:20.0:1616513068.045378:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688709092459776/t0(0) o400->6b4a3909-2c96-704e-f9d4-e5c60da08347@10.31.142.133@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513068.045392:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688709092459776, offset 192 00000400:00000200:20.0:1616513068.045397:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.133@tcp1 00000400:00000200:20.0:1616513068.045404:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.133@tcp1 routed destination 00000400:00000200:20.0:1616513068.045418:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.133@tcp1(10.31.142.133@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513068.045424:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:10.0:1616513068.045427:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709092459904 00000800:00000200:20.0:1616513068.045431:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513068.045433:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513068.045446:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513068.045451:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:10.0:1616513068.045455:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c694c80 x1688709092459904/t0(0) o400->f2bddaa5-3ae6-14f2-0ea5-c6d825e25ac3@10.31.142.133@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513068.045457:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513068.045457:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:10.0:1616513068.045469:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688709092459904, offset 192 00000800:00000200:20.0:1616513068.045471:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.045475:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:10.0:1616513068.045476:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.133@tcp1 00000400:00000200:20.0:1616513068.045477:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513068.045482:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:10.0:1616513068.045483:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.133@tcp1 routed destination 00000400:00000200:20.0:1616513068.045485:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22110 00000800:00000200:20.0:1616513068.045488:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.045490:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513068.045498:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.133@tcp1(10.31.142.133@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513068.045504:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513068.045510:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513068.045513:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513068.045515:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.045519:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.045524:0:48161:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513068.045575:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513068.045582:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513068.045585:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513068.045587:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513068.045591:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513068.045595:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82990 00000800:00000200:2.0:1616513068.045599:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513068.090673:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923056, portal 4 00000100:00000200:24.0:1616513068.090679:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923056, offset 0 00000400:00000200:24.0:1616513068.090685:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:24.0:1616513068.090690:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:24.0:1616513068.090703:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.090710:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:24.0:1616513068.090715:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.090718:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:24.0:1616513068.090722:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513068.090727:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513068.090736:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513068.090788:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513068.090795:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513068.090797:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513068.090800:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:4.0:1616513068.090803:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513068.090809:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4557ce8000 x1693067725923056/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513068.090820:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f078 00000800:00000200:4.0:1616513068.090824:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513068.090952:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513068.090962:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513068.090970:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:24.0:1616513068.090972:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:20.0:1616513068.090978:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513068.090986:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6262f0 00000400:00000200:20.0:1616513068.090992:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d1eb1 [1] + 192 00000400:00000200:20.0:1616513068.090997:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.091002:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4557ce8000 x1693067725923056/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513068.091015:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513068.091018:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513068.091020:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:24.0:1616513068.091021:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4557ce8000 x1693067725923056/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:24.0:1616513068.091034:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4557ce8000 x1693067725923056/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513068.091044:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fdc0 00000800:00000200:7.2:1616513068.140587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513068.140641:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513068.140651:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513068.140660:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.60@tcp1 : PUT - for me 00000400:00000200:26.0:1616513068.140669:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.60@tcp1 of length 224 into portal 26 MB=0x5f91dbfb87ec0 00000400:00000200:26.0:1616513068.140676:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.60@tcp1 of length 224/224 into md 0xf97c3269 [1] + 0 00000400:00000200:26.0:1616513068.140681:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcebda770 00000400:00000200:26.0:1616513068.140683:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513068.140686:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513068.140691:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcebda770 00000800:00000200:26.0:1616513068.140694:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.140694:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513068.140696:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513068.140698:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513068.140760:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681281049460416 00000800:00000200:11.2:1616513068.140765:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:20.0:1616513068.140771:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1681281049460416/t0(0) o400->85817ad3-90a6-f453-8945-63fa8b4f1ea2@10.31.131.60@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513068.140785:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681281049460416, offset 192 00000400:00000200:20.0:1616513068.140791:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.60@tcp1 00000400:00000200:20.0:1616513068.140799:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.60@tcp1 routed destination 00000400:00000200:20.0:1616513068.140815:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.60@tcp1(10.31.131.60@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513068.140821:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513068.140827:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513068.140829:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513068.140831:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.140833:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513068.140836:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513068.140838:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:20.0:1616513068.140845:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513068.140845:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.60@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.140854:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.60@tcp1 of length 224 into portal 12 MB=0x5f91dbfb87f40 00000400:00000200:12.0:1616513068.140860:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.60@tcp1 of length 224/224 into md 0x865b9f31 [1] + 22672 00000400:00000200:12.0:1616513068.140865:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.140867:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513068.140874:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513068.140877:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513068.140879:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513068.140881:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:8.0:1616513068.140883:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681281049460544 00010000:00000200:8.0:1616513068.140888:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa46c00 x1681281049460544/t0(0) o400->378fd162-53b1-8bce-b378-ba2bd6932fae@10.31.131.60@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:4.0:1616513068.140890:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513068.140893:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513068.140894:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681281049460544, offset 192 00000400:00000200:8.0:1616513068.140897:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.60@tcp1 00000400:00000200:4.0:1616513068.140899:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513068.140900:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.60@tcp1 routed destination 00000400:00000200:4.0:1616513068.140905:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22110 00000400:00000200:8.0:1616513068.140907:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.60@tcp1(10.31.131.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513068.140908:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513068.140909:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513068.140911:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513068.140912:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.140913:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513068.140914:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513068.140917:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.140921:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.140932:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.140938:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.140941:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513068.140948:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.140952:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71320 00000800:00000200:22.0:1616513068.140956:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.140959:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.163433:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513068.163466:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513068.163487:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513068.163497:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513068.163506:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.166@tcp1 : PUT - for me 00000400:00000200:18.0:1616513068.163515:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.166@tcp1 of length 224 into portal 26 MB=0x6001dfef960c0 00000400:00000200:18.0:1616513068.163521:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.166@tcp1 of length 224/224 into md 0xf97c3279 [1] + 0 00000800:00000200:20.0:1616513068.163522:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513068.163526:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcebdb7e8 00000400:00000200:18.0:1616513068.163528:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.163531:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513068.163532:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:18.0:1616513068.163536:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcebdb7e8 00000800:00000200:18.0:1616513068.163539:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513068.163541:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.166@tcp1 : PUT - for me 00000800:00000200:18.0:1616513068.163542:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513068.163543:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513068.163547:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513068.163550:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.166@tcp1 of length 224 into portal 12 MB=0x6001dfef96140 00000400:00000200:20.0:1616513068.163556:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.166@tcp1 of length 224/224 into md 0x865b9f31 [1] + 22896 00000400:00000200:20.0:1616513068.163561:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.163564:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513068.163570:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.163573:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.163575:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513068.163578:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688978692071744 00000800:00000200:20.0:1616513068.163579:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513068.163583:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cebf00 x1688978692071744/t0(0) o400->625a366c-8d76-8030-8636-1d4744cc1d90@10.31.141.166@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513068.163589:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688978692071744, offset 192 00000400:00000200:8.0:1616513068.163591:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.166@tcp1 00000400:00000200:8.0:1616513068.163594:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.166@tcp1 routed destination 00000400:00000200:8.0:1616513068.163600:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.166@tcp1(10.31.141.166@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.163602:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513068.163605:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.163606:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.163607:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.163609:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513068.163613:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688978692071616 00000800:00000200:11.2:1616513068.163613:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513068.163624:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688978692071616/t0(0) o400->228c0ff7-3f97-7e3b-072a-c36d58b29bdd@10.31.141.166@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.163638:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688978692071616, offset 192 00000400:00000200:24.0:1616513068.163644:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.166@tcp1 00000400:00000200:24.0:1616513068.163651:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.166@tcp1 routed destination 00000400:00000200:24.0:1616513068.163666:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.166@tcp1(10.31.141.166@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.163673:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513068.163676:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513068.163678:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.163681:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.163683:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513068.163685:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513068.163687:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513068.163688:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0F:1616513068.163690:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513068.163692:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513068.163694:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:24.0:1616513068.163695:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:4.0:1616513068.163698:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70440 00000800:00000200:22.0:1616513068.163701:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513068.163702:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513068.163704:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.163706:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513068.163711:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.163715:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fdc0 00000800:00000200:22.0:1616513068.163717:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.163719:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.164289:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513068.164300:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.164308:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513068.164309:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513068.164316:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.173@tcp1 : PUT - for me 00000800:00000200:11.2:1616513068.164324:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513068.164326:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.173@tcp1 of length 224 into portal 26 MB=0x600727bd154c0 00000400:00000200:12.0:1616513068.164332:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.173@tcp1 of length 224/224 into md 0xf97c3299 [1] + 0 00000800:00000200:28.0:1616513068.164335:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513068.164337:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcebdb870 00000400:00000200:12.0:1616513068.164340:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.164342:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513068.164343:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513068.164345:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513068.164347:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcebdb870 00000800:00000200:12.0:1616513068.164350:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513068.164352:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.173@tcp1 : PUT - for me 00000800:00000200:12.0:1616513068.164353:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.164355:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513068.164360:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689341563851968 00000400:00000200:28.0:1616513068.164361:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.173@tcp1 of length 224 into portal 12 MB=0x600727bd15540 00010000:00000200:24.0:1616513068.164365:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1689341563851968/t0(0) o400->92d32fc3-9549-9f26-c28a-b395e2540e7d@10.31.140.173@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513068.164367:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.173@tcp1 of length 224/224 into md 0x865b9f31 [1] + 23120 00000400:00000200:28.0:1616513068.164372:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.164375:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:24.0:1616513068.164381:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689341563851968, offset 192 00000800:00000200:28.0:1616513068.164382:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.164385:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.164386:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.173@tcp1 00000100:00000200:8.0:1616513068.164386:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689341563852096 00000800:00000200:28.0:1616513068.164387:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513068.164388:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89680 x1689341563852096/t0(0) o400->280c8cc6-0841-3fda-cfdb-b5abcd5239d1@10.31.140.173@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513068.164392:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.173@tcp1 routed destination 00000100:00000200:8.0:1616513068.164392:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689341563852096, offset 192 00000400:00000200:8.0:1616513068.164394:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.173@tcp1 00000400:00000200:24.0:1616513068.164405:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.173@tcp1(10.31.140.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513068.164410:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.173@tcp1 routed destination 00000800:00000200:24.0:1616513068.164411:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.164416:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:8.0:1616513068.164416:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.173@tcp1(10.31.140.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.164418:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.164420:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.164420:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.164421:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.164422:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.164422:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513068.164423:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513068.164425:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513068.164428:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.164429:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513068.164429:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513068.164441:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.164445:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513068.164447:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513068.164449:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513068.164453:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513068.164457:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e990 00000800:00000200:20.0:1616513068.164460:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513068.164477:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513068.164485:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513068.164489:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513068.164490:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513068.164496:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513068.164500:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71b18 00000800:00000200:2.0:1616513068.164504:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.178086:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513068.178141:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.178150:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513068.178160:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.205@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.178169:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.205@tcp1 of length 224 into portal 26 MB=0x5ffe0ff53d940 00000400:00000200:12.0:1616513068.178175:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.205@tcp1 of length 224/224 into md 0xf97c32b9 [1] + 0 00000400:00000200:12.0:1616513068.178180:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcebdadd0 00000400:00000200:12.0:1616513068.178183:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.178185:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513068.178190:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcebdadd0 00000800:00000200:12.0:1616513068.178193:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513068.178196:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.178198:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.178200:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.178205:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513068.178209:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513068.178212:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.205@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.178217:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.205@tcp1 of length 224 into portal 12 MB=0x5ffe0ff53da80 00000400:00000200:12.0:1616513068.178223:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.205@tcp1 of length 224/224 into md 0x865b9f31 [1] + 23344 00000400:00000200:12.0:1616513068.178227:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.178229:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513068.178234:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.178236:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.178238:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513068.178241:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716704995968 00010000:00000200:8.0:1616513068.178245:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa45a00 x1688716704995968/t0(0) o400->9d993849-c088-eed4-f90c-7b9e79556204@10.31.142.205@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513068.178249:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688716704995968, offset 192 00000400:00000200:8.0:1616513068.178251:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.205@tcp1 00000400:00000200:8.0:1616513068.178254:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.205@tcp1 routed destination 00000100:00000200:24.0:1616513068.178259:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716704995648 00000400:00000200:8.0:1616513068.178259:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.205@tcp1(10.31.142.205@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.178267:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513068.178269:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:24.0:1616513068.178270:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688716704995648/t0(0) o400->996a559d-d4c2-d711-3692-68cf2ea27e21@10.31.142.205@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:8.0:1616513068.178270:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.178270:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.178272:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513068.178284:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688716704995648, offset 192 00000800:00000200:11.2:1616513068.178284:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.178290:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.205@tcp1 00000400:00000200:24.0:1616513068.178297:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.205@tcp1 routed destination 00000400:00000200:24.0:1616513068.178312:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.205@tcp1(10.31.142.205@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.178319:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.178325:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.178328:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.178330:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.178335:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513068.178338:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513068.178342:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513068.178342:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513068.178345:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513068.178347:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513068.178348:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513068.178352:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513068.178354:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513068.178356:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.178357:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513068.178358:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f706e8 00000400:00000200:22.0:1616513068.178362:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513068.178362:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513068.178366:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:22.0:1616513068.178370:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.178372:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.188513:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513068.188568:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.188577:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513068.188578:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513068.188586:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.76@tcp1 : PUT - for me 00000400:00000200:28.0:1616513068.188595:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.76@tcp1 of length 224 into portal 26 MB=0x5ffdd880ef9c0 00000400:00000200:28.0:1616513068.188602:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.76@tcp1 of length 224/224 into md 0xf97c32c9 [1] + 0 00000400:00000200:28.0:1616513068.188607:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc000 00000400:00000200:28.0:1616513068.188609:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.188612:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513068.188616:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc000 00000800:00000200:28.0:1616513068.188619:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.188622:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.188624:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513068.188677:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701819091392 00010000:00000200:24.0:1616513068.188687:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688701819091392/t0(0) o400->da915f78-0538-b6e0-a1ed-5095b6ef9bc9@10.31.142.76@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.188700:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701819091392, offset 192 00000400:00000200:24.0:1616513068.188706:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.76@tcp1 00000800:00000200:7.2:1616513068.188706:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513068.188713:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.76@tcp1 routed destination 00000400:00000200:24.0:1616513068.188727:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.76@tcp1(10.31.142.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.188733:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.188738:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.188741:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.188743:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.188747:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.188752:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.188753:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513068.188763:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513068.188764:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.188770:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513068.188773:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:4.0:1616513068.188773:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513068.188775:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513068.188779:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.188783:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.76@tcp1 : PUT - for me 00000400:00000200:4.0:1616513068.188785:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fdc0 00000800:00000200:4.0:1616513068.188800:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513068.188804:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.76@tcp1 of length 224 into portal 12 MB=0x5ffdd880efa40 00000400:00000200:18.0:1616513068.188810:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.76@tcp1 of length 224/224 into md 0xf963e789 [1] + 94176 00000400:00000200:18.0:1616513068.188815:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.188818:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:6.0F:1616513068.188820:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513068.188825:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.188828:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513068.188829:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513068.188889:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701819091520 00010000:00000200:10.0:1616513068.188901:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d4380 x1688701819091520/t0(0) o400->9e7e17ab-d4f8-ab96-4a0f-3fa16c42bc32@10.31.142.76@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513068.188915:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701819091520, offset 192 00000400:00000200:10.0:1616513068.188922:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.76@tcp1 00000400:00000200:10.0:1616513068.188929:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.76@tcp1 routed destination 00000400:00000200:10.0:1616513068.188944:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.76@tcp1(10.31.142.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513068.188950:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513068.188956:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513068.188959:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513068.188961:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.188965:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.188969:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.188980:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.188988:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.188990:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:6.0:1616513068.188992:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513068.188997:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.189001:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83dc0 00000800:00000200:22.0:1616513068.189005:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.228712:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513068.228767:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513068.228776:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:6.0:1616513068.228785:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.48@tcp1 : PUT - for me 00000400:00000200:6.0:1616513068.228794:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.48@tcp1 of length 224 into portal 26 MB=0x602bfe947a1c0 00000400:00000200:6.0:1616513068.228800:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.48@tcp1 of length 224/224 into md 0xf97c32e9 [1] + 0 00000400:00000200:6.0:1616513068.228805:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc110 00000400:00000200:6.0:1616513068.228808:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513068.228810:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:6.0:1616513068.228815:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc110 00000800:00000200:6.0:1616513068.228818:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513068.228820:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513068.228821:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:6.0:1616513068.228829:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513068.228831:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513068.228832:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513068.228841:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.48@tcp1 : PUT - for me 00000400:00000200:18.0:1616513068.228849:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.48@tcp1 of length 224 into portal 12 MB=0x602bfe947a2c0 00000400:00000200:18.0:1616513068.228856:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.48@tcp1 of length 224/224 into md 0xf963e789 [1] + 94400 00000400:00000200:18.0:1616513068.228861:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.228866:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513068.228873:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.228876:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691873136058816 00000800:00000200:18.0:1616513068.228876:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513068.228878:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513068.228886:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1691873136058816/t0(0) o400->68f19b65-b371-ed66-8724-86778a1503a2@10.31.142.48@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.228900:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691873136058816, offset 192 00000400:00000200:24.0:1616513068.228905:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.48@tcp1 00000400:00000200:24.0:1616513068.228912:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.48@tcp1 routed destination 00000400:00000200:24.0:1616513068.228926:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.48@tcp1(10.31.142.48@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.228932:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:22.0:1616513068.228935:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691873136059072 00000800:00000200:24.0:1616513068.228937:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.228940:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.228942:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.228946:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513068.228946:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d6c00 x1691873136059072/t0(0) o400->3b36fe1a-66f7-6db0-3cd6-9043901d528d@10.31.142.48@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513068.228953:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513068.228953:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513068.228960:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691873136059072, offset 192 00000400:00000200:22.0:1616513068.228966:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.48@tcp1 00000400:00000200:22.0:1616513068.228972:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.48@tcp1 routed destination 00000400:00000200:22.0:1616513068.228986:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.48@tcp1(10.31.142.48@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513068.228992:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513068.228998:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513068.229001:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513068.229002:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.229007:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513068.229009:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513068.229011:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513068.229016:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513068.229018:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513068.229024:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513068.229025:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513068.229028:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.229029:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513068.229030:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eaa0 00000800:00000200:6.0:1616513068.229032:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513068.229035:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.229037:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ac38 00000800:00000200:22.0:1616513068.229046:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513068.229046:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.229049:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513068.243199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513068.243254:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513068.243262:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513068.243264:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513068.243273:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.47@tcp1 : PUT - for me 00000800:00000200:20.0:1616513068.243275:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513068.243282:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.47@tcp1 of length 224 into portal 26 MB=0x5ffdbf5ccb500 00000800:00000200:20.0:1616513068.243283:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:18.0:1616513068.243289:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.47@tcp1 of length 224/224 into md 0xf97c32f9 [1] + 0 00000400:00000200:20.0:1616513068.243292:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.47@tcp1 : PUT - for me 00000400:00000200:18.0:1616513068.243294:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc220 00000400:00000200:18.0:1616513068.243296:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.243299:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513068.243301:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.47@tcp1 of length 224 into portal 12 MB=0x5ffdbf5ccb680 00000400:00000200:18.0:1616513068.243303:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc220 00000800:00000200:18.0:1616513068.243306:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.243308:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513068.243309:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.47@tcp1 of length 224/224 into md 0x865b9f31 [1] + 23568 00000800:00000200:18.0:1616513068.243309:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513068.243312:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513068.243314:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.243317:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513068.243324:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.243327:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513068.243329:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513068.243331:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695070307968 00000800:00000200:4.0:1616513068.243331:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513068.243337:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cecc80 x1688695070307968/t0(0) o400->d0ee3a17-2816-7cd1-8bda-b06dfabe0fc8@10.31.143.47@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513068.243342:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695070307968, offset 192 00000400:00000200:8.0:1616513068.243344:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.47@tcp1 00000400:00000200:8.0:1616513068.243347:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.47@tcp1 routed destination 00000400:00000200:8.0:1616513068.243352:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.47@tcp1(10.31.143.47@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.243355:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513068.243357:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.243358:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.243359:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.243361:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.243366:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.243372:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695070307584 00010000:00000200:24.0:1616513068.243382:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688695070307584/t0(0) o400->f09d73cb-5570-06ab-6c95-ff4118a348e8@10.31.143.47@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.243397:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695070307584, offset 192 00000400:00000200:24.0:1616513068.243403:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.47@tcp1 00000400:00000200:24.0:1616513068.243410:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.47@tcp1 routed destination 00000400:00000200:24.0:1616513068.243425:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.47@tcp1(10.31.143.47@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513068.243428:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513068.243432:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513068.243449:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513068.243452:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513068.243453:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.243456:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.243458:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513068.243460:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513068.243463:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513068.243464:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70dd0 00000800:00000200:7.2:1616513068.243467:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513068.243468:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513068.243470:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513068.243471:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:6.0:1616513068.243527:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513068.243533:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513068.243536:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513068.243543:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513068.243548:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f870 00000800:00000200:6.0:1616513068.243552:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513068.243555:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.245647:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513068.245705:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513068.245714:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513068.245723:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.183@tcp1 : PUT - for me 00000400:00000200:20.0:1616513068.245734:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.183@tcp1 of length 224 into portal 26 MB=0x5ffd4a5cc3600 00000400:00000200:20.0:1616513068.245740:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.183@tcp1 of length 224/224 into md 0xf97c3319 [1] + 0 00000400:00000200:20.0:1616513068.245745:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc330 00000400:00000200:20.0:1616513068.245747:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.245750:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513068.245754:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc330 00000800:00000200:20.0:1616513068.245757:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.245758:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.245760:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513068.245765:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513068.245768:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513068.245769:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513068.245777:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.183@tcp1 : PUT - for me 00000400:00000200:4.0:1616513068.245787:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.183@tcp1 of length 224 into portal 12 MB=0x5ffd4a5cc36c0 00000400:00000200:4.0:1616513068.245793:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.183@tcp1 of length 224/224 into md 0x865b9f31 [1] + 23792 00000400:00000200:4.0:1616513068.245799:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513068.245801:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513068.245808:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513068.245811:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513068.245812:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663663326912 00000800:00000200:4.0:1616513068.245813:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513068.245814:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8542eef080 x1688663663326912/t0(0) o400->81452e03-916a-b7e9-893c-1e033b9b7d8c@10.31.138.183@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513068.245818:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663663326720 00000100:00000200:8.0:1616513068.245818:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663663326912, offset 192 00000400:00000200:8.0:1616513068.245820:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.183@tcp1 00000400:00000200:8.0:1616513068.245822:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.183@tcp1 routed destination 00000400:00000200:8.0:1616513068.245827:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.183@tcp1(10.31.138.183@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:24.0:1616513068.245829:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688663663326720/t0(0) o400->6f644ab1-0316-27f2-fedf-674768fd8307@10.31.138.183@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:8.0:1616513068.245829:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513068.245830:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.245831:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.245832:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.245833:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.245838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.245843:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663663326720, offset 192 00000400:00000200:24.0:1616513068.245851:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.183@tcp1 00000400:00000200:24.0:1616513068.245859:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.183@tcp1 routed destination 00000400:00000200:24.0:1616513068.245874:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.183@tcp1(10.31.138.183@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.245880:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.245886:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.245888:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.245891:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513068.245893:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513068.245895:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513068.245900:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513068.245900:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.245902:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513068.245903:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513068.245904:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513068.245910:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513068.245913:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71e48 00000800:00000200:28.0:1616513068.245917:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513068.245953:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.245960:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513068.245963:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513068.245969:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513068.245975:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eee0 00000800:00000200:10.0:1616513068.245981:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.245984:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:31.0F:1616513068.346695:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923072, portal 4 00000100:00000200:31.0:1616513068.346709:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923072, offset 0 00000400:00000200:31.0:1616513068.346716:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:31.0:1616513068.346724:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:31.0:1616513068.346738:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513068.346745:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000100:00000200:30.0F:1616513068.346752:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923088, portal 4 00000800:00000200:31.0:1616513068.346753:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000100:00000200:30.0:1616513068.346756:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923088, offset 0 00000800:00000200:31.0:1616513068.346757:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000400:00000200:30.0:1616513068.346759:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:31.0:1616513068.346760:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:30.0:1616513068.346762:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:31.0:1616513068.346767:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:30.0:1616513068.346768:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.346771:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513068.346775:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:5.2:1616513068.346775:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513068.346776:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513068.346777:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:30.0:1616513068.346781:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (24)-- 00000800:00000200:4.0:1616513068.346792:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:4.0:1616513068.346800:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (24)-- 00000400:00000200:4.0:1616513068.346803:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513068.346810:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513068.346816:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded2d00 x1693067725923072/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513068.346828:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c3540 00000800:00000200:4.0:1616513068.346833:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:4.0:1616513068.346835:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513068.346839:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513068.346841:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513068.346846:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:12.0:1616513068.346847:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:4.0:1616513068.346850:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b08d100 x1693067725923088/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513068.346860:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef540 00000800:00000200:4.0:1616513068.346864:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513068.347005:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513068.347061:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513068.347071:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.230@o2ib1 00000800:00000200:12.0:1616513068.347073:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:28.0:1616513068.347080:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:12.0:1616513068.347082:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:4.0:1616513068.347082:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513068.347088:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626300 00000400:00000200:12.0:1616513068.347090:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513068.347096:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d1fb1 [1] + 192 00000400:00000200:12.0:1616513068.347098:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626310 00000400:00000200:28.0:1616513068.347103:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:12.0:1616513068.347105:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d1fc1 [1] + 192 00000400:00000200:12.0:1616513068.347110:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.347112:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded2d00 x1693067725923072/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513068.347128:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000100:00000200:12.0:1616513068.347130:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b08d100 x1693067725923088/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513068.347131:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:28.0:1616513068.347133:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513068.347144:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513068.347147:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:31.0:1616513068.347148:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded2d00 x1693067725923072/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513068.347149:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:31.0:1616513068.347161:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded2d00 x1693067725923072/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513068.347172:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2000 00000100:00000200:30.0:1616513068.347175:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b08d100 x1693067725923088/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513068.347180:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b08d100 x1693067725923088/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513068.347184:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee198 00000800:00000200:7.2:1616513068.349175:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513068.349229:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513068.349239:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:6.0:1616513068.349248:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.230@tcp1 : PUT - for me 00000400:00000200:6.0:1616513068.349257:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.230@tcp1 of length 224 into portal 26 MB=0x5ffdf143e14c0 00000400:00000200:6.0:1616513068.349264:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.230@tcp1 of length 224/224 into md 0xf97c3339 [1] + 0 00000400:00000200:6.0:1616513068.349269:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc440 00000400:00000200:6.0:1616513068.349271:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513068.349274:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:6.0:1616513068.349279:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc440 00000800:00000200:6.0:1616513068.349282:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.349285:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513068.349285:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513068.349290:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513068.349346:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708465956032 00010000:00000200:24.0:1616513068.349358:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688708465956032/t0(0) o400->87517747-70fe-2a9d-2ff6-2d2bfbb0bf49@10.31.141.230@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.349372:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688708465956032, offset 192 00000400:00000200:24.0:1616513068.349379:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.230@tcp1 00000400:00000200:24.0:1616513068.349386:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.230@tcp1 routed destination 00000400:00000200:24.0:1616513068.349401:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.230@tcp1(10.31.141.230@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.349408:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.349414:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.349416:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.349419:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.349424:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513068.349431:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513068.349452:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513068.349464:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.349473:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513068.349481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513068.349481:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.230@tcp1 : PUT - for me 00000400:00000200:10.0:1616513068.349490:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.230@tcp1 of length 224 into portal 12 MB=0x5ffdf143e1580 00000400:00000200:10.0:1616513068.349496:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.230@tcp1 of length 224/224 into md 0xf963e789 [1] + 94624 00000400:00000200:10.0:1616513068.349502:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513068.349505:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513068.349512:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.349515:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513068.349516:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.349519:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513068.349536:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708465956224 00000800:00000200:20.0:1616513068.349539:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.349546:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:22.0:1616513068.349548:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bd9f80 x1688708465956224/t0(0) o400->0259a8c9-5adb-bf6d-d724-7946afd221c3@10.31.141.230@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513068.349549:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513068.349549:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513068.349555:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:22.0:1616513068.349561:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688708465956224, offset 192 00000400:00000200:20.0:1616513068.349561:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e2a8 00000800:00000200:20.0:1616513068.349564:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513068.349568:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.230@tcp1 00000400:00000200:22.0:1616513068.349577:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.230@tcp1 routed destination 00000400:00000200:22.0:1616513068.349592:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.230@tcp1(10.31.141.230@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513068.349598:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513068.349604:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513068.349607:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513068.349609:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.349614:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.349618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.349629:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.349634:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.349636:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:6.0:1616513068.349639:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513068.349642:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.349645:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ac38 00000800:00000200:22.0:1616513068.349647:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.392442:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513068.392499:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.392509:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513068.392518:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.39@tcp1 : PUT - for me 00000400:00000200:28.0:1616513068.392527:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.39@tcp1 of length 224 into portal 26 MB=0x604a297821740 00000400:00000200:28.0:1616513068.392534:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.39@tcp1 of length 224/224 into md 0xf97c3349 [1] + 0 00000400:00000200:28.0:1616513068.392538:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc550 00000400:00000200:28.0:1616513068.392541:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.392544:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513068.392548:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc550 00000800:00000200:28.0:1616513068.392551:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513068.392554:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513068.392556:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.392558:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.392563:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513068.392565:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513068.392570:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.39@tcp1 : PUT - for me 00000400:00000200:28.0:1616513068.392575:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.39@tcp1 of length 224 into portal 12 MB=0x604a2978219c0 00000400:00000200:28.0:1616513068.392581:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.39@tcp1 of length 224/224 into md 0x865b9f31 [1] + 24016 00000400:00000200:28.0:1616513068.392585:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.392587:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513068.392592:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.392594:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.392596:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513068.392603:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946233362880 00010000:00000200:8.0:1616513068.392608:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8e300 x1693946233362880/t0(0) o400->3c4b78fb-f921-cb2f-c062-39a9cca0de60@10.31.141.39@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513068.392611:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946233362240 00000100:00000200:8.0:1616513068.392613:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946233362880, offset 192 00000400:00000200:8.0:1616513068.392616:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.39@tcp1 00000400:00000200:8.0:1616513068.392619:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.39@tcp1 routed destination 00010000:00000200:24.0:1616513068.392622:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1693946233362240/t0(0) o400->f267e3ee-058d-2b3c-8177-ac8ff7d8f1ff@10.31.141.39@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513068.392624:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.39@tcp1(10.31.141.39@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.392627:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513068.392629:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.392630:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.392631:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.392633:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513068.392636:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946233362240, offset 192 00000400:00000200:24.0:1616513068.392642:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.39@tcp1 00000800:00000200:11.2:1616513068.392643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.392649:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.39@tcp1 routed destination 00000800:00000200:20.0:1616513068.392658:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513068.392664:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.39@tcp1(10.31.141.39@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513068.392665:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513068.392667:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513068.392668:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513068.392674:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513068.392675:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513068.392678:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71f58 00000800:00000200:24.0:1616513068.392681:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513068.392682:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513068.392683:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.392686:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.392690:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513068.392697:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513068.392697:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.392712:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.392718:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513068.392721:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513068.392727:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513068.392732:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e330 00000800:00000200:10.0:1616513068.392735:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.392738:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.460492:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513068.460550:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513068.460560:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:6.0:1616513068.460569:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.66@tcp1 : PUT - for me 00000400:00000200:6.0:1616513068.460578:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.66@tcp1 of length 224 into portal 26 MB=0x601f2e741d340 00000400:00000200:6.0:1616513068.460584:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.66@tcp1 of length 224/224 into md 0xf97c3359 [1] + 0 00000400:00000200:6.0:1616513068.460589:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc660 00000400:00000200:6.0:1616513068.460592:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513068.460595:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:6.0:1616513068.460599:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc660 00000800:00000200:6.0:1616513068.460602:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.460604:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513068.460605:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513068.460611:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513068.460614:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.460615:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513068.460624:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.66@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.460632:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.66@tcp1 of length 224 into portal 12 MB=0x601f2e741d3c0 00000400:00000200:22.0:1616513068.460639:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.66@tcp1 of length 224/224 into md 0xf963e789 [1] + 94848 00000400:00000200:22.0:1616513068.460644:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513068.460647:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513068.460655:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.460658:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.460660:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513068.460664:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690992633828288 00000100:00000200:24.0:1616513068.460666:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690992633828160 00010000:00000200:30.0:1616513068.460669:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b088480 x1690992633828288/t0(0) o400->b9e6dc2e-9163-8e54-2062-214a205e3a1f@10.31.141.66@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513068.460675:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690992633828288, offset 192 00000400:00000200:30.0:1616513068.460677:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.66@tcp1 00010000:00000200:24.0:1616513068.460677:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1690992633828160/t0(0) o400->917a0c4c-a4b4-2707-ab03-5495bece49f9@10.31.141.66@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:30.0:1616513068.460680:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.66@tcp1 routed destination 00000400:00000200:30.0:1616513068.460685:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.66@tcp1(10.31.141.66@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.460688:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.460690:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.460691:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513068.460691:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690992633828160, offset 192 00000800:00000200:30.0:1616513068.460692:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513068.460694:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.460697:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.66@tcp1 00000400:00000200:24.0:1616513068.460705:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.66@tcp1 routed destination 00000800:00000200:11.2:1616513068.460705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.460720:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.66@tcp1(10.31.141.66@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.460726:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.460732:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.460735:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.460737:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.460742:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.460746:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.460749:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513068.460762:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513068.460769:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513068.460772:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513068.460778:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513068.460785:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee198 00000800:00000200:28.0:1616513068.460789:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.460792:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513068.460807:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.460815:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513068.460817:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:6.0:1616513068.460818:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513068.460824:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.460830:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f4b8 00000800:00000200:18.0:1616513068.460833:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.465110:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513068.465169:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.465179:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513068.465188:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.241@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.465197:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.241@tcp1 of length 224 into portal 26 MB=0x5ffe39600c140 00000400:00000200:12.0:1616513068.465203:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.241@tcp1 of length 224/224 into md 0xf97c3379 [1] + 0 00000400:00000200:12.0:1616513068.465208:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc770 00000400:00000200:12.0:1616513068.465211:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.465213:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513068.465218:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc770 00000800:00000200:12.0:1616513068.465221:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.465223:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513068.465223:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.465232:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.465235:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.465236:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513068.465244:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.241@tcp1 : PUT - for me 00000400:00000200:20.0:1616513068.465253:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.241@tcp1 of length 224 into portal 12 MB=0x5ffe39600c400 00000400:00000200:20.0:1616513068.465259:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.241@tcp1 of length 224/224 into md 0x865b9f31 [1] + 24240 00000400:00000200:20.0:1616513068.465264:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.465267:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513068.465274:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513068.465277:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.465279:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513068.465280:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727822844224 00000100:00000200:8.0:1616513068.465282:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727822844928 00010000:00000200:8.0:1616513068.465287:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ced580 x1688727822844928/t0(0) o400->f644fd8e-6b2b-20b8-a056-d4e4fd3da97e@10.31.141.241@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513068.465291:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688727822844224/t0(0) o400->13b8b99c-1e8b-8f29-fb22-54925458153e@10.31.141.241@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513068.465292:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688727822844928, offset 192 00000400:00000200:8.0:1616513068.465295:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.241@tcp1 00000400:00000200:8.0:1616513068.465297:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.241@tcp1 routed destination 00000400:00000200:8.0:1616513068.465303:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.241@tcp1(10.31.141.241@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513068.465305:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688727822844224, offset 192 00000800:00000200:8.0:1616513068.465305:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513068.465308:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.465309:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.465309:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.465311:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.241@tcp1 00000800:00000200:8.0:1616513068.465312:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.465316:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.465318:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.241@tcp1 routed destination 00000400:00000200:24.0:1616513068.465333:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.241@tcp1(10.31.141.241@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.465340:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.465345:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.465348:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.465352:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.465357:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513068.465363:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513068.465363:0:48214:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513068.465371:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.465378:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513068.465381:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513068.465382:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513068.465388:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513068.465392:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71100 00000800:00000200:4.0:1616513068.465396:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513068.465418:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.465426:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.465428:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513068.465449:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.465454:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f6d8 00000800:00000200:22.0:1616513068.465458:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.465461:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.493643:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513068.493700:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513068.493711:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513068.493721:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.26@tcp1 : PUT - for me 00000400:00000200:20.0:1616513068.493731:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.26@tcp1 of length 224 into portal 26 MB=0x604a2d9fb6240 00000400:00000200:20.0:1616513068.493738:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.26@tcp1 of length 224/224 into md 0xf97c3399 [1] + 0 00000400:00000200:20.0:1616513068.493744:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc880 00000400:00000200:20.0:1616513068.493747:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.493751:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513068.493757:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc880 00000800:00000200:28.0:1616513068.493759:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.493761:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.493764:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.493766:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513068.493786:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947348607552 00010000:00000200:24.0:1616513068.493799:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1693947348607552/t0(0) o400->4e3993a5-b0e8-5bd2-c967-52724bc6e2ca@10.31.141.26@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.493816:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693947348607552, offset 192 00000400:00000200:24.0:1616513068.493822:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.26@tcp1 00000400:00000200:24.0:1616513068.493831:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.26@tcp1 routed destination 00000400:00000200:24.0:1616513068.493847:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.26@tcp1(10.31.141.26@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.493854:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.493860:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.493863:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.493865:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.493872:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.493876:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.493880:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513068.493936:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513068.493945:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513068.493949:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513068.493955:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513068.493962:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ecc0 00000800:00000200:12.0:1616513068.493965:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.493968:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513068.494037:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513068.494048:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.494056:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513068.494058:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513068.494066:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.26@tcp1 : PUT - for me 00000400:00000200:28.0:1616513068.494075:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.26@tcp1 of length 224 into portal 12 MB=0x604a2d9fb6d80 00000400:00000200:28.0:1616513068.494082:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.26@tcp1 of length 224/224 into md 0x865b9f31 [1] + 24464 00000400:00000200:28.0:1616513068.494087:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.494090:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513068.494098:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.494101:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.494103:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513068.494111:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947348610432 00010000:00000200:8.0:1616513068.494118:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8bf00 x1693947348610432/t0(0) o400->25493809-03e1-ea02-c339-21022cfedf15@10.31.141.26@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513068.494125:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693947348610432, offset 192 00000400:00000200:8.0:1616513068.494127:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.26@tcp1 00000400:00000200:8.0:1616513068.494131:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.26@tcp1 routed destination 00000400:00000200:8.0:1616513068.494137:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.26@tcp1(10.31.141.26@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.494140:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513068.494143:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.494144:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513068.494145:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513068.494149:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.494155:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513068.494216:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513068.494225:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513068.494227:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513068.494234:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513068.494238:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71c28 00000800:00000200:6.0:1616513068.494242:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513068.494245:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.499259:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513068.499317:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.499326:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513068.499335:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.129@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.499345:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.129@tcp1 of length 224 into portal 26 MB=0x5ffd8ff8a8140 00000400:00000200:12.0:1616513068.499352:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.129@tcp1 of length 224/224 into md 0xf97c33b9 [1] + 0 00000400:00000200:12.0:1616513068.499356:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc990 00000400:00000200:12.0:1616513068.499360:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.499363:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513068.499367:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc990 00000800:00000200:4.0:1616513068.499370:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.499371:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513068.499374:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.499376:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513068.499465:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688682348839232 00010000:00000200:24.0:1616513068.499479:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688682348839232/t0(0) o400->a6302840-e34d-63a3-77ea-0a75cbcbf62b@10.31.132.129@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.499495:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688682348839232, offset 192 00000400:00000200:24.0:1616513068.499502:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.129@tcp1 00000400:00000200:24.0:1616513068.499511:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.129@tcp1 routed destination 00000400:00000200:24.0:1616513068.499526:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.129@tcp1(10.31.132.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.499534:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.499541:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.499544:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.499546:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.499552:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.499557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.499560:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513068.499608:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.499615:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513068.499618:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513068.499624:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513068.499630:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f980 00000800:00000200:20.0:1616513068.499634:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.499637:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513068.499994:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513068.500006:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513068.500014:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513068.500016:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513068.500023:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.129@tcp1 : PUT - for me 00000400:00000200:4.0:1616513068.500033:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.129@tcp1 of length 224 into portal 12 MB=0x5ffd8ff8a8300 00000400:00000200:4.0:1616513068.500039:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.129@tcp1 of length 224/224 into md 0x865b9f31 [1] + 24688 00000400:00000200:4.0:1616513068.500045:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513068.500048:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513068.500055:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513068.500058:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513068.500060:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513068.500066:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688682348839680 00010000:00000200:8.0:1616513068.500072:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8542eebf00 x1688682348839680/t0(0) o400->fd20ecc0-0a5f-1c97-7614-81d8835c64a4@10.31.132.129@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513068.500079:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688682348839680, offset 192 00000400:00000200:8.0:1616513068.500082:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.129@tcp1 00000400:00000200:8.0:1616513068.500085:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.129@tcp1 routed destination 00000400:00000200:8.0:1616513068.500091:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.129@tcp1(10.31.132.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.500093:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513068.500096:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.500098:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513068.500099:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513068.500102:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.500110:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.500169:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.500176:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.500181:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513068.500187:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.500191:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f718f8 00000800:00000200:22.0:1616513068.500195:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.500198:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.500204:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513068.500206:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.500208:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.500210:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.500212:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.501805:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513068.501874:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513068.501884:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513068.501893:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.198@tcp1 : PUT - for me 00000400:00000200:18.0:1616513068.501903:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.198@tcp1 of length 224 into portal 26 MB=0x600af400276c0 00000400:00000200:18.0:1616513068.501910:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.198@tcp1 of length 224/224 into md 0xf97c33d9 [1] + 0 00000400:00000200:18.0:1616513068.501915:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fccaa0 00000400:00000200:18.0:1616513068.501918:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.501921:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513068.501925:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccaa0 00000800:00000200:18.0:1616513068.501928:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.501930:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.501931:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.501940:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513068.501940:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513068.501943:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513068.501949:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.198@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.501958:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.198@tcp1 of length 224 into portal 12 MB=0x600af400277c0 00000400:00000200:22.0:1616513068.501964:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.198@tcp1 of length 224/224 into md 0xf963e789 [1] + 95072 00000400:00000200:22.0:1616513068.501970:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513068.501973:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513068.501981:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.501985:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.501987:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513068.501992:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689602553444032 00000100:00000200:30.0:1616513068.501994:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689602553444288 00010000:00000200:30.0:1616513068.502000:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08d100 x1689602553444288/t0(0) o400->8c02b3ac-a21e-df50-a948-d6cdcbb08c8c@10.31.140.198@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513068.502003:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1689602553444032/t0(0) o400->8c615476-e25d-8bc8-f09a-d25c10d0cf70@10.31.140.198@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513068.502007:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689602553444288, offset 192 00000400:00000200:30.0:1616513068.502010:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.198@tcp1 00000400:00000200:30.0:1616513068.502013:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.198@tcp1 routed destination 00000400:00000200:30.0:1616513068.502019:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.198@tcp1(10.31.140.198@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513068.502020:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689602553444032, offset 192 00000800:00000200:30.0:1616513068.502022:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.502025:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.502026:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:24.0:1616513068.502026:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.198@tcp1 00000800:00000200:30.0:1616513068.502027:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513068.502030:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.502033:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.198@tcp1 routed destination 00000800:00000200:11.2:1616513068.502036:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.502049:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.198@tcp1(10.31.140.198@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.502055:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.502062:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.502065:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.502067:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.502072:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.502076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.502079:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513068.502089:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:6.0:1616513068.502091:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513068.502096:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513068.502098:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:6.0:1616513068.502099:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513068.502102:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513068.502103:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513068.502105:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:6.0:1616513068.502109:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513068.502110:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee770 00000400:00000200:6.0:1616513068.502115:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fd38 00000800:00000200:6.0:1616513068.502119:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513068.502127:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.502129:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513068.504488:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513068.504543:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.504551:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513068.504560:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.87@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.504570:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.87@tcp1 of length 224 into portal 26 MB=0x6014fb74aa7c0 00000400:00000200:12.0:1616513068.504577:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.87@tcp1 of length 224/224 into md 0xf97c33e9 [1] + 0 00000400:00000200:12.0:1616513068.504582:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fccbb0 00000400:00000200:12.0:1616513068.504584:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.504587:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513068.504592:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccbb0 00000800:00000200:12.0:1616513068.504595:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.504596:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.504597:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.504600:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513068.504615:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690291749431232 00010000:00000200:24.0:1616513068.504627:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1690291749431232/t0(0) o400->a31647ba-403b-ed89-e4e0-ee49bd2e0b93@10.31.140.87@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.504641:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690291749431232, offset 192 00000400:00000200:24.0:1616513068.504647:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.87@tcp1 00000400:00000200:24.0:1616513068.504656:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.87@tcp1 routed destination 00000400:00000200:24.0:1616513068.504670:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.87@tcp1(10.31.140.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.504677:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.504683:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.504686:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.504688:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.504693:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.504698:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.504701:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513068.504750:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.504756:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513068.504759:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513068.504765:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513068.504771:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e908 00000800:00000200:20.0:1616513068.504775:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.504778:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513068.504963:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513068.505018:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.505028:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513068.505037:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.87@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.505046:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.87@tcp1 of length 224 into portal 12 MB=0x6014fb74ab500 00000400:00000200:22.0:1616513068.505053:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.87@tcp1 of length 224/224 into md 0xf963e789 [1] + 95296 00000400:00000200:22.0:1616513068.505058:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513068.505061:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513068.505070:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.505073:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.505075:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:30.0:1616513068.505077:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690291749434624 00000800:00000200:10.0:1616513068.505078:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:30.0:1616513068.505081:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5580 x1690291749434624/t0(0) o400->7c5ba746-8e4b-06b0-d914-c482edc9ff2f@10.31.140.87@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513068.505086:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690291749434624, offset 192 00000400:00000200:30.0:1616513068.505088:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.87@tcp1 00000400:00000200:30.0:1616513068.505090:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.87@tcp1 routed destination 00000400:00000200:30.0:1616513068.505096:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.87@tcp1(10.31.140.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.505098:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513068.505101:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.505102:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513068.505102:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513068.505104:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.505110:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513068.505165:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.505173:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513068.505176:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513068.505177:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513068.505183:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.505187:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efd38 00000800:00000200:18.0:1616513068.505191:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.519033:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513068.519092:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513068.519103:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513068.519108:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513068.519113:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.147@tcp1 : PUT - for me 00000400:00000200:4.0:1616513068.519123:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.147@tcp1 of length 224 into portal 12 MB=0x60540e8502a80 00000400:00000200:4.0:1616513068.519129:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.147@tcp1 of length 224/224 into md 0x865b9f31 [1] + 24912 00000400:00000200:4.0:1616513068.519134:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513068.519137:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513068.519145:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513068.519149:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513068.519150:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513068.519161:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694626193877632 00010000:00000200:8.0:1616513068.519169:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8542ee9f80 x1694626193877632/t0(0) o400->f5292d1d-0d2d-2cef-769a-d3683a040167@10.31.143.147@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513068.519176:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694626193877632, offset 192 00000400:00000200:8.0:1616513068.519179:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.147@tcp1 00000400:00000200:8.0:1616513068.519183:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.147@tcp1 routed destination 00000400:00000200:8.0:1616513068.519190:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.147@tcp1(10.31.143.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513068.519190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513068.519193:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513068.519196:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.519197:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.519198:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.519202:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.519207:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513068.519247:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.519257:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513068.519261:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513068.519266:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.147@tcp1 : PUT - for me 00000800:00000200:20.0:1616513068.519268:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513068.519271:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513068.519277:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.519278:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.147@tcp1 of length 224 into portal 26 MB=0x60540e8502a00 00000400:00000200:20.0:1616513068.519281:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f718f8 00000800:00000200:20.0:1616513068.519285:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513068.519286:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.147@tcp1 of length 224/224 into md 0xf97c3419 [1] + 0 00000800:00000200:20.0:1616513068.519288:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513068.519291:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcccc0 00000400:00000200:22.0:1616513068.519294:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513068.519297:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:10.0:1616513068.519301:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513068.519302:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcccc0 00000800:00000200:22.0:1616513068.519305:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.519308:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.519310:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513068.519370:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694626193877504 00010000:00000200:24.0:1616513068.519382:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1694626193877504/t0(0) o400->64eeef89-d2db-436b-b406-9f165c00d683@10.31.143.147@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.519396:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694626193877504, offset 192 00000400:00000200:24.0:1616513068.519401:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.147@tcp1 00000400:00000200:24.0:1616513068.519408:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.147@tcp1 routed destination 00000400:00000200:24.0:1616513068.519425:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.147@tcp1(10.31.143.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.519431:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.519450:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.519452:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.519454:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.519460:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.519464:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.519467:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513068.519474:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.519480:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513068.519483:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513068.519489:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513068.519495:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f210 00000800:00000200:10.0:1616513068.519499:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.519501:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.527558:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513068.527617:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.527629:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513068.527639:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.189@tcp1 : PUT - for me 00000400:00000200:10.0:1616513068.527651:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.189@tcp1 of length 224 into portal 26 MB=0x605a073b19ac0 00000400:00000200:10.0:1616513068.527665:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.189@tcp1 of length 224/224 into md 0xf97c3429 [1] + 0 00000400:00000200:10.0:1616513068.527670:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fccdd0 00000400:00000200:10.0:1616513068.527674:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513068.527677:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513068.527678:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:10.0:1616513068.527683:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccdd0 00000800:00000200:10.0:1616513068.527687:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513068.527690:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513068.527690:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:10.0:1616513068.527692:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513068.527696:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513068.527700:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.189@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.527709:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.189@tcp1 of length 224 into portal 12 MB=0x605a073b19c40 00000400:00000200:22.0:1616513068.527716:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.189@tcp1 of length 224/224 into md 0xf963e789 [1] + 95520 00000400:00000200:22.0:1616513068.527722:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513068.527725:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513068.527733:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.527737:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.527739:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513068.527749:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036554189888 00000100:00000200:24.0:1616513068.527752:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036554189504 00010000:00000200:30.0:1616513068.527756:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad2d00 x1695036554189888/t0(0) o400->63067e44-3dd9-62ee-f96c-910b9469c2b0@10.31.129.189@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513068.527762:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1695036554189504/t0(0) o400->1ecf14bb-7fda-b60f-c27e-874491adedfe@10.31.129.189@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513068.527763:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036554189888, offset 192 00000400:00000200:30.0:1616513068.527765:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.189@tcp1 00000400:00000200:30.0:1616513068.527769:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.189@tcp1 routed destination 00000400:00000200:30.0:1616513068.527775:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.189@tcp1(10.31.129.189@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513068.527776:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036554189504, offset 192 00000800:00000200:30.0:1616513068.527778:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.527781:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513068.527782:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.189@tcp1 00000800:00000200:30.0:1616513068.527783:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513068.527784:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513068.527788:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.527788:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.189@tcp1 routed destination 00000800:00000200:11.2:1616513068.527794:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.527801:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.189@tcp1(10.31.129.189@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.527807:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.527813:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.527816:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.527817:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.527822:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.527826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.527829:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513068.527850:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513068.527860:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513068.527863:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513068.527869:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513068.527873:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:12.0:1616513068.527878:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513068.527881:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513068.527881:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513068.527889:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513068.527892:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513068.527893:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513068.527899:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.527904:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ef68 00000800:00000200:18.0:1616513068.527908:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.537590:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513068.537650:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513068.537661:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513068.537671:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.131@tcp1 : PUT - for me 00000400:00000200:20.0:1616513068.537692:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.131@tcp1 of length 224 into portal 26 MB=0x603fc0b788b40 00000400:00000200:20.0:1616513068.537700:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.131@tcp1 of length 224/224 into md 0xf97c3449 [1] + 0 00000400:00000200:20.0:1616513068.537705:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fccee0 00000400:00000200:20.0:1616513068.537708:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.537711:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513068.537712:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513068.537716:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccee0 00000800:00000200:20.0:1616513068.537720:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513068.537723:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513068.537724:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513068.537725:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513068.537728:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513068.537733:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.131@tcp1 : PUT - for me 00000400:00000200:28.0:1616513068.537743:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.131@tcp1 of length 224 into portal 12 MB=0x603fc0b788bc0 00000400:00000200:28.0:1616513068.537750:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.131@tcp1 of length 224/224 into md 0x865b9f31 [1] + 25136 00000400:00000200:28.0:1616513068.537755:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.537758:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513068.537767:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.537770:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.537772:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513068.537780:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230919355328 00000100:00000200:24.0:1616513068.537781:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230919355200 00010000:00000200:8.0:1616513068.537786:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8c380 x1693230919355328/t0(0) o400->d78e59ac-09d8-4c07-3307-9b2947cd480e@10.31.138.131@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513068.537791:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1693230919355200/t0(0) o400->8675b3c0-bde2-90f0-e97d-33b54491ca83@10.31.138.131@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513068.537792:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230919355328, offset 192 00000400:00000200:8.0:1616513068.537795:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.131@tcp1 00000400:00000200:8.0:1616513068.537798:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.131@tcp1 routed destination 00000400:00000200:8.0:1616513068.537803:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.131@tcp1(10.31.138.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513068.537804:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230919355200, offset 192 00000800:00000200:8.0:1616513068.537806:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513068.537809:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.131@tcp1 00000800:00000200:8.0:1616513068.537809:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.537810:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.537811:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.537813:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.537815:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.131@tcp1 routed destination 00000800:00000200:11.2:1616513068.537818:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.537828:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.131@tcp1(10.31.138.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.537834:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.537839:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.537844:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.537846:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.537850:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.537856:0:48223:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.537857:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513068.537875:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.537883:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513068.537887:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513068.537887:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513068.537894:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513068.537898:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71c28 00000800:00000200:4.0:1616513068.537902:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513068.537910:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.537916:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.537919:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513068.537925:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.537931:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:22.0:1616513068.537934:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.537937:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513068.538483:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923104, portal 4 00000100:00000200:30.0:1616513068.538485:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923104, offset 0 00000400:00000200:30.0:1616513068.538487:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:30.0:1616513068.538490:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:30.0:1616513068.538495:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.538497:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:30.0:1616513068.538500:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.538502:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:30.0:1616513068.538502:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:30.0:1616513068.538505:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2F:1616513068.538553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0F:1616513068.538575:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513068.538582:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:5.0F:1616513068.538583:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513068.538585:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513068.538593:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513068.538602:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5772fa7500 x1693067725923104/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513068.538616:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee088 00000800:00000200:1.0:1616513068.538622:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513068.538747:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513068.538757:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513068.538763:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513068.538764:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513068.538773:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513068.538783:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626320 00000400:00000200:1.0:1616513068.538791:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff6a5 [1] + 192 00000400:00000200:1.0:1616513068.538797:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513068.538801:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5772fa7500 x1693067725923104/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513068.538816:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:30.0:1616513068.538818:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5772fa7500 x1693067725923104/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513068.538819:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513068.538821:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:30.0:1616513068.538822:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5772fa7500 x1693067725923104/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513068.538826:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef8f8 00000800:00000200:7.2:1616513068.544526:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513068.544533:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513068.544598:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513068.544604:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513068.544611:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:28.0:1616513068.544615:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:10.0:1616513068.544621:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.115@tcp1 : PUT - for me 00000400:00000200:28.0:1616513068.544625:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.115@tcp1 : PUT - for me 00000400:00000200:10.0:1616513068.544632:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.115@tcp1 of length 224 into portal 26 MB=0x5ffde2278c600 00000400:00000200:28.0:1616513068.544635:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.115@tcp1 of length 224 into portal 12 MB=0x5ffde2278c6c0 00000400:00000200:10.0:1616513068.544640:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.115@tcp1 of length 224/224 into md 0xf97c3469 [1] + 0 00000400:00000200:28.0:1616513068.544643:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.115@tcp1 of length 224/224 into md 0x865b9f31 [1] + 25360 00000400:00000200:10.0:1616513068.544645:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fccff0 00000400:00000200:28.0:1616513068.544648:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:10.0:1616513068.544648:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.544651:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:10.0:1616513068.544652:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513068.544656:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513068.544657:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccff0 00000800:00000200:10.0:1616513068.544661:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513068.544662:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513068.544664:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513068.544666:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:10.0:1616513068.544666:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513068.544668:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.544672:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513068.544678:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704409716416 00010000:00000200:8.0:1616513068.544686:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8d100 x1688704409716416/t0(0) o400->98e0634c-e0e8-2683-c9e4-aba596f13daa@10.31.142.115@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513068.544696:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704409716416, offset 192 00000400:00000200:8.0:1616513068.544698:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.115@tcp1 00000400:00000200:8.0:1616513068.544701:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.115@tcp1 routed destination 00000400:00000200:8.0:1616513068.544707:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.115@tcp1(10.31.142.115@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.544709:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513068.544712:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.544713:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.544715:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.544718:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513068.544723:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704409716224 00000800:00000200:11.2:1616513068.544723:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513068.544734:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688704409716224/t0(0) o400->1642b296-e008-560e-da2d-c08fe3f4e2cc@10.31.142.115@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.544747:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704409716224, offset 192 00000400:00000200:24.0:1616513068.544752:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.115@tcp1 00000400:00000200:24.0:1616513068.544759:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.115@tcp1 routed destination 00000800:00000200:20.0:1616513068.544771:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513068.544772:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.115@tcp1(10.31.142.115@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.544778:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513068.544780:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513068.544782:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513068.544784:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.544786:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.544788:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513068.544791:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513068.544793:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513068.544795:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71100 00000800:00000200:7.2:1616513068.544798:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.544799:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513068.544800:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.544802:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513068.544860:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.544868:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.544870:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513068.544877:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.544883:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e220 00000800:00000200:22.0:1616513068.544886:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.544889:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.578705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513068.578763:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.578773:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513068.578783:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.84@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.578793:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.84@tcp1 of length 224 into portal 26 MB=0x6036786a15cc0 00000400:00000200:12.0:1616513068.578799:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.84@tcp1 of length 224/224 into md 0xf97c3479 [1] + 0 00000400:00000200:12.0:1616513068.578804:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcd100 00000400:00000200:12.0:1616513068.578808:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.578810:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513068.578816:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd100 00000800:00000200:12.0:1616513068.578819:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513068.578822:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513068.578823:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513068.578830:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.578833:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.578834:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513068.578843:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.84@tcp1 : PUT - for me 00000400:00000200:28.0:1616513068.578852:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.84@tcp1 of length 224 into portal 12 MB=0x6036786a15ec0 00000400:00000200:28.0:1616513068.578859:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.84@tcp1 of length 224/224 into md 0x865b9f31 [1] + 25584 00000400:00000200:28.0:1616513068.578864:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.578867:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513068.578874:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.578878:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.578880:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513068.578881:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692593035500736 00000100:00000200:8.0:1616513068.578885:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692593035501248 00010000:00000200:8.0:1616513068.578892:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89200 x1692593035501248/t0(0) o400->bcd57cf9-80ae-f4eb-0e84-230c64ec1506@10.31.142.84@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513068.578893:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1692593035500736/t0(0) o400->7a1067d1-7f28-1b25-b1c5-1e05769c118e@10.31.142.84@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513068.578898:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692593035501248, offset 192 00000400:00000200:8.0:1616513068.578901:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.84@tcp1 00000400:00000200:8.0:1616513068.578904:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.84@tcp1 routed destination 00000100:00000200:24.0:1616513068.578906:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692593035500736, offset 192 00000400:00000200:8.0:1616513068.578909:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.84@tcp1(10.31.142.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513068.578911:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.84@tcp1 00000800:00000200:8.0:1616513068.578912:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513068.578915:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.578916:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.578917:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.578918:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.84@tcp1 routed destination 00000800:00000200:8.0:1616513068.578920:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.578925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.578932:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.84@tcp1(10.31.142.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.578938:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.578943:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513068.578944:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513068.578945:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.578947:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513068.578951:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513068.578952:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513068.578954:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513068.578955:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513068.578959:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513068.578959:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513068.578961:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513068.578965:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71f58 00000800:00000200:4.0:1616513068.578969:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513068.579013:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.579021:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513068.579024:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513068.579030:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.579036:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e4c8 00000800:00000200:18.0:1616513068.579040:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513068.579043:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513068.586518:0:6484:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923120, portal 4 00000100:00000200:20.0:1616513068.586524:0:6484:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923120, offset 0 00000400:00000200:20.0:1616513068.586530:0:6484:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:20.0:1616513068.586536:0:6484:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:20.0:1616513068.586549:0:6484:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513068.586555:0:6484:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:20.0:1616513068.586561:0:6484:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513068.586564:0:6484:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:20.0:1616513068.586566:0:6484:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:20.0:1616513068.586572:0:6484:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2F:1616513068.586619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513068.586680:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513068.586687:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513068.586689:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513068.586696:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513068.586702:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cbeedb180 x1693067725923120/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513068.586714:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23c28 00000800:00000200:22.0:1616513068.586719:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513068.586721:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513068.586799:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513068.586854:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513068.586863:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513068.586872:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513068.586880:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626330 00000400:00000200:18.0:1616513068.586887:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c3a99 [1] + 192 00000400:00000200:18.0:1616513068.586892:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.586897:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cbeedb180 x1693067725923120/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:10.0:1616513068.586907:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513068.586911:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513068.586914:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513068.586916:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:20.0:1616513068.586920:0:6484:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cbeedb180 x1693067725923120/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513068.586933:0:6484:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cbeedb180 x1693067725923120/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513068.586943:0:6484:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23188 00000100:00000200:16.0:1616513068.602618:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923136, portal 4 00000100:00000200:16.0:1616513068.602623:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923136, offset 0 00000400:00000200:16.0:1616513068.602630:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:16.0:1616513068.602636:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:16.0:1616513068.602649:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513068.602656:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:16.0:1616513068.602662:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513068.602665:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:16.0:1616513068.602668:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:16.0:1616513068.602673:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2F:1616513068.602685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513068.602741:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513068.602753:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:31.0:1616513068.602756:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:31.0:1616513068.602763:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:31.0:1616513068.602771:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d846aa40d80 x1693067725923136/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513068.602783:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43dc0 00000800:00000200:31.0:1616513068.602788:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:31.0:1616513068.602791:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513068.602849:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0F:1616513068.602863:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513068.602873:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513068.602874:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513068.602883:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513068.602892:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626340 00000400:00000200:27.0:1616513068.602899:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164b8d [1] + 192 00000400:00000200:27.0:1616513068.602904:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513068.602911:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d846aa40d80 x1693067725923136/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513068.602927:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513068.602930:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513068.602932:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:16.0:1616513068.602935:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d846aa40d80 x1693067725923136/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513068.602948:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d846aa40d80 x1693067725923136/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513068.602959:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42220 00000800:00000200:11.2:1616513068.610633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513068.610689:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.610698:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513068.610708:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.110@tcp1 : PUT - for me 00000400:00000200:28.0:1616513068.610717:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.110@tcp1 of length 224 into portal 26 MB=0x5ffdee3e1adc0 00000400:00000200:28.0:1616513068.610724:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.110@tcp1 of length 224/224 into md 0xf97c3489 [1] + 0 00000400:00000200:28.0:1616513068.610729:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcd210 00000400:00000200:28.0:1616513068.610731:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.610734:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513068.610739:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd210 00000800:00000200:28.0:1616513068.610742:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.610743:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.610744:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513068.610748:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513068.610802:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707654593984 00010000:00000200:24.0:1616513068.610813:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688707654593984/t0(0) o400->578a9b81-4524-f0c5-91dd-a6a59ff659b0@10.31.132.110@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.610827:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688707654593984, offset 192 00000400:00000200:24.0:1616513068.610834:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.110@tcp1 00000400:00000200:24.0:1616513068.610841:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.110@tcp1 routed destination 00000400:00000200:24.0:1616513068.610856:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.110@tcp1(10.31.132.110@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.610863:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.610868:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.610871:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.610873:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.610878:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.610883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.610885:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513068.610936:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513068.610943:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513068.610946:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513068.610949:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513068.610953:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513068.610958:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f760 00000800:00000200:12.0:1616513068.610962:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513068.612360:0:48232:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513068.612417:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.612427:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513068.612450:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.110@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.612459:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.110@tcp1 of length 224 into portal 12 MB=0x5ffdee3e1b140 00000400:00000200:22.0:1616513068.612465:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.110@tcp1 of length 224/224 into md 0xf963e789 [1] + 95744 00000400:00000200:22.0:1616513068.612471:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0F:1616513068.612471:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513068.612473:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513068.612481:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.612484:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.612486:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513068.612490:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707654594880 00010000:00000200:30.0:1616513068.612495:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad4800 x1688707654594880/t0(0) o400->d97c8819-88a5-0a91-4d90-681cb8912882@10.31.132.110@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513068.612500:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688707654594880, offset 192 00000400:00000200:30.0:1616513068.612502:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.110@tcp1 00000400:00000200:30.0:1616513068.612505:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.110@tcp1 routed destination 00000400:00000200:30.0:1616513068.612511:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.110@tcp1(10.31.132.110@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.612513:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513068.612515:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.612516:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513068.612517:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513068.612519:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.612523:0:48232:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.612533:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.612539:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513068.612541:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513068.612547:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513068.612551:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef8f8 00000800:00000200:10.0:1616513068.612554:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.612557:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:9.0F:1616513068.619115:0:6489:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923152, portal 4 00000100:00000200:9.0:1616513068.619122:0:6489:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923152, offset 0 00000400:00000200:9.0:1616513068.619130:0:6489:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:9.0:1616513068.619136:0:6489:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:9.0:1616513068.619150:0:6489:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:9.0:1616513068.619157:0:6489:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:9.0:1616513068.619165:0:6489:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:9.0:1616513068.619168:0:6489:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:9.0:1616513068.619171:0:6489:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:9.0:1616513068.619177:0:6489:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513068.619186:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513068.619203:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513068.619208:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513068.619211:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513068.619212:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513068.619218:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513068.619225:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843f7bd100 x1693067725923152/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513068.619237:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca34b28 00000800:00000200:1.0:1616513068.619242:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513068.619395:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513068.619404:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513068.619409:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513068.619410:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513068.619418:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513068.619427:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626350 00000400:00000200:1.0:1616513068.619475:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff6b5 [1] + 192 00000400:00000200:1.0:1616513068.619481:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513068.619487:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843f7bd100 x1693067725923152/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513068.619503:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513068.619506:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513068.619508:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:13.0F:1616513068.619556:0:6489:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843f7bd100 x1693067725923152/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:13.0:1616513068.619572:0:6489:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843f7bd100 x1693067725923152/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:13.0:1616513068.619583:0:6489:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca34a18 00000800:00000200:11.2:1616513068.626596:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513068.626600:0:48233:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513068.626666:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513068.626672:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513068.626676:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513068.626682:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513068.626685:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.231@tcp1 : PUT - for me 00000400:00000200:14.0:1616513068.626690:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.231@tcp1 : PUT - for me 00000400:00000200:20.0:1616513068.626695:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.231@tcp1 of length 224 into portal 12 MB=0x603e0ec15ebc0 00000400:00000200:14.0:1616513068.626700:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.231@tcp1 of length 224 into portal 26 MB=0x603e0ec15eb00 00000400:00000200:20.0:1616513068.626701:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.231@tcp1 of length 224/224 into md 0x865b9f31 [1] + 25808 00000400:00000200:14.0:1616513068.626706:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.231@tcp1 of length 224/224 into md 0xf97c3499 [1] + 0 00000400:00000200:20.0:1616513068.626707:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.626709:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513068.626711:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcd320 00000400:00000200:14.0:1616513068.626714:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513068.626717:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:14.0:1616513068.626717:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513068.626720:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513068.626720:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513068.626721:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd320 00000800:00000200:20.0:1616513068.626724:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513068.626724:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:8.0:1616513068.626725:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114428681152 00000800:00000200:14.0:1616513068.626727:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513068.626728:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:8.0:1616513068.626730:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ced100 x1693114428681152/t0(0) o400->7a76b8f6-86c4-3cd3-e875-7ad9ade8907e@10.31.136.231@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513068.626731:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513068.626735:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114428681152, offset 192 00000400:00000200:8.0:1616513068.626738:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.231@tcp1 00000400:00000200:8.0:1616513068.626741:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.231@tcp1 routed destination 00000400:00000200:8.0:1616513068.626747:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.231@tcp1(10.31.136.231@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.626749:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513068.626752:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.626752:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.626753:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.626755:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.626762:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.626786:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114428680960 00010000:00000200:24.0:1616513068.626797:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1693114428680960/t0(0) o400->eecd8dcb-a3f5-e505-1b2d-cd52385233f5@10.31.136.231@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.626811:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114428680960, offset 192 00000400:00000200:24.0:1616513068.626817:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.231@tcp1 00000800:00000200:4.0:1616513068.626818:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513068.626825:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.231@tcp1 routed destination 00000800:00000200:4.0:1616513068.626825:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513068.626828:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513068.626835:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513068.626839:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71e48 00000400:00000200:24.0:1616513068.626840:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.231@tcp1(10.31.136.231@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513068.626843:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513068.626846:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513068.626847:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.626852:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.626855:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.626857:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.626862:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.626866:0:48233:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.626869:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513068.626921:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.626927:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513068.626929:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513068.626935:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513068.626949:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:10.0:1616513068.626952:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.626955:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.631844:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513068.631901:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513068.631910:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513068.631919:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.103@tcp1 : PUT - for me 00000400:00000200:14.0:1616513068.631927:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.103@tcp1 of length 224 into portal 26 MB=0x6001d372d4c80 00000400:00000200:14.0:1616513068.631934:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.103@tcp1 of length 224/224 into md 0xf97c34b9 [1] + 0 00000400:00000200:14.0:1616513068.631938:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcd430 00000400:00000200:14.0:1616513068.631941:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513068.631944:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513068.631948:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd430 00000800:00000200:14.0:1616513068.631951:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513068.631953:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.631955:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513068.631959:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513068.632011:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975340031104 00010000:00000200:24.0:1616513068.632021:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688975340031104/t0(0) o400->219f6a46-fb9d-538b-fbb3-8217041f4224@10.31.142.103@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513068.632031:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513068.632036:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688975340031104, offset 192 00000400:00000200:24.0:1616513068.632044:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.103@tcp1 00000400:00000200:24.0:1616513068.632051:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.103@tcp1 routed destination 00000400:00000200:24.0:1616513068.632066:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.103@tcp1(10.31.142.103@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.632072:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.632078:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.632081:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.632083:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513068.632084:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.632088:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513068.632093:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:24.0:1616513068.632094:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513068.632094:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513068.632102:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.103@tcp1 : PUT - for me 00000400:00000200:10.0:1616513068.632110:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.103@tcp1 of length 224 into portal 12 MB=0x6001d372d4f00 00000400:00000200:10.0:1616513068.632116:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.103@tcp1 of length 224/224 into md 0xf963e789 [1] + 95968 00000400:00000200:10.0:1616513068.632121:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513068.632124:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513068.632130:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.632133:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513068.632135:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:30.0:1616513068.632138:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975340031744 00000800:00000200:18.0:1616513068.632138:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:30.0:1616513068.632142:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdde80 x1688975340031744/t0(0) o400->d3be5659-6829-150c-a91c-e57f3173c585@10.31.142.103@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513068.632148:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688975340031744, offset 192 00000400:00000200:30.0:1616513068.632150:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.103@tcp1 00000400:00000200:30.0:1616513068.632152:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.103@tcp1 routed destination 00000400:00000200:30.0:1616513068.632158:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.103@tcp1(10.31.142.103@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513068.632159:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513068.632160:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513068.632162:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.632163:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513068.632164:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513068.632166:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513068.632166:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513068.632168:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513068.632170:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513068.632175:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513068.632180:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513068.632181:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f3a8 00000800:00000200:28.0:1616513068.632184:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513068.632186:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.632188:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513068.632189:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513068.632190:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513068.632195:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.632199:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee440 00000800:00000200:22.0:1616513068.632203:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0F:1616513068.634636:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923168, portal 4 00000100:00000200:15.0:1616513068.634643:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923168, offset 0 00000400:00000200:15.0:1616513068.634649:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:15.0:1616513068.634655:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:15.0:1616513068.634667:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513068.634673:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:15.0:1616513068.634679:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513068.634681:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:15.0:1616513068.634683:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:15.0:1616513068.634688:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513068.634736:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513068.634754:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513068.634760:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:5.0:1616513068.634762:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513068.634762:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513068.634769:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513068.634776:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded3a80 x1693067725923168/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513068.634788:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774f68 00000800:00000200:1.0:1616513068.634793:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513068.634913:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513068.634921:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513068.634927:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513068.634928:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513068.634936:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513068.634944:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626360 00000400:00000200:1.0:1616513068.634960:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff6d5 [1] + 192 00000400:00000200:1.0:1616513068.634965:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513068.634969:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded3a80 x1693067725923168/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513068.634982:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513068.634984:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513068.634986:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:15.0:1616513068.634990:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded3a80 x1693067725923168/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513068.635003:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded3a80 x1693067725923168/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513068.635013:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774c38 00000800:00000200:11.2:1616513068.638293:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513068.638347:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513068.638356:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513068.638366:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.114@tcp1 : PUT - for me 00000400:00000200:4.0:1616513068.638387:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.114@tcp1 of length 224 into portal 26 MB=0x5ffdfb05a7600 00000400:00000200:4.0:1616513068.638394:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.114@tcp1 of length 224/224 into md 0xf97c34d9 [1] + 0 00000400:00000200:4.0:1616513068.638398:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcd540 00000400:00000200:4.0:1616513068.638401:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513068.638404:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:4.0:1616513068.638404:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513068.638408:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd540 00000800:00000200:4.0:1616513068.638411:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513068.638414:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513068.638415:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513068.638416:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513068.638419:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513068.638423:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.114@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.638432:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.114@tcp1 of length 224 into portal 12 MB=0x5ffdfb05a77c0 00000400:00000200:12.0:1616513068.638453:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.114@tcp1 of length 224/224 into md 0x865b9f31 [1] + 26032 00000400:00000200:12.0:1616513068.638458:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.638460:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513068.638470:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.638474:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.638476:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513068.638484:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711085061632 00000100:00000200:8.0:1616513068.638484:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711085062080 00010000:00000200:8.0:1616513068.638491:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa40480 x1688711085062080/t0(0) o400->d1cbcfad-3d8b-bd87-f5a1-22b2ce4a5f63@10.31.141.114@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513068.638495:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688711085061632/t0(0) o400->bc4c70fd-44fa-fb42-ee8a-926abb41eb30@10.31.141.114@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513068.638500:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688711085062080, offset 192 00000400:00000200:8.0:1616513068.638504:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.114@tcp1 00000100:00000200:24.0:1616513068.638509:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688711085061632, offset 192 00000400:00000200:8.0:1616513068.638509:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.114@tcp1 routed destination 00000400:00000200:24.0:1616513068.638516:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.114@tcp1 00000400:00000200:8.0:1616513068.638518:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.114@tcp1(10.31.141.114@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.638522:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513068.638526:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.114@tcp1 routed destination 00000800:00000200:8.0:1616513068.638532:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.638533:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.638535:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.638538:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.638541:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.114@tcp1(10.31.141.114@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513068.638543:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.638548:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.638553:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.638556:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.638558:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.638563:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.638569:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.638570:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513068.638596:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.638613:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513068.638617:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513068.638618:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513068.638625:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513068.638629:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70dd0 00000800:00000200:20.0:1616513068.638632:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513068.638632:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.638639:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513068.638642:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513068.638649:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513068.638654:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fed0 00000800:00000200:10.0:1616513068.638658:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.638660:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513068.666644:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923184, portal 4 00000100:00000200:4.0:1616513068.666649:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923184, offset 0 00000400:00000200:4.0:1616513068.666657:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:4.0:1616513068.666662:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000100:00000200:20.0:1616513068.666669:0:6487:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923200, portal 4 00000100:00000200:20.0:1616513068.666675:0:6487:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923200, offset 0 00000400:00000200:4.0:1616513068.666675:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513068.666681:0:6487:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:4.0:1616513068.666681:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:20.0:1616513068.666687:0:6487:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:4.0:1616513068.666687:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513068.666690:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:4.0:1616513068.666692:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:4.0:1616513068.666697:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:20.0:1616513068.666700:0:6487:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513068.666706:0:6487:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:20.0:1616513068.666712:0:6487:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513068.666715:0:6487:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:20.0:1616513068.666718:0:6487:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:20.0:1616513068.666723:0:6487:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.2:1616513068.666746:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513068.666766:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:27.0:1616513068.666773:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:27.0:1616513068.666776:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513068.666783:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513068.666790:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d846aa42400 x1693067725923184/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513068.666803:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb540 00000800:00000200:27.0:1616513068.666807:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513068.666810:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513068.666814:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513068.666815:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513068.666820:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:3.0F:1616513068.666821:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:27.0:1616513068.666826:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdeb8f080 x1693067725923200/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513068.666835:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee222a8 00000800:00000200:27.0:1616513068.666838:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513068.666877:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513068.666886:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513068.666892:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000800:00000200:3.0:1616513068.666895:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513068.666900:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:3.0:1616513068.666906:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513068.666908:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513068.666908:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626370 00000400:00000200:27.0:1616513068.666915:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164bad [1] + 192 00000400:00000200:3.0:1616513068.666915:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513068.666920:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:3.0:1616513068.666923:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626380 00000100:00000200:27.0:1616513068.666926:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d846aa42400 x1693067725923184/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513068.666940:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:3.0:1616513068.666942:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164bbd [1] + 192 00000800:00000200:27.0:1616513068.666943:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513068.666945:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:3.0:1616513068.666948:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513068.666951:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d846aa42400 x1693067725923184/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513068.666963:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d846aa42400 x1693067725923184/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513068.666973:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbcb0 00000100:00000200:3.0:1616513068.666978:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdeb8f080 x1693067725923200/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:3.0:1616513068.666994:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:3.0:1616513068.666997:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:3.0:1616513068.666999:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:20.0:1616513068.667003:0:6487:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdeb8f080 x1693067725923200/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513068.667017:0:6487:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdeb8f080 x1693067725923200/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513068.667027:0:6487:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23188 00000100:00000200:18.0:1616513068.682949:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923216, portal 4 00000100:00000200:18.0:1616513068.682954:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923216, offset 0 00000400:00000200:18.0:1616513068.682960:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:18.0:1616513068.682966:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:18.0:1616513068.682979:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513068.682985:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513068.682991:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513068.682994:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513068.682996:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513068.683001:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513068.683049:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513068.683106:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513068.683113:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513068.683116:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513068.683119:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513068.683123:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513068.683129:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d747c690d80 x1693067725923216/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513068.683141:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdb18 00000800:00000200:14.0:1616513068.683147:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513068.683181:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513068.683236:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513068.683246:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513068.683248:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:10.0:1616513068.683255:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:10.0:1616513068.683262:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626390 00000400:00000200:10.0:1616513068.683269:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c3b09 [1] + 192 00000400:00000200:10.0:1616513068.683274:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513068.683279:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d747c690d80 x1693067725923216/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:10.0:1616513068.683294:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513068.683297:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513068.683298:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:18.0:1616513068.683300:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d747c690d80 x1693067725923216/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:18.0:1616513068.683313:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d747c690d80 x1693067725923216/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513068.683323:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc000 00000800:00000200:7.2:1616513068.684391:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513068.684450:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.684460:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513068.684469:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.113@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.684478:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.113@tcp1 of length 224 into portal 26 MB=0x5ffdf6b237200 00000400:00000200:22.0:1616513068.684485:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.113@tcp1 of length 224/224 into md 0xf97c34e9 [1] + 0 00000400:00000200:22.0:1616513068.684490:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcd650 00000400:00000200:22.0:1616513068.684494:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513068.684497:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513068.684501:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd650 00000800:00000200:14.0:1616513068.684502:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.684505:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.684507:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513068.684509:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.684512:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513068.684513:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513068.684521:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.113@tcp1 : PUT - for me 00000400:00000200:14.0:1616513068.684530:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.113@tcp1 of length 224 into portal 12 MB=0x5ffdf6b237280 00000400:00000200:14.0:1616513068.684536:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.113@tcp1 of length 224/224 into md 0xf963e789 [1] + 96192 00000400:00000200:14.0:1616513068.684541:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8283b76220 00000400:00000200:14.0:1616513068.684544:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513068.684546:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513068.684551:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8283b76220 00000800:00000200:14.0:1616513068.684554:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513068.684557:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513068.684559:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513068.684560:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709923828352 00000100:00000200:24.0:1616513068.684562:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709923828224 00010000:00000200:30.0:1616513068.684565:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74d9abf050 x1688709923828352/t0(0) o400->22d1d69e-2c51-f49a-fd7b-b3e1736aacd0@10.31.142.113@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513068.684570:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688709923828352, offset 192 00000400:00000200:30.0:1616513068.684572:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.113@tcp1 00010000:00000200:24.0:1616513068.684572:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688709923828224/t0(0) o400->8a2e2732-c516-88cd-d37b-d7e5617cf424@10.31.142.113@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:30.0:1616513068.684575:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.113@tcp1 routed destination 00000400:00000200:30.0:1616513068.684580:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.113@tcp1(10.31.142.113@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.684582:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.684584:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.684585:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513068.684585:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688709923828224, offset 192 00000800:00000200:30.0:1616513068.684586:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513068.684588:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.684591:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.113@tcp1 00000800:00000200:11.2:1616513068.684596:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.684598:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.113@tcp1 routed destination 00000400:00000200:24.0:1616513068.684612:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.113@tcp1(10.31.142.113@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.684621:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.684627:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.684629:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.684631:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.684635:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.684640:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.684642:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513068.684652:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513068.684659:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513068.684661:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513068.684668:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513068.684673:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee000 00000800:00000200:28.0:1616513068.684677:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.684680:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513068.684692:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.684701:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513068.684704:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513068.684705:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513068.684711:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513068.684716:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ed48 00000800:00000200:10.0:1616513068.684720:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513068.684845:0:188463:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 12 00000800:00000200:7.2:1616513068.701914:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513068.701956:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513068.701995:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513068.702004:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513068.702013:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.135@tcp1 : PUT - for me 00000800:00000200:4.0:1616513068.702015:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513068.702022:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.135@tcp1 of length 224 into portal 12 MB=0x5ffdebdf30800 00000800:00000200:4.0:1616513068.702024:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:14.0:1616513068.702029:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.135@tcp1 of length 224/224 into md 0xf96450b9 [1] + 0 00000400:00000200:4.0:1616513068.702033:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.135@tcp1 : PUT - for me 00000400:00000200:14.0:1616513068.702034:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513068.702037:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:4.0:1616513068.702042:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.135@tcp1 of length 224 into portal 26 MB=0x5ffdebdf30600 00000800:00000200:14.0:1616513068.702043:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513068.702046:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513068.702048:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513068.702049:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.135@tcp1 of length 224/224 into md 0xf97c3509 [1] + 0 00000800:00000200:18.0:1616513068.702050:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513068.702051:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707018196992 00000400:00000200:4.0:1616513068.702054:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcd760 00010000:00000200:30.0:1616513068.702055:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa4800 x1688707018196992/t0(0) o400->e00f2fab-a587-fcfb-464a-0453a6ce3294@10.31.142.135@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:4.0:1616513068.702057:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513068.702059:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513068.702064:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd760 00000100:00000200:30.0:1616513068.702066:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688707018196992, offset 192 00000800:00000200:4.0:1616513068.702067:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:30.0:1616513068.702069:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.135@tcp1 00000800:00000200:4.0:1616513068.702070:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:30.0:1616513068.702072:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.135@tcp1 routed destination 00000800:00000200:4.0:1616513068.702072:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.702074:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513068.702077:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.135@tcp1(10.31.142.135@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.702079:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.702081:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.702082:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513068.702083:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513068.702085:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.702090:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.702133:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707018196480 00010000:00000200:24.0:1616513068.702144:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688707018196480/t0(0) o400->c14e7d60-6210-dc27-ba18-651bc4ca80fd@10.31.142.135@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513068.702160:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513068.702162:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688707018196480, offset 192 00000400:00000200:24.0:1616513068.702168:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.135@tcp1 00000800:00000200:28.0:1616513068.702172:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513068.702175:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513068.702176:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.135@tcp1 routed destination 00000400:00000200:28.0:1616513068.702181:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513068.702185:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef430 00000800:00000200:28.0:1616513068.702189:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.702191:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.135@tcp1(10.31.142.135@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513068.702192:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513068.702197:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.702203:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.702206:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.702208:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.702213:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.702218:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.702220:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513068.702273:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.702280:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.702283:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513068.702289:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:10.0:1616513068.702289:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513068.702295:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f320 00000800:00000200:22.0:1616513068.702299:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.742323:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513068.742378:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513068.742387:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513068.742396:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.160@tcp1 : PUT - for me 00000400:00000200:14.0:1616513068.742405:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.160@tcp1 of length 224 into portal 26 MB=0x6048d8b97df00 00000400:00000200:14.0:1616513068.742417:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.160@tcp1 of length 224/224 into md 0xf97c3519 [1] + 0 00000400:00000200:14.0:1616513068.742422:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcd870 00000400:00000200:14.0:1616513068.742424:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513068.742427:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513068.742431:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd870 00000800:00000200:18.0:1616513068.742450:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513068.742453:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513068.742456:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513068.742458:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513068.742460:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513068.742461:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513068.742470:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.160@tcp1 : PUT - for me 00000400:00000200:18.0:1616513068.742479:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.160@tcp1 of length 224 into portal 12 MB=0x6048d8b97df80 00000400:00000200:18.0:1616513068.742485:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.160@tcp1 of length 224/224 into md 0xf96450b9 [1] + 224 00000400:00000200:18.0:1616513068.742490:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.742492:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:24.0:1616513068.742495:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855839149824 00000800:00000200:18.0:1616513068.742500:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513068.742503:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513068.742505:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513068.742506:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1693855839149824/t0(0) o400->87122ca9-64c8-12cd-cea7-8f307566a9b7@10.31.140.160@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513068.742508:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855839149952 00010000:00000200:30.0:1616513068.742512:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d4800 x1693855839149952/t0(0) o400->7ed0cd10-4212-095d-b31d-50e3393b80a3@10.31.140.160@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513068.742517:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855839149952, offset 192 00000400:00000200:30.0:1616513068.742519:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.160@tcp1 00000100:00000200:24.0:1616513068.742521:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855839149824, offset 192 00000400:00000200:30.0:1616513068.742522:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.160@tcp1 routed destination 00000400:00000200:30.0:1616513068.742527:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.160@tcp1(10.31.140.160@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513068.742527:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.160@tcp1 00000800:00000200:30.0:1616513068.742530:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.742532:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.742533:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513068.742533:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513068.742535:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.742535:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.160@tcp1 routed destination 00000800:00000200:11.2:1616513068.742543:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.742549:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.160@tcp1(10.31.140.160@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.742556:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.742562:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.742565:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.742569:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.742574:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.742578:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.742581:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513068.742596:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.742604:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513068.742606:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513068.742613:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513068.742617:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeff0 00000800:00000200:4.0:1616513068.742621:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513068.742624:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513068.742631:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.742639:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.742641:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513068.742643:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513068.742648:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.742654:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f7e8 00000800:00000200:22.0:1616513068.742657:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.761268:0:48250:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513068.761322:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513068.761331:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513068.761339:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.25@tcp1 : PUT - for me 00000400:00000200:18.0:1616513068.761348:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.25@tcp1 of length 224 into portal 26 MB=0x60183559fd580 00000400:00000200:18.0:1616513068.761356:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.25@tcp1 of length 224/224 into md 0xf97c3549 [1] + 0 00000400:00000200:18.0:1616513068.761360:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcd980 00000400:00000200:18.0:1616513068.761363:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.761365:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513068.761369:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd980 00000800:00000200:18.0:1616513068.761372:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.761375:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513068.761376:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.761379:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513068.761449:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513449145728 00010000:00000200:24.0:1616513068.761460:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1690513449145728/t0(0) o400->7185a3d6-6344-9522-72fd-9eedbd21699b@10.31.136.25@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.761475:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513449145728, offset 192 00000400:00000200:24.0:1616513068.761481:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.25@tcp1 00000400:00000200:24.0:1616513068.761489:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.25@tcp1 routed destination 00000400:00000200:24.0:1616513068.761504:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.25@tcp1(10.31.136.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.761510:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.761516:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.761518:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.761521:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.761525:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513068.761532:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513068.761533:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.761588:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513068.761595:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513068.761597:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513068.761604:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513068.761609:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fb18 00000800:00000200:28.0:1616513068.761613:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.761616:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513068.761747:0:48250:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513068.761807:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513068.761816:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513068.761825:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.25@tcp1 : PUT - for me 00000400:00000200:14.0:1616513068.761834:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.25@tcp1 of length 224 into portal 12 MB=0x60183559fd600 00000400:00000200:14.0:1616513068.761840:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.25@tcp1 of length 224/224 into md 0xf96450b9 [1] + 448 00000400:00000200:14.0:1616513068.761845:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513068.761848:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513068.761855:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513068.761858:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513068.761860:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:30.0:1616513068.761863:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513449145856 00000800:00000200:22.0:1616513068.761863:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:30.0:1616513068.761868:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5100 x1690513449145856/t0(0) o400->2d3c97bf-475a-6cb6-cda3-5073b3cb25e3@10.31.136.25@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513068.761873:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513449145856, offset 192 00000400:00000200:30.0:1616513068.761875:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.25@tcp1 00000400:00000200:30.0:1616513068.761878:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.25@tcp1 routed destination 00000400:00000200:30.0:1616513068.761884:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.25@tcp1(10.31.136.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.761886:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513068.761888:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.761889:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513068.761890:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513068.761892:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.761896:0:48250:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513068.761905:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.761909:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513068.761911:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513068.761914:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513068.761916:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.761919:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eecc0 00000800:00000200:18.0:1616513068.761922:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513068.762466:0:6480:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923232, portal 4 00000100:00000200:16.0:1616513068.762471:0:6480:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923232, offset 0 00000400:00000200:16.0:1616513068.762478:0:6480:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:16.0:1616513068.762484:0:6480:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:16.0:1616513068.762497:0:6480:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513068.762504:0:6480:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:16.0:1616513068.762512:0:6480:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513068.762515:0:6480:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:16.0:1616513068.762517:0:6480:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:16.0:1616513068.762521:0:6480:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513068.762568:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513068.762581:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513068.762585:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513068.762588:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513068.762590:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513068.762593:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513068.762599:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cbeedc800 x1693067725923232/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513068.762611:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42088 00000800:00000200:14.0:1616513068.762614:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513068.762754:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513068.762766:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513068.762773:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513068.762774:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:10.0:1616513068.762782:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:10.0:1616513068.762790:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6263a0 00000400:00000200:10.0:1616513068.762796:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c3bb9 [1] + 192 00000400:00000200:10.0:1616513068.762802:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513068.762807:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cbeedc800 x1693067725923232/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:10.0:1616513068.762821:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513068.762824:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513068.762826:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:16.0:1616513068.762829:0:6480:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cbeedc800 x1693067725923232/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513068.762842:0:6480:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cbeedc800 x1693067725923232/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513068.762852:0:6480:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42220 00000100:00000200:3.0:1616513068.778609:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923248, portal 4 00000100:00000200:3.0:1616513068.778615:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923248, offset 0 00000400:00000200:3.0:1616513068.778622:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:3.0:1616513068.778629:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:3.0:1616513068.778642:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513068.778649:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:3.0:1616513068.778655:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513068.778658:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:3.0:1616513068.778661:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:3.0:1616513068.778667:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513068.778712:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513068.778732:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513068.778737:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513068.778740:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513068.778742:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513068.778747:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513068.778754:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456e9b00 x1693067725923248/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513068.778766:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2330 00000800:00000200:27.0:1616513068.778770:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513068.778885:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513068.778894:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513068.778900:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:3.0:1616513068.778903:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513068.778908:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513068.778915:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6263b0 00000400:00000200:27.0:1616513068.778922:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164bcd [1] + 192 00000400:00000200:27.0:1616513068.778927:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513068.778931:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456e9b00 x1693067725923248/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513068.778944:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513068.778947:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513068.778949:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:3.0:1616513068.778951:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456e9b00 x1693067725923248/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513068.778964:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456e9b00 x1693067725923248/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513068.778974:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2000 00000800:00000200:11.2:1616513068.804924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513068.804979:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513068.804988:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513068.804998:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.37@tcp1 : PUT - for me 00000400:00000200:4.0:1616513068.805019:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.37@tcp1 of length 224 into portal 26 MB=0x5ffdc39937fc0 00000400:00000200:4.0:1616513068.805025:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.37@tcp1 of length 224/224 into md 0xf97c3569 [1] + 0 00000400:00000200:4.0:1616513068.805030:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcda90 00000400:00000200:4.0:1616513068.805033:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513068.805035:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513068.805036:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:4.0:1616513068.805040:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcda90 00000800:00000200:4.0:1616513068.805043:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513068.805046:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513068.805047:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513068.805048:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513068.805051:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513068.805056:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.37@tcp1 : PUT - for me 00000400:00000200:20.0:1616513068.805065:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.37@tcp1 of length 224 into portal 12 MB=0x5ffdc39938240 00000400:00000200:20.0:1616513068.805071:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.37@tcp1 of length 224/224 into md 0x865b9f31 [1] + 26256 00000400:00000200:20.0:1616513068.805077:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.805080:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513068.805087:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513068.805090:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513068.805092:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513068.805098:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696207409728 00000100:00000200:24.0:1616513068.805104:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696207409088 00010000:00000200:8.0:1616513068.805104:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cea400 x1688696207409728/t0(0) o400->61fb08f2-1f71-62cd-ef08-329d80fc67dc@10.31.142.37@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513068.805111:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696207409728, offset 192 00000400:00000200:8.0:1616513068.805114:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.37@tcp1 00010000:00000200:24.0:1616513068.805115:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688696207409088/t0(0) o400->343ddfa4-589e-8a98-433d-ff5c4fade341@10.31.142.37@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513068.805117:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.37@tcp1 routed destination 00000400:00000200:8.0:1616513068.805124:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.37@tcp1(10.31.142.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.805127:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513068.805129:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696207409088, offset 192 00000800:00000200:8.0:1616513068.805130:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.805131:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.805132:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.805135:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.37@tcp1 00000800:00000200:8.0:1616513068.805135:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.805140:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.805142:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.37@tcp1 routed destination 00000400:00000200:24.0:1616513068.805157:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.37@tcp1(10.31.142.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.805164:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.805169:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.805172:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.805174:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.805179:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513068.805186:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513068.805188:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513068.805195:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513068.805202:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513068.805204:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513068.805205:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513068.805211:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513068.805215:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f706e8 00000800:00000200:12.0:1616513068.805219:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513068.805245:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513068.805252:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513068.805254:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513068.805261:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513068.805267:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e660 00000800:00000200:22.0:1616513068.805270:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.805273:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513068.810582:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923264, portal 4 00000100:00000200:16.0:1616513068.810587:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923264, offset 0 00000400:00000200:16.0:1616513068.810594:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:16.0:1616513068.810599:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:16.0:1616513068.810612:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513068.810619:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:16.0:1616513068.810625:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513068.810628:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:16.0:1616513068.810630:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:16.0:1616513068.810635:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:31.0:1616513068.810661:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923280, portal 4 00000100:00000200:31.0:1616513068.810667:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923280, offset 0 00000400:00000200:31.0:1616513068.810674:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:31.0:1616513068.810681:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:25.2:1616513068.810684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000400:00000200:31.0:1616513068.810696:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513068.810703:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:31.0:1616513068.810712:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513068.810715:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:31.0:1616513068.810718:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:31.0:1616513068.810724:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513068.810733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:18.0:1616513068.810740:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513068.810747:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000800:00000200:20.0:1616513068.810748:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:18.0:1616513068.810749:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513068.810755:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:18.0:1616513068.810755:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:20.0:1616513068.810760:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.810761:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d846aa47980 x1693067725923264/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513068.810763:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:20.0:1616513068.810767:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.810773:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42550 00000100:00000200:20.0:1616513068.810774:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bff500 x1693067725923280/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513068.810776:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513068.810779:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513068.810785:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:20.0:1616513068.810786:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2aa0 00000800:00000200:20.0:1616513068.810790:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000400:00000200:18.0:1616513068.810792:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513068.810798:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6263c0 00000400:00000200:18.0:1616513068.810804:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c3bd9 [1] + 192 00000800:00000200:10.0:1616513068.810805:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513068.810808:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.810812:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d846aa47980 x1693067725923264/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513068.810825:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513068.810828:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513068.810830:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:16.0:1616513068.810833:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d846aa47980 x1693067725923264/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513068.810846:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d846aa47980 x1693067725923264/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513068.810857:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42220 00000800:00000200:5.2:1616513068.810964:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:4.0:1616513068.811018:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513068.811028:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:12.0:1616513068.811031:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:4.0:1616513068.811036:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:4.0:1616513068.811044:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6263d0 00000400:00000200:4.0:1616513068.811052:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d2241 [1] + 192 00000400:00000200:4.0:1616513068.811057:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513068.811062:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bff500 x1693067725923280/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:4.0:1616513068.811077:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513068.811080:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513068.811082:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:31.0:1616513068.811087:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bff500 x1693067725923280/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513068.811100:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bff500 x1693067725923280/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513068.811111:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2000 00000800:00000200:7.2:1616513068.828346:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513068.828402:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513068.828412:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513068.828421:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.122@tcp1 : PUT - for me 00000400:00000200:14.0:1616513068.828430:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.122@tcp1 of length 224 into portal 26 MB=0x603ff793ff080 00000400:00000200:14.0:1616513068.828449:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.122@tcp1 of length 224/224 into md 0xf97c3579 [1] + 0 00000400:00000200:14.0:1616513068.828454:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcdba0 00000400:00000200:14.0:1616513068.828457:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513068.828459:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513068.828463:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdba0 00000800:00000200:14.0:1616513068.828466:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.828466:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513068.828471:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513068.828473:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513068.828476:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.828477:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513068.828485:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.122@tcp1 : PUT - for me 00000400:00000200:10.0:1616513068.828494:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.122@tcp1 of length 224 into portal 12 MB=0x603ff793ff180 00000400:00000200:10.0:1616513068.828500:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.122@tcp1 of length 224/224 into md 0xf96450b9 [1] + 672 00000400:00000200:10.0:1616513068.828505:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513068.828508:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513068.828515:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.828518:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.828520:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513068.828523:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245646041472 00010000:00000200:30.0:1616513068.828528:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdcc80 x1693245646041472/t0(0) o400->c401ffd1-8b20-2693-5b5c-ca6d2459c9d1@10.31.140.122@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513068.828533:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245646041472, offset 192 00000400:00000200:30.0:1616513068.828535:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.122@tcp1 00000400:00000200:30.0:1616513068.828538:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.122@tcp1 routed destination 00000400:00000200:30.0:1616513068.828543:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.122@tcp1(10.31.140.122@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513068.828544:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245646041216 00000800:00000200:30.0:1616513068.828546:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.828548:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.828549:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513068.828550:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513068.828552:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:24.0:1616513068.828555:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1693245646041216/t0(0) o400->39ce0356-d294-990d-a06e-0a930160975d@10.31.140.122@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513068.828560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.828571:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245646041216, offset 192 00000800:00000200:28.0:1616513068.828577:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513068.828577:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.122@tcp1 00000800:00000200:28.0:1616513068.828583:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513068.828584:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.122@tcp1 routed destination 00000400:00000200:28.0:1616513068.828586:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513068.828593:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513068.828597:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee220 00000400:00000200:24.0:1616513068.828599:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.122@tcp1(10.31.140.122@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513068.828600:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.828613:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513068.828616:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.828622:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.828624:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.828627:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.828631:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.828636:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.828638:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513068.828651:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.828657:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513068.828660:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513068.828662:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513068.828667:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.828672:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f650 00000800:00000200:18.0:1616513068.828675:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:3.0:1616513068.842591:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923296, portal 4 00000100:00000200:3.0:1616513068.842596:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923296, offset 0 00000400:00000200:3.0:1616513068.842603:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:3.0:1616513068.842610:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:3.0:1616513068.842626:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513068.842633:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:3.0:1616513068.842640:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513068.842643:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:3.0:1616513068.842646:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:3.0:1616513068.842651:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513068.842660:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513068.842676:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513068.842682:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:5.0:1616513068.842684:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513068.842685:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513068.842691:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513068.842698:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8006cdb180 x1693067725923296/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513068.842710:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2e58 00000800:00000200:1.0:1616513068.842714:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513068.842846:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513068.842855:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513068.842860:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513068.842861:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513068.842869:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513068.842876:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6263e0 00000400:00000200:1.0:1616513068.842882:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff6e5 [1] + 192 00000400:00000200:1.0:1616513068.842887:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513068.842891:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8006cdb180 x1693067725923296/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513068.842903:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513068.842906:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513068.842908:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:3.0:1616513068.842912:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8006cdb180 x1693067725923296/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513068.842925:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8006cdb180 x1693067725923296/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513068.842936:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2000 00000800:00000200:11.2:1616513068.843199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513068.843255:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.843264:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513068.843273:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.206@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.843281:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.206@tcp1 of length 224 into portal 26 MB=0x5ffdbdc55e0c0 00000800:00000200:7.2:1616513068.843284:0:48260:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513068.843287:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.206@tcp1 of length 224/224 into md 0xf97c3589 [1] + 0 00000400:00000200:12.0:1616513068.843291:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcdcb0 00000400:00000200:12.0:1616513068.843294:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.843297:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513068.843301:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdcb0 00000800:00000200:12.0:1616513068.843304:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513068.843306:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.843308:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.843311:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513068.843358:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.843368:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:24.0:1616513068.843372:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694643089600 00000400:00000200:10.0:1616513068.843377:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.206@tcp1 : PUT - for me 00010000:00000200:24.0:1616513068.843383:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688694643089600/t0(0) o400->e08f60cf-07e5-8e3e-e706-922a195e8d66@10.31.141.206@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:10.0:1616513068.843387:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.206@tcp1 of length 224 into portal 12 MB=0x5ffdbdc55e180 00000400:00000200:10.0:1616513068.843393:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.206@tcp1 of length 224/224 into md 0xf96450b9 [1] + 896 00000100:00000200:24.0:1616513068.843398:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694643089600, offset 192 00000400:00000200:10.0:1616513068.843399:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513068.843401:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513068.843404:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.206@tcp1 00000800:00000200:10.0:1616513068.843408:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513068.843411:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.206@tcp1 routed destination 00000800:00000200:10.0:1616513068.843411:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.843412:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:30.0:1616513068.843415:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694643089792 00000800:00000200:10.0:1616513068.843416:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:30.0:1616513068.843419:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdad00 x1688694643089792/t0(0) o400->3da51561-26d9-4502-ddb2-ca9e2724b409@10.31.141.206@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513068.843425:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694643089792, offset 192 00000400:00000200:30.0:1616513068.843427:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.206@tcp1 00000400:00000200:24.0:1616513068.843427:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.206@tcp1(10.31.141.206@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513068.843430:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.206@tcp1 routed destination 00000400:00000200:30.0:1616513068.843435:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.206@tcp1(10.31.141.206@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.843442:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513068.843445:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.843446:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.843446:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.843447:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513068.843449:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513068.843452:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:7.2:1616513068.843454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.843455:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.843457:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.843462:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.843466:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.843469:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513068.843510:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513068.843517:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513068.843519:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513068.843525:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513068.843526:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513068.843528:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513068.843530:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee110 00000800:00000200:14.0:1616513068.843533:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513068.843535:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513068.843538:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513068.843544:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513068.843550:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f650 00000800:00000200:28.0:1616513068.843555:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.843558:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513068.866267:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513068.866268:0:48263:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513068.866330:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.866339:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:12.0:1616513068.866345:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513068.866348:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.154@tcp1 : PUT - for me 00000800:00000200:12.0:1616513068.866354:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513068.866357:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.154@tcp1 of length 224 into portal 12 MB=0x5ffd628cd1c00 00000400:00000200:12.0:1616513068.866363:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.154@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.866364:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.154@tcp1 of length 224/224 into md 0xf96450b9 [1] + 1120 00000400:00000200:22.0:1616513068.866371:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:12.0:1616513068.866372:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.154@tcp1 of length 224 into portal 26 MB=0x5ffd628cd1b40 00000100:00000200:22.0:1616513068.866373:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513068.866380:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513068.866382:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.154@tcp1 of length 224/224 into md 0xf97c3599 [1] + 0 00000800:00000200:10.0:1616513068.866382:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.866383:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.866386:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513068.866387:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcddc0 00000100:00000200:30.0:1616513068.866388:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688670156168192 00000400:00000200:12.0:1616513068.866390:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.866392:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00010000:00000200:30.0:1616513068.866393:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad1f80 x1688670156168192/t0(0) o400->1d9ffdaf-3e56-7e03-145d-2826dfc13af7@10.31.138.154@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513068.866397:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcddc0 00000100:00000200:30.0:1616513068.866399:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688670156168192, offset 192 00000800:00000200:12.0:1616513068.866400:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.866400:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513068.866401:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.154@tcp1 00000400:00000200:30.0:1616513068.866404:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.154@tcp1 routed destination 00000800:00000200:12.0:1616513068.866404:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.866406:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513068.866410:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.154@tcp1(10.31.138.154@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.866412:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.866414:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.866415:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513068.866416:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513068.866418:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.866423:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.866468:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688670156168000 00000800:00000200:28.0:1616513068.866477:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:24.0:1616513068.866479:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688670156168000/t0(0) o400->172edc3c-caa6-54d0-5f0e-26a2022f3a00@10.31.138.154@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513068.866484:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513068.866487:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:24.0:1616513068.866493:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688670156168000, offset 192 00000400:00000200:28.0:1616513068.866494:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513068.866498:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee908 00000400:00000200:24.0:1616513068.866500:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.154@tcp1 00000800:00000200:28.0:1616513068.866502:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.866505:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513068.866520:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.154@tcp1 routed destination 00000400:00000200:24.0:1616513068.866537:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.154@tcp1(10.31.138.154@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.866543:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.866549:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.866552:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.866554:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.866559:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.866564:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.866566:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513068.866617:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.866623:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513068.866626:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513068.866632:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513068.866633:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.866638:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fb18 00000800:00000200:18.0:1616513068.866642:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:19.0:1616513068.870617:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:19.0:1616513068.870625:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513068.874405:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513068.874460:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513068.874470:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513068.874479:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.207@tcp1 : PUT - for me 00000400:00000200:4.0:1616513068.874487:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.207@tcp1 of length 224 into portal 26 MB=0x5ffe346ce6200 00000400:00000200:4.0:1616513068.874494:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.207@tcp1 of length 224/224 into md 0xf97c35a9 [1] + 0 00000400:00000200:4.0:1616513068.874499:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcded0 00000400:00000200:4.0:1616513068.874503:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513068.874506:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513068.874510:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcded0 00000800:00000200:12.0:1616513068.874513:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.874522:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513068.874524:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513068.874525:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513068.874527:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513068.874530:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513068.874533:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.207@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.874541:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.207@tcp1 of length 224 into portal 12 MB=0x5ffe346ce6280 00000400:00000200:12.0:1616513068.874548:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.207@tcp1 of length 224/224 into md 0x865b9f31 [1] + 26480 00000400:00000200:12.0:1616513068.874553:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.874556:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513068.874562:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.874566:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513068.874567:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513068.874572:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726494143104 00000100:00000200:24.0:1616513068.874573:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726494142976 00010000:00000200:8.0:1616513068.874577:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa46780 x1688726494143104/t0(0) o400->90823477-8f9a-b0e9-d50a-5ae90bf3b262@10.31.141.207@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513068.874583:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726494143104, offset 192 00010000:00000200:24.0:1616513068.874584:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688726494142976/t0(0) o400->446b1079-1bda-e86a-8dd5-1544ad673ef6@10.31.141.207@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513068.874586:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.207@tcp1 00000400:00000200:8.0:1616513068.874589:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.207@tcp1 routed destination 00000400:00000200:8.0:1616513068.874595:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.207@tcp1(10.31.141.207@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.874597:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513068.874598:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726494142976, offset 192 00000800:00000200:8.0:1616513068.874600:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.874601:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.874601:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.874604:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.207@tcp1 00000800:00000200:8.0:1616513068.874604:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.874610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.874611:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.207@tcp1 routed destination 00000400:00000200:24.0:1616513068.874626:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.207@tcp1(10.31.141.207@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.874638:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.874644:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.874647:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.874649:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.874654:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513068.874660:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513068.874660:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.874676:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513068.874682:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513068.874682:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513068.874685:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513068.874689:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513068.874691:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513068.874692:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:4.0:1616513068.874692:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513068.874697:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f320 00000400:00000200:20.0:1616513068.874698:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:10.0:1616513068.874700:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513068.874702:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71b18 00000800:00000200:10.0:1616513068.874703:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513068.874706:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:27.0:1616513068.906520:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923312, portal 4 00000100:00000200:27.0:1616513068.906526:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923312, offset 0 00000400:00000200:27.0:1616513068.906533:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000100:00000200:11.0F:1616513068.906537:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923328, portal 4 00000400:00000200:27.0:1616513068.906540:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000100:00000200:11.0:1616513068.906543:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923328, offset 0 00000400:00000200:11.0:1616513068.906548:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:27.0:1616513068.906554:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000400:00000200:11.0:1616513068.906554:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:27.0:1616513068.906561:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:11.0:1616513068.906566:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513068.906568:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513068.906572:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:27.0:1616513068.906573:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:27.0:1616513068.906576:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:11.0:1616513068.906579:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513068.906581:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:11.0:1616513068.906582:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:11.0:1616513068.906584:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:11.0:1616513068.906589:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:1.2:1616513068.906628:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:25.2:1616513068.906645:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:1.0:1616513068.906648:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513068.906654:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:5.0:1616513068.906657:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513068.906657:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513068.906664:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513068.906671:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd639b00 x1693067725923312/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513068.906683:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d2486e8 00000800:00000200:1.0:1616513068.906687:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:14.0:1616513068.906701:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513068.906709:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513068.906712:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513068.906719:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513068.906725:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bff500 x1693067725923328/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513068.906737:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a440 00000800:00000200:14.0:1616513068.906741:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513068.906744:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:1.2:1616513068.906800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:25.2:1616513068.906805:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:1.0:1616513068.906809:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513068.906815:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513068.906815:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513068.906824:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513068.906832:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6263f0 00000400:00000200:1.0:1616513068.906839:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff6f5 [1] + 192 00000400:00000200:1.0:1616513068.906844:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513068.906849:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd639b00 x1693067725923312/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:10.0:1616513068.906860:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:1.0:1616513068.906862:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513068.906865:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513068.906867:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:10.0:1616513068.906870:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000100:00000200:27.0:1616513068.906871:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd639b00 x1693067725923312/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513068.906879:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000100:00000200:27.0:1616513068.906884:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd639b00 x1693067725923312/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513068.906887:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626400 00000400:00000200:10.0:1616513068.906894:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c3c59 [1] + 192 00000400:00000200:27.0:1616513068.906895:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249e48 00000400:00000200:10.0:1616513068.906899:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513068.906904:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bff500 x1693067725923328/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513068.906915:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513068.906919:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513068.906922:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513068.906924:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:11.0:1616513068.906927:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bff500 x1693067725923328/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513068.906937:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bff500 x1693067725923328/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513068.906946:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3aaa0 00000800:00000200:11.2:1616513068.912769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513068.912826:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.912836:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513068.912845:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.216@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.912854:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.216@tcp1 of length 224 into portal 26 MB=0x602167588b800 00000400:00000200:12.0:1616513068.912860:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.216@tcp1 of length 224/224 into md 0xf97c35c9 [1] + 0 00000400:00000200:12.0:1616513068.912865:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcde48 00000400:00000200:12.0:1616513068.912868:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.912870:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513068.912875:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcde48 00000800:00000200:12.0:1616513068.912878:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513068.912881:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513068.912881:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.912888:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.912891:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.912892:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513068.912900:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.216@tcp1 : PUT - for me 00000400:00000200:28.0:1616513068.912909:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.216@tcp1 of length 224 into portal 12 MB=0x602167588b880 00000400:00000200:28.0:1616513068.912916:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.216@tcp1 of length 224/224 into md 0x865b9f31 [1] + 26704 00000400:00000200:28.0:1616513068.912921:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513068.912924:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513068.912931:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513068.912934:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.912936:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513068.912940:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145344694272 00000100:00000200:8.0:1616513068.912940:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145344694400 00010000:00000200:8.0:1616513068.912944:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8cc80 x1691145344694400/t0(0) o400->b6909258-4e72-a61c-593e-f4f2b04a7f7d@10.31.140.216@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513068.912950:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691145344694400, offset 192 00010000:00000200:24.0:1616513068.912951:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1691145344694272/t0(0) o400->3e477f1c-5187-0074-c406-bf1053db8d85@10.31.140.216@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513068.912952:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.216@tcp1 00000400:00000200:8.0:1616513068.912956:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.216@tcp1 routed destination 00000400:00000200:8.0:1616513068.912961:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.216@tcp1(10.31.140.216@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.912964:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513068.912965:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691145344694272, offset 192 00000800:00000200:8.0:1616513068.912966:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.912967:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513068.912968:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513068.912971:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.912972:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.216@tcp1 00000800:00000200:11.2:1616513068.912976:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.912979:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.216@tcp1 routed destination 00000400:00000200:24.0:1616513068.912993:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.216@tcp1(10.31.140.216@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.913000:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.913006:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.913010:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.913012:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.913017:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.913023:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.913024:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513068.913030:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.913037:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513068.913039:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513068.913040:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513068.913046:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513068.913050:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70440 00000800:00000200:20.0:1616513068.913054:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513068.913077:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513068.913084:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513068.913087:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513068.913090:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513068.913093:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513068.913099:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fed0 00000800:00000200:14.0:1616513068.913102:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.940192:0:48272:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513068.940205:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513068.940248:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.940258:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513068.940260:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513068.940266:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513068.940267:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.206@tcp1 : PUT - for me 00000800:00000200:28.0:1616513068.940276:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513068.940277:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.206@tcp1 of length 224 into portal 12 MB=0x60187036d8340 00000400:00000200:22.0:1616513068.940284:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.206@tcp1 of length 224/224 into md 0xf96450b9 [1] + 1344 00000400:00000200:28.0:1616513068.940285:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.206@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.940290:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513068.940292:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513068.940294:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.206@tcp1 of length 224 into portal 26 MB=0x60187036d8280 00000800:00000200:22.0:1616513068.940300:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513068.940301:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.206@tcp1 of length 224/224 into md 0xf97c35d9 [1] + 0 00000800:00000200:22.0:1616513068.940303:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513068.940305:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcd8f8 00000800:00000200:22.0:1616513068.940305:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513068.940307:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690529249985344 00000400:00000200:28.0:1616513068.940308:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:30.0:1616513068.940311:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad6300 x1690529249985344/t0(0) o400->13fd1262-3184-db45-0e4e-43a03fc78510@10.31.142.206@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513068.940311:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513068.940315:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd8f8 00000100:00000200:30.0:1616513068.940318:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690529249985344, offset 192 00000800:00000200:28.0:1616513068.940318:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:30.0:1616513068.940320:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.206@tcp1 00000800:00000200:4.0:1616513068.940320:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513068.940321:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:30.0:1616513068.940323:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.206@tcp1 routed destination 00000800:00000200:28.0:1616513068.940324:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513068.940328:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.206@tcp1(10.31.142.206@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.940330:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.940332:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.940333:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513068.940334:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513068.940336:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.940341:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.940392:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690529249985152 00000800:00000200:12.0:1616513068.940398:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:24.0:1616513068.940403:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1690529249985152/t0(0) o400->09c3fd86-ce3d-462e-df42-f8afbca8fd3d@10.31.142.206@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513068.940405:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513068.940407:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513068.940414:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:24.0:1616513068.940417:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690529249985152, offset 192 00000400:00000200:12.0:1616513068.940418:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef3a8 00000800:00000200:12.0:1616513068.940422:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.940424:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.206@tcp1 00000800:00000200:12.0:1616513068.940425:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513068.940431:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.206@tcp1 routed destination 00000400:00000200:24.0:1616513068.940471:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.206@tcp1(10.31.142.206@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.940478:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.940484:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.940487:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.940489:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.940494:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.940498:0:48272:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.940501:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513068.940551:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.940559:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513068.940561:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513068.940568:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513068.940574:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:18.0:1616513068.940577:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513068.940580:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.982903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513068.982963:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.982972:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513068.982980:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.128@tcp1 : PUT - for me 00000400:00000200:10.0:1616513068.982989:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.128@tcp1 of length 224 into portal 26 MB=0x5ffde86516e00 00000400:00000200:10.0:1616513068.982995:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.128@tcp1 of length 224/224 into md 0xf97c35e9 [1] + 0 00000400:00000200:10.0:1616513068.983001:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240ee0 00000400:00000200:10.0:1616513068.983004:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513068.983006:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513068.983010:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ee0 00000800:00000200:10.0:1616513068.983013:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.983016:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.983017:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:10.0:1616513068.983024:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513068.983027:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.983028:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513068.983036:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.128@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.983045:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.128@tcp1 of length 224 into portal 12 MB=0x5ffde86516f40 00000400:00000200:22.0:1616513068.983052:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.128@tcp1 of length 224/224 into md 0xf96450b9 [1] + 1568 00000400:00000200:22.0:1616513068.983057:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513068.983059:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513068.983066:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.983069:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.983071:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513068.983074:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706084859712 00000100:00000200:24.0:1616513068.983076:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706084859392 00010000:00000200:30.0:1616513068.983079:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5100 x1688706084859712/t0(0) o400->ea246c4c-e122-7f40-06dd-aa05667d0e37@10.31.142.128@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513068.983084:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706084859712, offset 192 00000400:00000200:30.0:1616513068.983087:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.128@tcp1 00010000:00000200:24.0:1616513068.983087:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688706084859392/t0(0) o400->59edb4b9-21b8-6147-6b03-19cad337e399@10.31.142.128@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:30.0:1616513068.983089:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.128@tcp1 routed destination 00000400:00000200:30.0:1616513068.983095:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.128@tcp1(10.31.142.128@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513068.983097:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513068.983100:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513068.983101:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513068.983101:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706084859392, offset 192 00000800:00000200:30.0:1616513068.983102:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513068.983104:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513068.983107:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.128@tcp1 00000800:00000200:11.2:1616513068.983112:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513068.983114:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.128@tcp1 routed destination 00000400:00000200:24.0:1616513068.983129:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.128@tcp1(10.31.142.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.983136:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513068.983142:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.983145:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513068.983147:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513068.983152:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.983156:0:48277:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513068.983159:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513068.983167:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.983175:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513068.983178:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513068.983184:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513068.983188:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef4b8 00000800:00000200:4.0:1616513068.983192:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513068.983195:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513068.983220:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513068.983227:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513068.983230:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513068.983231:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513068.983237:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513068.983242:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e4c8 00000800:00000200:14.0:1616513068.983246:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513068.992828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513068.992889:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513068.992898:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513068.992907:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.177@tcp1 : PUT - for me 00000400:00000200:12.0:1616513068.992916:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.177@tcp1 of length 224 into portal 26 MB=0x5ffe9f9c71fc0 00000400:00000200:12.0:1616513068.992923:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.177@tcp1 of length 224/224 into md 0xf97c3609 [1] + 0 00000400:00000200:12.0:1616513068.992928:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240dd0 00000400:00000200:12.0:1616513068.992930:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513068.992933:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513068.992937:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240dd0 00000800:00000200:12.0:1616513068.992940:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513068.992943:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.992943:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513068.992947:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513068.993000:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688755266592704 00010000:00000200:24.0:1616513068.993011:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688755266592704/t0(0) o400->dae678e5-50ab-6c32-2a5a-7e9a563ccf80@10.31.141.177@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513068.993025:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688755266592704, offset 192 00000400:00000200:24.0:1616513068.993031:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.177@tcp1 00000400:00000200:24.0:1616513068.993038:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.177@tcp1 routed destination 00000400:00000200:24.0:1616513068.993053:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.177@tcp1(10.31.141.177@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.993060:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.993065:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.993068:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513068.993070:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.993075:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513068.993079:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513068.993081:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513068.993134:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513068.993141:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513068.993144:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513068.993150:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513068.993156:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:4.0:1616513068.993159:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513068.993162:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513068.993448:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513068.993505:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.993515:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513068.993521:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513068.993524:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.177@tcp1 : PUT - for me 00000400:00000200:22.0:1616513068.993532:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.177@tcp1 of length 224 into portal 12 MB=0x5ffe9f9c72780 00000400:00000200:22.0:1616513068.993539:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.177@tcp1 of length 224/224 into md 0xf96450b9 [1] + 1792 00000400:00000200:22.0:1616513068.993548:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513068.993551:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513068.993559:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513068.993562:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513068.993564:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513068.993632:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688755266594688 00010000:00000200:2.0:1616513068.993644:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0d80 x1688755266594688/t0(0) o400->354846b6-cee7-c74e-3a0c-ab8c06c18d7d@10.31.141.177@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513068.993658:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688755266594688, offset 192 00000400:00000200:2.0:1616513068.993664:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.177@tcp1 00000400:00000200:2.0:1616513068.993671:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.177@tcp1 routed destination 00000400:00000200:2.0:1616513068.993686:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.177@tcp1(10.31.141.177@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513068.993693:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513068.993698:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513068.993701:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513068.993703:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513068.993708:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.993714:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.993771:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.993778:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513068.993781:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513068.993787:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513068.993791:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed100 00000800:00000200:10.0:1616513068.993795:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.993798:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513068.998989:0:48279:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513068.999042:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513068.999051:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513068.999060:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.125@tcp1 : PUT - for me 00000400:00000200:18.0:1616513068.999069:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.125@tcp1 of length 224 into portal 26 MB=0x6033f1037f600 00000400:00000200:18.0:1616513068.999076:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.125@tcp1 of length 224/224 into md 0xf97c3619 [1] + 0 00000400:00000200:18.0:1616513068.999080:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240660 00000400:00000200:18.0:1616513068.999083:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513068.999086:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513068.999090:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:18.0:1616513068.999093:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513068.999097:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513068.999099:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513068.999103:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513068.999154:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692419250189824 00010000:00000200:24.0:1616513068.999165:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1692419250189824/t0(0) o400->66f2387f-51ca-4fc3-19ad-39e9fc8b3378@10.31.143.125@tcp1:25/0 lens 224/224 e 0 to 0 dl 1616513160 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513068.999171:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513068.999179:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692419250189824, offset 192 00000400:00000200:24.0:1616513068.999185:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.125@tcp1 00000400:00000200:24.0:1616513068.999192:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.125@tcp1 routed destination 00000400:00000200:24.0:1616513068.999207:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.125@tcp1(10.31.143.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513068.999213:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513068.999219:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513068.999222:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513068.999224:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513068.999229:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513068.999229:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:24.0:1616513068.999236:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513068.999238:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513068.999247:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.125@tcp1 : PUT - for me 00000400:00000200:20.0:1616513068.999256:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.125@tcp1 of length 224 into portal 12 MB=0x6033f1037f980 00000400:00000200:20.0:1616513068.999263:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.125@tcp1 of length 224/224 into md 0x865b9f31 [1] + 26928 00000400:00000200:20.0:1616513068.999269:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513068.999271:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513068.999278:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513068.999281:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513068.999282:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513068.999287:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513068.999288:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:8.0:1616513068.999288:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692419250190720 00000800:00000200:20.0:1616513068.999290:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513068.999291:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:8.0:1616513068.999293:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cef980 x1692419250190720/t0(0) o400->b457c8eb-40b6-f38e-9865-8a0757e0b0a5@10.31.143.125@tcp1:694/0 lens 224/224 e 0 to 0 dl 1616513074 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513068.999297:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:8.0:1616513068.999298:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692419250190720, offset 192 00000400:00000200:12.0:1616513068.999302:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f210 00000800:00000200:12.0:1616513068.999305:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513068.999305:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.125@tcp1 00000400:00000200:8.0:1616513068.999308:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.125@tcp1 routed destination 00000400:00000200:8.0:1616513068.999314:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.125@tcp1(10.31.143.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513068.999316:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513068.999318:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513068.999319:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513068.999320:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513068.999322:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513068.999326:0:48279:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513068.999379:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513068.999386:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513068.999389:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513068.999396:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513068.999400:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71320 00000800:00000200:10.0:1616513068.999403:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513068.999406:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.001846:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.004503:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.004506:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.004509:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.145@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.004513:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.145@tcp1 of length 224 into portal 26 MB=0x60393b5103b00 00000400:00000200:8.0:1616513069.004515:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.145@tcp1 of length 224/224 into md 0xf97c3649 [1] + 0 00000400:00000200:8.0:1616513069.004517:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241210 00000400:00000200:8.0:1616513069.004518:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.004519:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513069.004521:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241210 00000800:00000200:8.0:1616513069.004522:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.004538:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.004539:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.004540:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.004542:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.004544:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.145@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.004546:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.145@tcp1 of length 224 into portal 12 MB=0x60393b5103c00 00000400:00000200:8.0:1616513069.004548:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.145@tcp1 of length 224/224 into md 0x865b9f31 [1] + 27152 00000400:00000200:8.0:1616513069.004549:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.004550:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.004553:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.004554:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.004554:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.004556:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.004589:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692782793079552 00010000:00000200:24.0:1616513069.004599:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1692782793079552/t0(0) o400->8745b68f-059f-23a0-cbaa-151b34de39d5@10.31.140.145@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.004612:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692782793079552, offset 192 00000400:00000200:24.0:1616513069.004619:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.145@tcp1 00000400:00000200:24.0:1616513069.004626:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.145@tcp1 routed destination 00000100:00000200:16.0:1616513069.004629:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692782793079808 00000400:00000200:24.0:1616513069.004639:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.145@tcp1(10.31.140.145@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:16.0:1616513069.004642:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedc800 x1692782793079808/t0(0) o400->80acf1b9-b175-b6b4-b388-d6cf67a79ace@10.31.140.145@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513069.004645:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.004650:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.004653:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.004655:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513069.004656:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692782793079808, offset 192 00000800:00000200:24.0:1616513069.004659:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513069.004662:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.145@tcp1 00000100:00000200:24.0:1616513069.004665:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.004666:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513069.004669:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.145@tcp1 routed destination 00000400:00000200:16.0:1616513069.004684:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.145@tcp1(10.31.140.145@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.004693:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.004699:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.004701:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.004704:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.004709:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.004714:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513069.004719:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.004726:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513069.004729:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513069.004731:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513069.004736:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513069.004741:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fd38 00000800:00000200:4.0:1616513069.004744:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513069.004766:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.004773:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.004775:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.004781:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.004785:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42220 00000800:00000200:22.0:1616513069.004789:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.004791:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.011132:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513069.011192:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.011201:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513069.011210:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:10.0:1616513069.011220:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 328 into portal 26 MB=0x6046d825e67c0 00000400:00000200:10.0:1616513069.011226:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.120@tcp1 of length 328/328 into md 0xf97c3669 [1] + 0 00000400:00000200:10.0:1616513069.011231:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a438f8 00000400:00000200:10.0:1616513069.011234:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513069.011237:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513069.011241:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a438f8 00000800:00000200:10.0:1616513069.011244:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.011245:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.011247:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513069.011250:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.011304:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245435328 00010000:00000200:24.0:1616513069.011332:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1693718245435328/t0(0) o101->f7ff8064-8582-7463-157e-dedf721c5d06@10.31.138.120@tcp1:26/0 lens 328/344 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.011345:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 312 bytes to portal 25, xid 1693718245435328, offset 192 00000400:00000200:24.0:1616513069.011351:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.011358:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.011372:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.011378:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 312 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.011383:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.011386:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.011388:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.011393:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.011399:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.011400:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.011408:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.011410:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.011412:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.011414:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.011416:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ecc0 00000800:00000200:8.0:1616513069.011418:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.011419:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.013867:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513069.013925:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.013935:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513069.013944:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.013954:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 304 into portal 26 MB=0x6046d825e6800 00000400:00000200:20.0:1616513069.013961:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.120@tcp1 of length 304/304 into md 0xf97c3689 [1] + 0 00000400:00000200:20.0:1616513069.013965:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43980 00000400:00000200:20.0:1616513069.013968:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.013971:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513069.013975:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43980 00000800:00000200:20.0:1616513069.013978:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.013978:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.013981:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513069.013983:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.014033:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245435392 20000000:01000000:24.0:1616513069.014043:0:245737:0:(mgs_nids.c:632:mgs_get_ir_logs()) Reading IR log boslfs02-cliir bufsize 16384. 20000000:01000000:24.0:1616513069.014046:0:245737:0:(mgs_nids.c:204:mgs_nidtbl_read()) Read IR logs boslfs02 return with 0, version 871 00000400:00000200:24.0:1616513069.014055:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.014062:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.014068:0:245737:0:(lib-move.c:1738:lnet_handle_send()) rspt_next_hop_nid = 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513069.014078:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.014083:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.014090:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.014092:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.014094:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.014098:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.014102:0:245737:0:(niobuf.c:262:ptlrpc_start_bulk_transfer()) Transferring 0 pages 0 bytes via portal 33 id 12345-10.31.138.120@tcp1 mbits 0x6046d825e6800-0x6046d825e6800 00000800:00000200:7.2:1616513069.014102:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.014155:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.014162:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.014164:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.014171:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:22.0:1616513069.014175:0:6472:0:(events.c:450:server_bulk_callback()) event type 5, status 0, desc ffff8d5cc07c1c00 00000800:00000200:22.0:1616513069.014179:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.014182:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.016529:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.016536:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.016539:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.016541:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.163@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.016544:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.163@tcp1 of length 224 into portal 26 MB=0x5ffdc19aa5400 00000400:00000200:8.0:1616513069.016547:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.163@tcp1 of length 224/224 into md 0xf97c3699 [1] + 0 00000400:00000200:8.0:1616513069.016548:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23078 00000400:00000200:8.0:1616513069.016549:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.016550:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513069.016559:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23078 00000800:00000200:8.0:1616513069.016560:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.016561:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.016562:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.016562:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513069.016627:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695672034304 00010000:00000200:20.0:1616513069.016638:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688695672034304/t0(0) o400->b7b352e6-0878-f656-ee9e-c2b5215d57c1@10.31.141.163@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513069.016652:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695672034304, offset 192 00000400:00000200:20.0:1616513069.016658:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.163@tcp1 00000400:00000200:20.0:1616513069.016666:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.163@tcp1 routed destination 00000400:00000200:20.0:1616513069.016680:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.163@tcp1(10.31.141.163@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513069.016687:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513069.016692:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513069.016695:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513069.016697:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.016702:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.016707:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513069.016707:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513069.016708:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513069.016760:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.016767:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513069.016770:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513069.016772:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513069.016776:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513069.016782:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23188 00000800:00000200:4.0:1616513069.016787:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513069.016788:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.016798:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513069.016809:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : ACK - for me 00000400:00000200:14.0:1616513069.016820:0:6469:0:(lib-move.c:3970:lnet_parse_ack()) 10.242.46.233@o2ib1: ACK from 12345-10.31.138.120@tcp1 into md 0x866d2341 00000400:00000200:14.0:1616513069.016823:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.016826:0:6469:0:(events.c:450:server_bulk_callback()) event type 4, status 0, desc ffff8d5cc07c1c00 00000400:00000200:14.0:1616513069.016830:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f4b8 00000800:00000200:14.0:1616513069.016833:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.016836:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.016837:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.016839:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.016845:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513069.016850:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513069.016851:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.163@tcp1 : PUT - for me 00000400:00000200:14.0:1616513069.016857:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.163@tcp1 of length 224 into portal 12 MB=0x5ffdc19aa5680 00000400:00000200:14.0:1616513069.016863:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.163@tcp1 of length 224/224 into md 0xf96450b9 [1] + 2016 00000400:00000200:14.0:1616513069.016868:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.016882:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513069.016888:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.016890:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:24.0:1616513069.016891:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1693718245435392/t0(0) o256->f7ff8064-8582-7463-157e-dedf721c5d06@10.31.138.120@tcp1:26/0 lens 304/240 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513069.016892:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.016905:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 208 bytes to portal 25, xid 1693718245435392, offset 192 00000400:00000200:24.0:1616513069.016910:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.016916:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.016929:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.016935:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 208 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.016940:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.016942:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.016944:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.016948:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:2.0:1616513069.016950:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695672034944 00000800:00000200:7.2:1616513069.016954:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.016956:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:2.0:1616513069.016961:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa2400 x1688695672034944/t0(0) o400->962ef3c6-c998-bbbf-46b7-da92c5b7acb6@10.31.141.163@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513069.016968:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.016974:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:2.0:1616513069.016975:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695672034944, offset 192 00000400:00000200:18.0:1616513069.016977:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513069.016978:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513069.016981:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.163@tcp1 00000400:00000200:18.0:1616513069.016983:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513069.016987:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.163@tcp1 routed destination 00000400:00000200:18.0:1616513069.016988:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e330 00000800:00000200:18.0:1616513069.016993:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:2.0:1616513069.017001:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.163@tcp1(10.31.141.163@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.017007:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.017013:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.017015:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.017017:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.017022:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.017027:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.017033:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.017035:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.017036:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.017038:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.017039:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed210 00000800:00000200:12.0:1616513069.017041:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.017045:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513069.018511:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923344, portal 4 00000100:00000200:18.0:1616513069.018516:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923344, offset 0 00000400:00000200:18.0:1616513069.018523:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:18.0:1616513069.018529:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:18.0:1616513069.018541:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513069.018548:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:18.0:1616513069.018554:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513069.018557:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:18.0:1616513069.018560:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:18.0:1616513069.018565:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513069.018613:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.018633:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513069.018639:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513069.018643:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513069.018650:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513069.018657:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cd948e300 x1693067725923344/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513069.018670:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc220 00000800:00000200:27.0:1616513069.018675:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513069.018678:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513069.018739:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513069.018748:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.018754:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513069.018758:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513069.018763:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513069.018771:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626410 00000400:00000200:27.0:1616513069.018777:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164c2d [1] + 192 00000400:00000200:27.0:1616513069.018783:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513069.018787:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cd948e300 x1693067725923344/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513069.018802:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.018805:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513069.018806:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:18.0:1616513069.018810:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cd948e300 x1693067725923344/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:18.0:1616513069.018823:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cd948e300 x1693067725923344/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513069.018834:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc000 00000800:00000200:11.2:1616513069.019702:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513069.019758:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.019767:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513069.019776:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.019784:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 328 into portal 26 MB=0x6046d825e6840 00000400:00000200:20.0:1616513069.019790:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.120@tcp1 of length 328/328 into md 0xf97c36a9 [1] + 0 00000400:00000200:20.0:1616513069.019795:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22990 00000400:00000200:20.0:1616513069.019797:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.019800:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513069.019805:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22990 00000800:00000200:20.0:1616513069.019808:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.019811:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513069.019811:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513069.019815:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.019864:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245435456 00010000:00000200:24.0:1616513069.019885:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1693718245435456/t0(0) o101->f7ff8064-8582-7463-157e-dedf721c5d06@10.31.138.120@tcp1:26/0 lens 328/344 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.019897:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 312 bytes to portal 25, xid 1693718245435456, offset 192 00000400:00000200:24.0:1616513069.019902:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.019909:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.019922:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.019928:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 312 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.019933:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.019935:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.019937:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.019941:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.019945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.019949:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513069.019997:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.020004:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.020006:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.020013:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.020018:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eee0 00000800:00000200:14.0:1616513069.020022:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.020025:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.022731:0:48282:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513069.022785:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.022794:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513069.022802:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:10.0:1616513069.022810:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 296 into portal 26 MB=0x6046d825e6880 00000400:00000200:10.0:1616513069.022816:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.120@tcp1 of length 296/296 into md 0xf97c36c9 [1] + 0 00000400:00000200:10.0:1616513069.022822:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22880 00000400:00000200:10.0:1616513069.022824:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513069.022826:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513069.022830:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22880 00000800:00000200:10.0:1616513069.022833:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513069.022835:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513069.022837:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.022840:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.022889:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245435520 00010000:00000200:24.0:1616513069.022926:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1693718245435520/t0(0) o501->f7ff8064-8582-7463-157e-dedf721c5d06@10.31.138.120@tcp1:26/0 lens 296/272 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.022938:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 240 bytes to portal 25, xid 1693718245435520, offset 192 00000400:00000200:24.0:1616513069.022943:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.022950:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.022962:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.022968:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 240 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.022973:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.022975:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.022977:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.022981:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.022987:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.022987:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.023040:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.023047:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.023048:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.023050:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.023056:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.023062:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eaa0 00000800:00000200:12.0:1616513069.023065:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.025460:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513069.025535:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513069.025544:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513069.025553:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:4.0:1616513069.025562:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 272 into portal 26 MB=0x6046d825e68c0 00000400:00000200:4.0:1616513069.025568:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.120@tcp1 of length 272/272 into md 0xf97c36e9 [1] + 0 00000400:00000200:4.0:1616513069.025573:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22908 00000400:00000200:4.0:1616513069.025576:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513069.025578:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513069.025583:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22908 00000800:00000200:4.0:1616513069.025586:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.025588:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513069.025588:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513069.025592:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.025640:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245435584 00010000:00000200:24.0:1616513069.025670:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1693718245435584/t0(0) o503->f7ff8064-8582-7463-157e-dedf721c5d06@10.31.138.120@tcp1:26/0 lens 272/8416 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.025682:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 8384 bytes to portal 25, xid 1693718245435584, offset 192 00000400:00000200:24.0:1616513069.025686:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.025692:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.025704:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.025710:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 8384 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.025716:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.025718:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.025720:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.025724:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.025730:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.025731:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.025785:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.025792:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.025795:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.025801:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513069.025805:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513069.025809:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513069.025811:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.025813:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.025816:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.025840:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.025851:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.025853:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513069.025853:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513069.025860:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.025867:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:22.0:1616513069.025871:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.029084:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513069.029134:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.029142:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513069.029150:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:14.0:1616513069.029158:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 328 into portal 26 MB=0x6046d825e6900 00000400:00000200:14.0:1616513069.029173:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.120@tcp1 of length 328/328 into md 0xf97c36f9 [1] + 0 00000400:00000200:14.0:1616513069.029178:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22d48 00000400:00000200:14.0:1616513069.029181:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.029183:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513069.029187:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22d48 00000800:00000200:14.0:1616513069.029190:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.029191:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.029192:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.029196:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.029245:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245435648 00010000:00000200:24.0:1616513069.029264:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1693718245435648/t0(0) o101->f7ff8064-8582-7463-157e-dedf721c5d06@10.31.138.120@tcp1:26/0 lens 328/344 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.029275:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 312 bytes to portal 25, xid 1693718245435648, offset 192 00000400:00000200:24.0:1616513069.029282:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.029288:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.029300:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.029305:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 312 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.029310:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.029312:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.029313:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.029317:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.029323:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.029325:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.029332:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.029334:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.029335:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.029337:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.029340:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eee0 00000800:00000200:8.0:1616513069.029341:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.029342:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.031789:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513069.031841:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.031849:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513069.031858:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.031866:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 288 into portal 26 MB=0x6046d825e6940 00000400:00000200:20.0:1616513069.031872:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.120@tcp1 of length 288/288 into md 0xf97c3709 [1] + 0 00000400:00000200:20.0:1616513069.031876:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23760 00000400:00000200:20.0:1616513069.031879:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.031881:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513069.031885:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23760 00000800:00000200:20.0:1616513069.031888:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.031890:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513069.031892:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513069.031895:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.031942:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245435712 00010000:00000200:24.0:1616513069.031967:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1693718245435712/t0(0) o501->f7ff8064-8582-7463-157e-dedf721c5d06@10.31.138.120@tcp1:26/0 lens 288/272 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.031979:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 240 bytes to portal 25, xid 1693718245435712, offset 192 00000400:00000200:24.0:1616513069.031984:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.031990:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.032002:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.032007:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 240 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.032011:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.032014:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.032015:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.032019:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.032025:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.032025:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.032040:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.032049:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513069.032052:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513069.032058:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.032064:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f4b8 00000800:00000200:18.0:1616513069.032067:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.032070:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.034721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513069.034772:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.034780:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513069.034789:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:22.0:1616513069.034797:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 272 into portal 26 MB=0x6046d825e6980 00000400:00000200:22.0:1616513069.034803:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.120@tcp1 of length 272/272 into md 0xf97c3719 [1] + 0 00000400:00000200:22.0:1616513069.034807:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22a18 00000400:00000200:22.0:1616513069.034811:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.034814:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513069.034817:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22a18 00000800:00000200:22.0:1616513069.034820:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.034822:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513069.034824:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.034827:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.034875:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245435776 00010000:00000200:24.0:1616513069.034898:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1693718245435776/t0(0) o503->f7ff8064-8582-7463-157e-dedf721c5d06@10.31.138.120@tcp1:26/0 lens 272/8416 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.034910:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 8384 bytes to portal 25, xid 1693718245435776, offset 192 00000400:00000200:24.0:1616513069.034915:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.034921:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.034933:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.034941:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 8384 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.034947:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.034949:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.034950:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.034954:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.034960:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.034961:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.034968:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.034970:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.034972:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.034988:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.035000:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.035008:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513069.035012:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513069.035016:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513069.035017:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513069.035020:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.035023:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.035029:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.035032:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.035032:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513069.035035:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513069.035035:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.035037:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fd38 00000800:00000200:8.0:1616513069.035039:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.038348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513069.038400:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.038409:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513069.038417:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.038425:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 328 into portal 26 MB=0x6046d825e69c0 00000400:00000200:20.0:1616513069.038451:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.120@tcp1 of length 328/328 into md 0xf97c3739 [1] + 0 00000400:00000200:20.0:1616513069.038455:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22088 00000800:00000200:12.0:1616513069.038455:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513069.038458:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.038460:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513069.038465:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22088 00000800:00000200:20.0:1616513069.038468:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.038470:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.038474:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.038520:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245435840 00010000:00000200:24.0:1616513069.038538:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1693718245435840/t0(0) o101->f7ff8064-8582-7463-157e-dedf721c5d06@10.31.138.120@tcp1:26/0 lens 328/344 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.038549:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 312 bytes to portal 25, xid 1693718245435840, offset 192 00000400:00000200:24.0:1616513069.038553:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.038559:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.038570:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.038575:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 312 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.038579:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.038582:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.038583:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.038587:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.038592:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.038592:0:48284:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.038645:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.038652:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513069.038654:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513069.038662:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.038667:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:18.0:1616513069.038670:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.038673:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.041348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513069.041406:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.041415:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513069.041423:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:14.0:1616513069.041431:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 304 into portal 26 MB=0x6046d825e6a00 00000400:00000200:14.0:1616513069.041448:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.120@tcp1 of length 304/304 into md 0xf97c3759 [1] + 0 00000400:00000200:14.0:1616513069.041452:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23dc0 00000400:00000200:14.0:1616513069.041455:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.041457:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513069.041458:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513069.041461:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23dc0 00000800:00000200:14.0:1616513069.041464:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.041466:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.041468:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.041517:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245435904 00010000:00000200:24.0:1616513069.041534:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1693718245435904/t0(0) o501->f7ff8064-8582-7463-157e-dedf721c5d06@10.31.138.120@tcp1:26/0 lens 304/272 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc -2/0 00000100:00000200:24.0:1616513069.041545:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 240 bytes to portal 25, xid 1693718245435904, offset 192 00000400:00000200:24.0:1616513069.041550:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:24.0:1616513069.041556:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:24.0:1616513069.041567:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.041572:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 240 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.041576:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.041578:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.041581:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.041585:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.041591:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.041591:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.041598:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.041600:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.041601:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.041603:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.041605:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:4.0:1616513069.041606:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.041609:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.043996:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.044048:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.044057:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.044065:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.32@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.044074:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.32@tcp1 of length 224 into portal 26 MB=0x60393c3408700 00000400:00000200:12.0:1616513069.044080:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.32@tcp1 of length 224/224 into md 0xf97c3779 [1] + 0 00000400:00000200:12.0:1616513069.044090:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22e58 00000400:00000200:12.0:1616513069.044092:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.044095:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513069.044099:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22e58 00000800:00000200:12.0:1616513069.044101:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.044103:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.044104:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.044109:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.044111:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.044112:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513069.044119:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.32@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.044127:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.32@tcp1 of length 224 into portal 12 MB=0x60393c3408780 00000400:00000200:20.0:1616513069.044133:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.32@tcp1 of length 224/224 into md 0x865b9f31 [1] + 27376 00000400:00000200:20.0:1616513069.044137:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.044139:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513069.044146:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.044148:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.044150:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.044154:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783031125760 00010000:00000200:24.0:1616513069.044163:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1692783031125760/t0(0) o400->26a14279-ea92-0ffb-ca9b-8bde2cea7a3d@10.31.140.32@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.044174:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783031125760, offset 192 00000400:00000200:24.0:1616513069.044178:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.32@tcp1 00000400:00000200:24.0:1616513069.044184:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.32@tcp1 routed destination 00000400:00000200:24.0:1616513069.044196:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.32@tcp1(10.31.140.32@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.044200:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.044205:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.044207:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.044209:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.044212:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513069.044212:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783031125888 00000100:00000200:24.0:1616513069.044217:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.044217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:16.0:1616513069.044224:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceda00 x1692783031125888/t0(0) o400->27048e73-bbde-63f1-91d1-b2166723c821@10.31.140.32@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.044239:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783031125888, offset 192 00000400:00000200:16.0:1616513069.044245:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.32@tcp1 00000400:00000200:16.0:1616513069.044253:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.32@tcp1 routed destination 00000400:00000200:16.0:1616513069.044268:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.32@tcp1(10.31.140.32@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513069.044270:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513069.044274:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513069.044277:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513069.044279:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513069.044280:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.044283:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.044285:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513069.044286:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513069.044289:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513069.044291:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f320 00000800:00000200:18.0:1616513069.044294:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513069.044294:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513069.044297:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513069.044300:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.044302:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.044303:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.044304:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.044305:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a434b8 00000800:00000200:12.0:1616513069.044306:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.044308:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.056551:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513069.056604:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.056612:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513069.056620:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.42@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.056628:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.42@tcp1 of length 224 into portal 26 MB=0x60196194cfd00 00000400:00000200:20.0:1616513069.056633:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.42@tcp1 of length 224/224 into md 0xf97c3799 [1] + 0 00000400:00000200:20.0:1616513069.056640:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22ee0 00000400:00000200:20.0:1616513069.056643:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.056645:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513069.056649:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ee0 00000800:00000200:20.0:1616513069.056652:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.056654:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513069.056655:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.056658:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.056705:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690594041462016 00010000:00000200:24.0:1616513069.056713:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1690594041462016/t0(0) o400->5c82b6a9-d705-4a57-fa4f-efe89fc76560@10.31.141.42@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.056723:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690594041462016, offset 192 00000400:00000200:24.0:1616513069.056728:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.42@tcp1 00000400:00000200:24.0:1616513069.056734:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.42@tcp1 routed destination 00000400:00000200:24.0:1616513069.056745:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.42@tcp1(10.31.141.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.056750:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.056754:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.056756:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.056758:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.056761:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.056766:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.056769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.056827:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.056833:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.056836:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.056843:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.056848:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f650 00000800:00000200:22.0:1616513069.056852:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.056855:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.056915:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.056969:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.056977:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.056978:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.056986:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.42@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.056994:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.42@tcp1 of length 224 into portal 12 MB=0x60196194d0500 00000400:00000200:12.0:1616513069.056999:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.42@tcp1 of length 224/224 into md 0x865b9f31 [1] + 27600 00000400:00000200:12.0:1616513069.057004:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.057006:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513069.057013:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.057016:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.057017:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.057076:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690594041464064 00010000:00000200:16.0:1616513069.057089:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa46300 x1690594041464064/t0(0) o400->94bcf895-19f2-0a79-85c2-ac32ff6cf753@10.31.141.42@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.057103:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690594041464064, offset 192 00000400:00000200:16.0:1616513069.057109:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.42@tcp1 00000400:00000200:16.0:1616513069.057116:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.42@tcp1 routed destination 00000400:00000200:16.0:1616513069.057131:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.42@tcp1(10.31.141.42@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.057138:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.057144:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.057147:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.057149:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.057153:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.057158:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513069.057169:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.057176:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513069.057179:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513069.057179:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513069.057185:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513069.057190:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a435c8 00000800:00000200:4.0:1616513069.057194:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513069.066588:0:6482:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923360, portal 4 00000100:00000200:12.0:1616513069.066593:0:6482:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923360, offset 0 00000400:00000200:12.0:1616513069.066600:0:6482:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:12.0:1616513069.066606:0:6482:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:12.0:1616513069.066618:0:6482:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513069.066625:0:6482:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:12.0:1616513069.066631:0:6482:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513069.066634:0:6482:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:12.0:1616513069.066636:0:6482:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:12.0:1616513069.066641:0:6482:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513069.066688:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.066706:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513069.066712:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513069.066715:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513069.066722:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513069.066732:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdeb8f980 x1693067725923360/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513069.066745:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241188 00000800:00000200:27.0:1616513069.066750:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513069.066753:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513069.066860:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513069.066869:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.066876:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513069.066879:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513069.066884:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513069.066892:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626420 00000400:00000200:27.0:1616513069.066899:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164c3d [1] + 192 00000400:00000200:27.0:1616513069.066904:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513069.066908:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdeb8f980 x1693067725923360/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513069.066923:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.066926:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513069.066927:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:12.0:1616513069.066932:0:6482:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdeb8f980 x1693067725923360/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:12.0:1616513069.066945:0:6482:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdeb8f980 x1693067725923360/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513069.066958:0:6482:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240dd0 00000800:00000200:9.2F:1616513069.072516:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da656200] (20)++ 00000800:00000200:18.0:1616513069.072537:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da656200] (21)++ 00000800:00000200:18.0:1616513069.072547:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.161@o2ib1 00000400:00000200:18.0:1616513069.072556:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.161@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513069.072564:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.161@o2ib1 of length 224 into portal 26 MB=0x5ffeee7079d80 00000400:00000200:18.0:1616513069.072570:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.161@o2ib1 of length 224/224 into md 0xf97c37a9 [1] + 0 00000400:00000200:18.0:1616513069.072575:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee235c8 00000400:00000200:18.0:1616513069.072578:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.072580:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513069.072584:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee235c8 00000800:00000200:18.0:1616513069.072588:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da656200] (22)++ 00000800:00000200:18.0:1616513069.072590:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da656200] (23)-- 00000800:00000200:10.0:1616513069.072592:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (23)-- 00000800:00000200:18.0:1616513069.072596:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (21)-- 00000100:00000200:24.0:1616513069.072639:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688776426888576 00010000:00000200:24.0:1616513069.072647:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688776426888576/t0(0) o400->1362588c-42ef-7a52-81b5-56fe9de631e5@10.242.46.161@o2ib1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.072659:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688776426888576, offset 192 00000400:00000200:24.0:1616513069.072664:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.161@o2ib1 00000400:00000200:24.0:1616513069.072669:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.161@o2ib1 local destination 00000400:00000200:24.0:1616513069.072679:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.161@o2ib1(10.242.46.161@o2ib1:10.242.46.161@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.072684:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.161@o2ib1 00000800:00000200:24.0:1616513069.072688:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9d80] -> 10.242.46.161@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.072690:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da656200] (20)++ 00000800:00000200:24.0:1616513069.072691:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da656200] (21)++ 00000800:00000200:24.0:1616513069.072695:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da656200] (22)-- 00000100:00000200:24.0:1616513069.072700:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:9.2:1616513069.072701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da656200] (21)++ 00000800:00000200:22.0:1616513069.072756:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da656200] (22)++ 00000800:00000200:22.0:1616513069.072763:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da656200] (23)-- 00000400:00000200:22.0:1616513069.072766:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.072772:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.161@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.072778:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:22.0:1616513069.072781:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (22)-- 00000800:00000200:22.0:1616513069.072784:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da656200] (21)++ 00000800:00000200:22.0:1616513069.072790:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.161@o2ib1 00000800:00000200:10.0:1616513069.072793:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (22)-- 00000400:00000200:22.0:1616513069.072798:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.161@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513069.072814:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.161@o2ib1 of length 224 into portal 12 MB=0x5ffeee7079e00 00000400:00000200:22.0:1616513069.072820:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.161@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 2240 00000400:00000200:22.0:1616513069.072825:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.072828:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513069.072834:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da656200] (21)++ 00000800:00000200:22.0:1616513069.072837:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da656200] (22)-- 00000800:00000200:22.0:1616513069.072838:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (21)-- 00000100:00000200:2.0:1616513069.072898:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688776426888704 00010000:00000200:2.0:1616513069.072910:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad3180 x1688776426888704/t0(0) o400->c1c098a5-4d9c-25e4-ad0e-8a772977eb00@10.242.46.161@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513069.072927:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688776426888704, offset 192 00000400:00000200:2.0:1616513069.072933:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.161@o2ib1 00000400:00000200:2.0:1616513069.072940:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.161@o2ib1 local destination 00000400:00000200:2.0:1616513069.072954:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.161@o2ib1(10.242.46.161@o2ib1:10.242.46.161@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.072960:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.161@o2ib1 00000800:00000200:2.0:1616513069.072966:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9d80] -> 10.242.46.161@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.072969:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da656200] (20)++ 00000800:00000200:2.0:1616513069.072971:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da656200] (21)++ 00000800:00000200:2.0:1616513069.072975:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da656200] (22)-- 00000800:00000200:9.2:1616513069.072981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da656200] (21)++ 00000800:00000200:18.0:1616513069.072990:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da656200] (22)++ 00000800:00000200:18.0:1616513069.072994:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da656200] (23)-- 00000400:00000200:18.0:1616513069.072996:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513069.072998:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (22)-- 00000400:00000200:18.0:1616513069.073002:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.161@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.073005:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed078 00000800:00000200:18.0:1616513069.073008:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (21)-- 00000800:00000200:11.2:1616513069.081807:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.081816:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.081818:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.081821:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.140@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.081826:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.140@tcp1 of length 224 into portal 26 MB=0x60393cafb0e00 00000400:00000200:8.0:1616513069.081828:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.140@tcp1 of length 224/224 into md 0xf97c37c9 [1] + 0 00000400:00000200:8.0:1616513069.081830:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23f58 00000400:00000200:8.0:1616513069.081831:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.081832:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513069.081834:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23f58 00000800:00000200:8.0:1616513069.081835:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.081836:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.081837:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.081837:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.081840:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.081842:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.140@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.081844:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.140@tcp1 of length 224 into portal 12 MB=0x60393cafb0e80 00000400:00000200:8.0:1616513069.081846:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.140@tcp1 of length 224/224 into md 0x865b9f31 [1] + 27824 00000400:00000200:8.0:1616513069.081847:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.081848:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.081851:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.081851:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.081852:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.081853:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.081919:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783160790528 00010000:00000200:24.0:1616513069.081930:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1692783160790528/t0(0) o400->e6116f41-df48-51ca-11c1-c7bf0061a15f@10.31.140.140@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.081937:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783160790656 00000100:00000200:24.0:1616513069.081944:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783160790528, offset 192 00010000:00000200:16.0:1616513069.081949:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedb180 x1692783160790656/t0(0) o400->a9beb37b-1a17-eb13-d225-d8538a236edc@10.31.140.140@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513069.081950:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.140@tcp1 00000400:00000200:24.0:1616513069.081958:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.140@tcp1 routed destination 00000100:00000200:16.0:1616513069.081963:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783160790656, offset 192 00000400:00000200:16.0:1616513069.081968:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.140@tcp1 00000400:00000200:24.0:1616513069.081973:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.140@tcp1(10.31.140.140@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513069.081975:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.140@tcp1 routed destination 00000800:00000200:24.0:1616513069.081979:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513069.081989:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.140@tcp1(10.31.140.140@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.081995:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.081996:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.081999:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.082001:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.082001:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.082003:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.082006:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513069.082006:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.082010:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.082012:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.082014:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513069.082015:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513069.082069:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.082076:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.082077:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513069.082079:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513069.082085:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513069.082087:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513069.082089:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43188 00000800:00000200:4.0:1616513069.082093:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513069.082094:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.082096:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.082103:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.082108:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fd38 00000800:00000200:14.0:1616513069.082112:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.082115:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.091006:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513069.091016:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513069.091023:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513069.091025:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513069.091033:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.253@tcp1 : PUT - for me 00000400:00000200:10.0:1616513069.091042:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.253@tcp1 of length 224 into portal 12 MB=0x6040a1cd99000 00000400:00000200:10.0:1616513069.091047:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.253@tcp1 of length 224/224 into md 0xf96450b9 [1] + 2464 00000400:00000200:10.0:1616513069.091052:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513069.091055:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513069.091061:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513069.091064:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513069.091066:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.091068:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513069.091087:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.091096:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.091105:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.253@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.091113:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.253@tcp1 of length 224 into portal 26 MB=0x6040a1cd98f40 00000400:00000200:12.0:1616513069.091119:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.253@tcp1 of length 224/224 into md 0xf97c37d9 [1] + 0 00000400:00000200:12.0:1616513069.091123:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23320 00000400:00000200:12.0:1616513069.091126:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513069.091127:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693291340468224 00000100:00000200:12.0:1616513069.091128:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513069.091132:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23320 00000800:00000200:12.0:1616513069.091135:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.091138:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.091139:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:2.0:1616513069.091139:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdf080 x1693291340468224/t0(0) o400->4c03e2c7-90e5-030b-6a12-1f0d5de50fe4@10.31.132.253@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513069.091142:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:2.0:1616513069.091154:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693291340468224, offset 192 00000400:00000200:2.0:1616513069.091160:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.253@tcp1 00000400:00000200:2.0:1616513069.091167:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.253@tcp1 routed destination 00000400:00000200:2.0:1616513069.091182:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.253@tcp1(10.31.132.253@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.091188:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513069.091194:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:24.0:1616513069.091196:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693291340468032 00000800:00000200:2.0:1616513069.091197:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513069.091199:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513069.091204:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:24.0:1616513069.091207:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1693291340468032/t0(0) o400->7c68e3f9-776e-09c2-1d52-090eb080571e@10.31.132.253@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513069.091209:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.091221:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693291340468032, offset 192 00000400:00000200:24.0:1616513069.091227:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.253@tcp1 00000400:00000200:24.0:1616513069.091235:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.253@tcp1 routed destination 00000400:00000200:24.0:1616513069.091261:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.253@tcp1(10.31.132.253@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.091268:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513069.091268:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.091277:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513069.091278:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.091280:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513069.091281:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513069.091283:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.091287:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513069.091288:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.091292:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed188 00000800:00000200:11.2:1616513069.091292:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.091294:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513069.091296:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.091299:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513069.091314:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.091316:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.091317:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.091319:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.091321:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eee0 00000800:00000200:8.0:1616513069.091322:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.091323:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.096643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513069.096703:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.096712:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513069.096721:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.50@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.096731:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.50@tcp1 of length 224 into portal 26 MB=0x6019810b14cc0 00000400:00000200:20.0:1616513069.096737:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.50@tcp1 of length 224/224 into md 0xf97c37f9 [1] + 0 00000400:00000200:20.0:1616513069.096742:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23e48 00000400:00000200:20.0:1616513069.096745:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.096749:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513069.096753:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23e48 00000800:00000200:20.0:1616513069.096756:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.096757:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.096759:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513069.096764:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513069.096767:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.096767:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.096776:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.50@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.096784:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.50@tcp1 of length 224 into portal 12 MB=0x6019810b14dc0 00000400:00000200:12.0:1616513069.096791:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.50@tcp1 of length 224/224 into md 0x865b9f31 [1] + 28048 00000400:00000200:12.0:1616513069.096796:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.096798:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513069.096805:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.096808:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.096810:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.096814:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690602486975680 00010000:00000200:24.0:1616513069.096824:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1690602486975680/t0(0) o400->784ef5b9-30f5-46c3-1bb3-b75bfc2c8edd@10.31.142.50@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.096838:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690602486975680, offset 192 00000400:00000200:24.0:1616513069.096844:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.50@tcp1 00000400:00000200:24.0:1616513069.096850:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.50@tcp1 routed destination 00000400:00000200:24.0:1616513069.096864:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.50@tcp1(10.31.142.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.096870:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:16.0:1616513069.096871:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690602486975936 00000800:00000200:24.0:1616513069.096875:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.096878:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.096880:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:16.0:1616513069.096883:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa44800 x1690602486975936/t0(0) o400->9ff99390-55da-5b1c-bc5c-605ecbf31f8d@10.31.142.50@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513069.096884:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.096890:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.096891:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:16.0:1616513069.096898:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690602486975936, offset 192 00000400:00000200:16.0:1616513069.096904:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.50@tcp1 00000400:00000200:16.0:1616513069.096911:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.50@tcp1 routed destination 00000400:00000200:16.0:1616513069.096926:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.50@tcp1(10.31.142.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.096933:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.096938:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.096941:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513069.096942:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513069.096943:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.096948:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513069.096949:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513069.096952:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513069.096953:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513069.096954:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.096959:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.096964:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eee0 00000800:00000200:18.0:1616513069.096970:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513069.097006:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.097013:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.097014:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513069.097016:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513069.097022:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513069.097026:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43298 00000800:00000200:4.0:1616513069.097030:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.101433:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513069.101487:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.101496:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513069.101505:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.80@tcp1 : PUT - for me 00000400:00000200:14.0:1616513069.101514:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.80@tcp1 of length 224 into portal 26 MB=0x601894bebbfc0 00000400:00000200:14.0:1616513069.101521:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.80@tcp1 of length 224/224 into md 0xf97c3819 [1] + 0 00000400:00000200:14.0:1616513069.101526:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22ff0 00000400:00000200:14.0:1616513069.101528:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.101531:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513069.101535:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ff0 00000800:00000200:14.0:1616513069.101538:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.101541:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513069.101541:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.101548:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.101551:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.101552:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513069.101561:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.80@tcp1 : PUT - for me 00000400:00000200:22.0:1616513069.101572:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.80@tcp1 of length 224 into portal 12 MB=0x601894bebc240 00000400:00000200:22.0:1616513069.101578:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.80@tcp1 of length 224/224 into md 0xf96450b9 [1] + 2688 00000400:00000200:22.0:1616513069.101583:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.101586:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513069.101593:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.101596:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.101597:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690539056152512 00000800:00000200:22.0:1616513069.101598:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513069.101607:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1690539056152512/t0(0) o400->4984a490-f375-d6fd-8591-24425c60b23e@10.31.140.80@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.101620:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690539056152512, offset 192 00000400:00000200:24.0:1616513069.101626:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.80@tcp1 00000400:00000200:24.0:1616513069.101633:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.80@tcp1 routed destination 00000400:00000200:24.0:1616513069.101647:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.80@tcp1(10.31.140.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.101653:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:2.0:1616513069.101655:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690539056153152 00000800:00000200:24.0:1616513069.101658:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.101663:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.101665:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:2.0:1616513069.101666:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad4c80 x1690539056153152/t0(0) o400->5678821d-ded0-72f0-091a-71590aa63bc4@10.31.140.80@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513069.101669:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.101675:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.101676:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:2.0:1616513069.101680:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690539056153152, offset 192 00000400:00000200:2.0:1616513069.101685:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.80@tcp1 00000400:00000200:2.0:1616513069.101693:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.80@tcp1 routed destination 00000400:00000200:2.0:1616513069.101706:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.80@tcp1(10.31.140.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.101714:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.101719:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.101722:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.101724:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.101728:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513069.101731:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513069.101733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513069.101738:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513069.101741:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513069.101741:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.101748:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.101753:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:18.0:1616513069.101768:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513069.101799:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.101806:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513069.101809:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513069.101816:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513069.101822:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec5d8 00000800:00000200:12.0:1616513069.101822:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.101826:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.138310:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.138320:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.138324:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.138327:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.235@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.138331:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.235@tcp1 of length 224 into portal 12 MB=0x603e0ecf18dc0 00000400:00000200:8.0:1616513069.138333:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.235@tcp1 of length 224/224 into md 0x865b9f31 [1] + 28272 00000400:00000200:8.0:1616513069.138335:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.138337:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.138341:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.138342:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.138342:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.138343:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.138412:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114443075008 00010000:00000200:16.0:1616513069.138425:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b98480 x1693114443075008/t0(0) o400->8317c132-c299-5929-28a2-e62d0b0bd498@10.31.136.235@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.138456:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114443075008, offset 192 00000400:00000200:16.0:1616513069.138463:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.235@tcp1 00000400:00000200:16.0:1616513069.138471:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.235@tcp1 routed destination 00000400:00000200:16.0:1616513069.138488:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.235@tcp1(10.31.136.235@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.138494:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:7.2:1616513069.138498:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.138500:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.138503:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.138505:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513069.138510:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.138552:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.138561:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513069.138569:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.235@tcp1 : PUT - for me 00000400:00000200:14.0:1616513069.138577:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.235@tcp1 of length 224 into portal 26 MB=0x603e0ecf18d40 00000400:00000200:14.0:1616513069.138584:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.235@tcp1 of length 224/224 into md 0xf97c3829 [1] + 0 00000400:00000200:14.0:1616513069.138588:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22770 00000400:00000200:14.0:1616513069.138590:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.138593:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513069.138597:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22770 00000800:00000200:14.0:1616513069.138600:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513069.138602:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513069.138604:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513069.138606:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.138614:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.138617:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513069.138618:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513069.138623:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.138627:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42330 00000800:00000200:22.0:1616513069.138631:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.138672:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114443074880 00010000:00000200:24.0:1616513069.138681:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1693114443074880/t0(0) o400->9007707d-6d1e-307d-5b3c-a6c76beab57f@10.31.136.235@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.138694:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114443074880, offset 192 00000400:00000200:24.0:1616513069.138699:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.235@tcp1 00000400:00000200:24.0:1616513069.138705:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.235@tcp1 routed destination 00000400:00000200:24.0:1616513069.138718:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.235@tcp1(10.31.136.235@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.138724:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.138729:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.138731:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.138733:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.138737:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.138742:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.138742:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.138753:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.138759:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.138761:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513069.138765:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513069.138767:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.138772:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:12.0:1616513069.138775:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.143736:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.143744:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.143746:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.143748:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.140@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.143751:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.140@tcp1 of length 224 into portal 12 MB=0x5ffd79dbda480 00000400:00000200:8.0:1616513069.143752:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.140@tcp1 of length 224/224 into md 0x865b9f31 [1] + 28496 00000400:00000200:8.0:1616513069.143754:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.143755:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.143757:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.143757:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.143758:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.143758:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.143761:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.143778:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.140@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.143780:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.140@tcp1 of length 224 into portal 26 MB=0x5ffd79dbda400 00000400:00000200:8.0:1616513069.143782:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.140@tcp1 of length 224/224 into md 0xf97c3839 [1] + 0 00000400:00000200:8.0:1616513069.143783:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22cc0 00000400:00000200:8.0:1616513069.143784:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.143785:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513069.143787:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22cc0 00000800:00000200:8.0:1616513069.143787:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.143788:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.143788:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.143789:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.143831:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676413056128 00010000:00000200:16.0:1616513069.143843:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9f080 x1688676413056128/t0(0) o400->edf524bd-80b1-550b-6d8f-1c2c613ef9d7@10.31.143.140@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513069.143852:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676413056000 00000100:00000200:16.0:1616513069.143857:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688676413056128, offset 192 00010000:00000200:24.0:1616513069.143863:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688676413056000/t0(0) o400->789efaf2-d09c-a38b-a9d0-dcc2fe51eb41@10.31.143.140@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513069.143864:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.140@tcp1 00000400:00000200:16.0:1616513069.143871:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.140@tcp1 routed destination 00000100:00000200:24.0:1616513069.143878:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688676413056000, offset 192 00000400:00000200:24.0:1616513069.143884:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.140@tcp1 00000400:00000200:16.0:1616513069.143886:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.140@tcp1(10.31.143.140@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513069.143892:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.140@tcp1 routed destination 00000800:00000200:16.0:1616513069.143893:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513069.143906:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.140@tcp1(10.31.143.140@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.143913:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.143914:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.143916:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.143919:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.143919:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.143922:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.143923:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513069.143924:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.143928:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513069.143929:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513069.143933:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.143935:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513069.143982:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.143988:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513069.143991:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513069.143997:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513069.144000:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513069.144001:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a424c8 00000800:00000200:18.0:1616513069.144005:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.144007:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513069.144007:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.144008:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.144010:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.144016:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.144020:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:12.0:1616513069.144024:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:1.0:1616513069.162494:0:6491:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923376, portal 4 00000100:00000200:1.0:1616513069.162501:0:6491:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923376, offset 0 00000400:00000200:1.0:1616513069.162510:0:6491:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:1.0:1616513069.162518:0:6491:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:1.0:1616513069.162532:0:6491:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:1.0:1616513069.162540:0:6491:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:1.0:1616513069.162547:0:6491:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513069.162550:0:6491:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:1.0:1616513069.162553:0:6491:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:1.0:1616513069.162560:0:6491:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513069.162620:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.162639:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513069.162644:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513069.162647:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513069.162654:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513069.162661:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4558ad5580 x1693067725923376/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513069.162674:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84a18 00000800:00000200:27.0:1616513069.162679:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513069.162681:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513069.162779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513069.162789:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.162795:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513069.162799:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513069.162803:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513069.162812:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626430 00000400:00000200:27.0:1616513069.162818:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164c4d [1] + 192 00000400:00000200:27.0:1616513069.162824:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513069.162830:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4558ad5580 x1693067725923376/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513069.162844:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.162847:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513069.162848:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:1.0:1616513069.162853:0:6491:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4558ad5580 x1693067725923376/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513069.162866:0:6491:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4558ad5580 x1693067725923376/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513069.162877:0:6491:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84330 00000800:00000200:7.2:1616513069.170726:0:48299:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513069.170780:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.170790:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513069.170799:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.138@tcp1 : PUT - for me 00000400:00000200:10.0:1616513069.170809:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.138@tcp1 of length 224 into portal 26 MB=0x5ffdd699dfec0 00000400:00000200:10.0:1616513069.170816:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.138@tcp1 of length 224/224 into md 0xf97c3849 [1] + 0 00000400:00000200:10.0:1616513069.170821:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23100 00000400:00000200:10.0:1616513069.170824:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513069.170826:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513069.170831:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23100 00000800:00000200:10.0:1616513069.170834:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.170835:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513069.170837:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513069.170841:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513069.170844:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.170845:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513069.170854:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.138@tcp1 : PUT - for me 00000400:00000200:22.0:1616513069.170864:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.138@tcp1 of length 224 into portal 12 MB=0x5ffdd699e0140 00000400:00000200:22.0:1616513069.170870:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.138@tcp1 of length 224/224 into md 0xf96450b9 [1] + 2912 00000400:00000200:22.0:1616513069.170875:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.170878:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513069.170885:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.170889:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.170890:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.170896:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701308370624 00010000:00000200:24.0:1616513069.170907:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688701308370624/t0(0) o400->b5d401c2-4724-a8ad-f2ff-62498a1a191c@10.31.142.138@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.170921:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701308370624, offset 192 00000400:00000200:24.0:1616513069.170927:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.138@tcp1 00000400:00000200:24.0:1616513069.170935:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.138@tcp1 routed destination 00000400:00000200:24.0:1616513069.170950:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.138@tcp1(10.31.142.138@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:2.0:1616513069.170951:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701308371264 00000800:00000200:24.0:1616513069.170957:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.170963:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:2.0:1616513069.170964:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0480 x1688701308371264/t0(0) o400->0917a45c-3403-e825-17bb-6c9614cfad7b@10.31.142.138@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513069.170966:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.170968:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.170973:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.170977:0:48299:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:2.0:1616513069.170978:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701308371264, offset 192 00000100:00000200:24.0:1616513069.170980:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513069.170984:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.138@tcp1 00000400:00000200:2.0:1616513069.170992:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.138@tcp1 routed destination 00000400:00000200:2.0:1616513069.171007:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.138@tcp1(10.31.142.138@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.171013:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.171019:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.171022:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.171024:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.171028:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513069.171031:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513069.171037:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513069.171038:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.171041:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513069.171042:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513069.171049:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.171055:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:14.0:1616513069.171058:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513069.171097:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.171105:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513069.171107:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513069.171114:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513069.171118:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec440 00000800:00000200:4.0:1616513069.171122:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513069.171125:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.217880:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513069.217939:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.217948:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513069.217957:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.74@tcp1 : PUT - for me 00000400:00000200:22.0:1616513069.217966:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.74@tcp1 of length 224 into portal 26 MB=0x604cfcc81d800 00000400:00000200:22.0:1616513069.217973:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.74@tcp1 of length 224/224 into md 0xf97c3869 [1] + 0 00000400:00000200:22.0:1616513069.217978:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23650 00000400:00000200:22.0:1616513069.217980:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.217983:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513069.217990:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23650 00000800:00000200:22.0:1616513069.217993:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.217994:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.217996:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513069.217998:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.218056:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694140396066816 00010000:00000200:24.0:1616513069.218067:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1694140396066816/t0(0) o400->15eeeff6-290f-74df-820e-81fdda7eb951@10.31.136.74@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.218081:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694140396066816, offset 192 00000400:00000200:24.0:1616513069.218087:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.74@tcp1 00000400:00000200:24.0:1616513069.218095:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.74@tcp1 routed destination 00000400:00000200:24.0:1616513069.218110:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.74@tcp1(10.31.136.74@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.218116:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.218122:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.218125:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.218127:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.218132:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.218136:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.218139:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513069.218147:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513069.218153:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513069.218156:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513069.218162:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513069.218168:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:10.0:1616513069.218171:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.218174:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.218179:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513069.218182:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513069.218186:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.74@tcp1 : PUT - for me 00000400:00000200:10.0:1616513069.218194:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.74@tcp1 of length 224 into portal 12 MB=0x604cfcc81d880 00000400:00000200:10.0:1616513069.218199:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.74@tcp1 of length 224/224 into md 0xf96450b9 [1] + 3136 00000400:00000200:10.0:1616513069.218204:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513069.218206:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513069.218213:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.218215:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.218217:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.218278:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694140396066944 00010000:00000200:2.0:1616513069.218290:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bde780 x1694140396066944/t0(0) o400->9f077256-8c9e-849e-e1d3-a1cb7e9f6a27@10.31.136.74@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513069.218305:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694140396066944, offset 192 00000400:00000200:2.0:1616513069.218311:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.74@tcp1 00000400:00000200:2.0:1616513069.218318:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.74@tcp1 routed destination 00000400:00000200:2.0:1616513069.218333:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.74@tcp1(10.31.136.74@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.218339:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.218345:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.218348:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.218350:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.218355:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.218362:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.218372:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.218377:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.218378:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.218382:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.218385:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed320 00000800:00000200:8.0:1616513069.218388:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.218390:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.229307:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513069.229362:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513069.229372:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513069.229381:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.128@tcp1 : PUT - for me 00000400:00000200:4.0:1616513069.229390:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.128@tcp1 of length 224 into portal 26 MB=0x601876540d5c0 00000400:00000200:4.0:1616513069.229397:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.128@tcp1 of length 224/224 into md 0xf97c3889 [1] + 0 00000400:00000200:4.0:1616513069.229402:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22aa0 00000400:00000200:4.0:1616513069.229405:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513069.229407:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513069.229413:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22aa0 00000800:00000200:4.0:1616513069.229416:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.229418:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513069.229421:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.229428:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513069.229431:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513069.229431:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513069.229451:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.128@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.229460:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.128@tcp1 of length 224 into portal 12 MB=0x601876540d640 00000400:00000200:20.0:1616513069.229466:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.128@tcp1 of length 224/224 into md 0x865b9f31 [1] + 28720 00000400:00000200:20.0:1616513069.229471:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.229474:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:24.0:1616513069.229476:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690530891224512 00000800:00000200:20.0:1616513069.229482:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.229486:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:24.0:1616513069.229487:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1690530891224512/t0(0) o400->973bdf1d-8e66-e493-f9ad-e54475465b60@10.31.132.128@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513069.229488:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.229501:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690530891224512, offset 192 00000400:00000200:24.0:1616513069.229509:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.128@tcp1 00000400:00000200:24.0:1616513069.229516:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.128@tcp1 routed destination 00000400:00000200:24.0:1616513069.229531:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.128@tcp1(10.31.132.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.229538:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.229544:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.229546:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.229548:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.229553:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513069.229556:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690530891224640 00000100:00000200:24.0:1616513069.229560:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.229560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:16.0:1616513069.229568:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cef080 x1690530891224640/t0(0) o400->d56c65fd-4ea0-3601-29b9-a1b4c892f80b@10.31.132.128@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.229583:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690530891224640, offset 192 00000400:00000200:16.0:1616513069.229590:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.128@tcp1 00000400:00000200:16.0:1616513069.229597:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.128@tcp1 routed destination 00000400:00000200:16.0:1616513069.229612:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.128@tcp1(10.31.132.128@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513069.229615:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513069.229618:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513069.229622:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.229624:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513069.229624:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.229626:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.229630:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513069.229631:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513069.229635:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513069.229636:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3d38 00000800:00000200:22.0:1616513069.229640:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513069.229640:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513069.229643:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513069.229696:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.229704:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.229706:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.229707:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.229713:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.229717:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ed0 00000800:00000200:12.0:1616513069.229721:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:11.0:1616513069.258585:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923392, portal 4 00000100:00000200:11.0:1616513069.258590:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923392, offset 0 00000100:00000200:30.0:1616513069.258593:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923408, portal 4 00000100:00000200:30.0:1616513069.258596:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923408, offset 0 00000400:00000200:11.0:1616513069.258596:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:30.0:1616513069.258599:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513069.258602:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:11.0:1616513069.258603:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:30.0:1616513069.258608:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513069.258611:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513069.258614:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513069.258615:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513069.258616:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:11.0:1616513069.258616:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513069.258618:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:11.0:1616513069.258622:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:5.2:1616513069.258627:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:11.0:1616513069.258629:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513069.258631:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:11.0:1616513069.258634:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:11.0:1616513069.258639:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:4.0:1616513069.258681:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:27.2:1616513069.258685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:4.0:1616513069.258688:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513069.258690:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513069.258697:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513069.258703:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d1680 x1693067725923408/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513069.258705:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513069.258710:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513069.258713:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513069.258715:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efcb0 00000800:00000200:31.0:1616513069.258716:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513069.258720:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:4.0:1616513069.258720:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513069.258723:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:27.0:1616513069.258727:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8006cdd580 x1693067725923392/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513069.258740:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a770 00000800:00000200:27.0:1616513069.258744:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513069.258853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513069.258863:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.258870:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513069.258871:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:5.2:1616513069.258871:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000400:00000200:27.0:1616513069.258878:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:8.0:1616513069.258880:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513069.258885:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:27.0:1616513069.258886:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626440 00000400:00000200:8.0:1616513069.258891:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513069.258893:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164c5d [1] + 192 00000400:00000200:8.0:1616513069.258896:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626450 00000400:00000200:27.0:1616513069.258898:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.258900:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d2531 [1] + 192 00000100:00000200:27.0:1616513069.258903:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8006cdd580 x1693067725923392/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513069.258904:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.258908:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d1680 x1693067725923408/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513069.258917:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:8.0:1616513069.258917:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000100:00000200:30.0:1616513069.258919:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d1680 x1693067725923408/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513069.258919:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:27.0:1616513069.258920:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:8.0:1616513069.258920:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513069.258921:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.0:1616513069.258922:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:11.0:1616513069.258922:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8006cdd580 x1693067725923392/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513069.258924:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d1680 x1693067725923408/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513069.258927:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee660 00000100:00000200:11.0:1616513069.258934:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8006cdd580 x1693067725923392/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513069.258942:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3aaa0 00000800:00000200:11.2:1616513069.262011:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513069.262070:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513069.262079:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513069.262088:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.115@tcp1 : PUT - for me 00000400:00000200:4.0:1616513069.262098:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.115@tcp1 of length 224 into portal 26 MB=0x5ffe02374da80 00000400:00000200:4.0:1616513069.262106:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.115@tcp1 of length 224/224 into md 0xf97c3899 [1] + 0 00000400:00000200:4.0:1616513069.262111:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee226e8 00000400:00000200:4.0:1616513069.262114:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513069.262117:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513069.262121:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee226e8 00000800:00000200:20.0:1616513069.262124:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.262124:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.262127:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513069.262131:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513069.262134:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.262135:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513069.262143:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.115@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.262152:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.115@tcp1 of length 224 into portal 12 MB=0x5ffe02374dd00 00000400:00000200:20.0:1616513069.262159:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.115@tcp1 of length 224/224 into md 0x865b9f31 [1] + 28944 00000400:00000200:20.0:1616513069.262164:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.262166:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513069.262173:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.262177:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.262178:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.262182:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688713016171136 00010000:00000200:24.0:1616513069.262192:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688713016171136/t0(0) o400->bcbafb4a-56d9-1e89-704f-d328a06aa208@10.31.141.115@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.262205:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688713016171136, offset 192 00000400:00000200:24.0:1616513069.262211:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.115@tcp1 00000400:00000200:24.0:1616513069.262218:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.115@tcp1 routed destination 00000400:00000200:24.0:1616513069.262232:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.115@tcp1(10.31.141.115@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.262238:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:16.0:1616513069.262238:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688713016171776 00000800:00000200:24.0:1616513069.262244:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.262246:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.262248:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:16.0:1616513069.262250:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8900 x1688713016171776/t0(0) o400->abf2b060-4bc7-c244-5e23-8496ec3fcacd@10.31.141.115@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513069.262253:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.262259:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.262259:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513069.262265:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688713016171776, offset 192 00000400:00000200:16.0:1616513069.262271:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.115@tcp1 00000400:00000200:16.0:1616513069.262279:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.115@tcp1 routed destination 00000400:00000200:16.0:1616513069.262294:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.115@tcp1(10.31.141.115@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.262301:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.262306:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.262309:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.262311:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513069.262314:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513069.262316:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.262321:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513069.262322:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.262325:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513069.262327:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513069.262331:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.262337:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2198 00000800:00000200:14.0:1616513069.262340:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513069.262376:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.262383:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.262385:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.262386:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.262392:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.262396:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43760 00000800:00000200:12.0:1616513069.262400:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:31.2F:1616513069.270491:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:22.0:1616513069.270550:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:22.0:1616513069.270560:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:22.0:1616513069.270569:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513069.270577:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 224 into portal 24 MB=0x603db0154b8c0 00000400:00000200:22.0:1616513069.270583:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 18 from 12345-10.242.46.234@o2ib1 of length 224/224 into md 0xf97316b9 [1] + 0 00000400:00000200:22.0:1616513069.270588:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d4553e83540 00000400:00000200:22.0:1616513069.270591:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.270594:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt_out 00000400:00000200:22.0:1616513069.270599:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83540 00000800:00000200:22.0:1616513069.270602:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:22.0:1616513069.270605:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:18.0:1616513069.270606:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:22.0:1616513069.270610:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:6.0:1616513069.270656:0:185125:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693089015314624 00010000:00000200:6.0:1616513069.270674:0:185125:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d741ca81850 x1693089015314624/t0(0) o41->boslfs02-MDT0000-mdtlov_UUID@10.242.46.234@o2ib1:695/0 lens 224/368 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:6.0:1616513069.270684:0:185125:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 4, xid 1693089015314624, offset 192 00000400:00000200:6.0:1616513069.270689:0:185125:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:6.0:1616513069.270695:0:185125:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:6.0:1616513069.270705:0:185125:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513069.270710:0:185125:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:6.0:1616513069.270714:0:185125:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513069.270716:0:185125:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:6.0:1616513069.270718:0:185125:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:6.0:1616513069.270721:0:185125:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513069.270727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:6.0:1616513069.270727:0:185125:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000800:00000200:10.0:1616513069.270778:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:10.0:1616513069.270785:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:10.0:1616513069.270789:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513069.270795:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:10.0:1616513069.270800:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf002a8 00000800:00000200:10.0:1616513069.270803:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:10.0:1616513069.270806:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:11.2:1616513069.273470:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513069.273525:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513069.273535:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513069.273544:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.125@tcp1 : PUT - for me 00000400:00000200:4.0:1616513069.273553:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.125@tcp1 of length 224 into portal 26 MB=0x604a356864540 00000400:00000200:4.0:1616513069.273560:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.125@tcp1 of length 224/224 into md 0xf97c38b9 [1] + 0 00000400:00000200:4.0:1616513069.273565:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22330 00000400:00000200:4.0:1616513069.273568:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513069.273571:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513069.273575:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22330 00000800:00000200:4.0:1616513069.273578:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.273579:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513069.273581:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513069.273584:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.273592:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.273600:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.273605:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.273610:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.125@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.273615:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.125@tcp1 of length 224 into portal 12 MB=0x604a356864680 00000400:00000200:8.0:1616513069.273618:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.125@tcp1 of length 224/224 into md 0x865b9f31 [1] + 29168 00000400:00000200:8.0:1616513069.273621:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.273623:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.273628:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.273629:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.273630:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.273632:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.273645:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949438084416 00010000:00000200:24.0:1616513069.273656:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1693949438084416/t0(0) o400->1edd3867-34fd-b7c7-ed30-a263f36debf8@10.31.142.125@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.273670:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949438084416, offset 192 00000400:00000200:24.0:1616513069.273676:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.125@tcp1 00000400:00000200:24.0:1616513069.273684:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.125@tcp1 routed destination 00000400:00000200:24.0:1616513069.273699:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.125@tcp1(10.31.142.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.273710:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:16.0:1616513069.273715:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949438084736 00000800:00000200:24.0:1616513069.273716:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.273719:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.273721:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.273726:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:16.0:1616513069.273728:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b99680 x1693949438084736/t0(0) o400->fb7fc460-fbae-6428-cd17-8cbf7d38a4e0@10.31.142.125@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513069.273732:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.273733:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:16.0:1616513069.273742:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949438084736, offset 192 00000400:00000200:16.0:1616513069.273749:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.125@tcp1 00000400:00000200:16.0:1616513069.273757:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.125@tcp1 routed destination 00000400:00000200:16.0:1616513069.273772:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.125@tcp1(10.31.142.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.273779:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.273785:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513069.273788:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513069.273788:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.273790:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.273794:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513069.273795:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513069.273798:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513069.273799:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513069.273805:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.273810:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3540 00000800:00000200:20.0:1616513069.273811:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513069.273814:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513069.273817:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513069.273817:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513069.273819:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513069.273821:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513069.273823:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513069.273825:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513069.273826:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:10.0:1616513069.273826:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.273827:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513069.273830:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42198 00000800:00000200:18.0:1616513069.273830:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513069.273834:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:3.0:1616513069.306604:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923424, portal 4 00000100:00000200:3.0:1616513069.306610:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923424, offset 0 00000400:00000200:3.0:1616513069.306617:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:3.0:1616513069.306624:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:3.0:1616513069.306638:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513069.306645:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:3.0:1616513069.306653:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513069.306656:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:3.0:1616513069.306658:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:3.0:1616513069.306664:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513069.306711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513069.306768:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513069.306776:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513069.306779:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513069.306781:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513069.306786:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513069.306793:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8006cdde80 x1693067725923424/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513069.306805:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2550 00000800:00000200:14.0:1616513069.306809:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513069.306841:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513069.306896:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513069.306906:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513069.306914:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513069.306923:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626460 00000400:00000200:18.0:1616513069.306930:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c3f29 [1] + 192 00000400:00000200:18.0:1616513069.306936:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.306943:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8006cdde80 x1693067725923424/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513069.306951:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513069.306958:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513069.306961:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513069.306963:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:3.0:1616513069.306969:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8006cdde80 x1693067725923424/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513069.306982:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8006cdde80 x1693067725923424/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513069.306992:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2000 00000800:00000200:11.2:1616513069.309452:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.309510:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.309515:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.309519:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.121@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.309525:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.121@tcp1 of length 224 into portal 26 MB=0x603fe746c7400 00000400:00000200:8.0:1616513069.309528:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.121@tcp1 of length 224/224 into md 0xf97c38c9 [1] + 0 00000400:00000200:8.0:1616513069.309531:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fa90 00000400:00000200:8.0:1616513069.309533:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.309535:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513069.309544:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa90 00000800:00000200:8.0:1616513069.309546:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.309548:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.309549:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.309550:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.309554:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.309557:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.121@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.309560:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.121@tcp1 of length 224 into portal 12 MB=0x603fe746c7480 00000400:00000200:8.0:1616513069.309562:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.121@tcp1 of length 224/224 into md 0x865b9f31 [1] + 29392 00000800:00000200:12.0:1616513069.309563:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513069.309564:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.309565:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.309569:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.309569:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.309570:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.309607:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693241270105088 00010000:00000200:24.0:1616513069.309617:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1693241270105088/t0(0) o400->c5895b71-66ff-1a70-6a76-1557a9599452@10.31.138.121@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.309631:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693241270105088, offset 192 00000400:00000200:24.0:1616513069.309637:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.121@tcp1 00000100:00000200:16.0:1616513069.309642:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693241270105216 00000400:00000200:24.0:1616513069.309644:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.121@tcp1 routed destination 00010000:00000200:16.0:1616513069.309654:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b98d80 x1693241270105216/t0(0) o400->b9e155e4-1700-8a83-37c9-bd4b66c85e32@10.31.138.121@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513069.309658:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.121@tcp1(10.31.138.121@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.309664:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:16.0:1616513069.309667:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693241270105216, offset 192 00000800:00000200:24.0:1616513069.309670:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.309673:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.309675:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513069.309675:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.121@tcp1 00000800:00000200:24.0:1616513069.309679:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513069.309682:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.121@tcp1 routed destination 00000100:00000200:24.0:1616513069.309685:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.309685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513069.309696:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.121@tcp1(10.31.138.121@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.309702:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.309707:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.309710:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.309712:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.309717:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.309722:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513069.309734:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513069.309735:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513069.309740:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513069.309742:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:10.0:1616513069.309742:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513069.309744:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513069.309745:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513069.309748:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513069.309751:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:10.0:1616513069.309752:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000400:00000200:20.0:1616513069.309755:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43f58 00000800:00000200:10.0:1616513069.309755:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.309758:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513069.309759:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.313069:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513069.313127:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.313137:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513069.313146:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.114@tcp1 : PUT - for me 00000400:00000200:22.0:1616513069.313155:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.114@tcp1 of length 224 into portal 26 MB=0x60194fe571140 00000400:00000200:22.0:1616513069.313162:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.114@tcp1 of length 224/224 into md 0xf97c38d9 [1] + 0 00000400:00000200:22.0:1616513069.313166:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777eff0 00000400:00000200:22.0:1616513069.313169:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.313172:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513069.313175:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eff0 00000800:00000200:22.0:1616513069.313180:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.313181:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.313183:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513069.313185:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.313239:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589294170432 00010000:00000200:24.0:1616513069.313250:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1690589294170432/t0(0) o400->f160f0b9-ee0e-5912-9241-293fef8c108e@10.31.140.114@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.313264:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690589294170432, offset 192 00000400:00000200:24.0:1616513069.313270:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.114@tcp1 00000400:00000200:24.0:1616513069.313278:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.114@tcp1 routed destination 00000400:00000200:24.0:1616513069.313293:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.114@tcp1(10.31.140.114@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.313299:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.313305:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.313308:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.313312:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.313317:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.313322:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.313323:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513069.313333:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513069.313337:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513069.313339:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513069.313344:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513069.313347:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ba0 00000800:00000200:10.0:1616513069.313350:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.313352:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.313853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513069.313864:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.313872:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513069.313874:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.313880:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.114@tcp1 : PUT - for me 00000400:00000200:18.0:1616513069.313889:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.114@tcp1 of length 224 into portal 12 MB=0x60194fe571480 00000400:00000200:18.0:1616513069.313895:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.114@tcp1 of length 224/224 into md 0xf96450b9 [1] + 3360 00000400:00000200:18.0:1616513069.313900:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.313903:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513069.313911:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.313914:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.313916:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.313978:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589294171264 00010000:00000200:2.0:1616513069.313990:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d2d00 x1690589294171264/t0(0) o400->b1a9aa1b-ed12-5556-d924-d7b316910745@10.31.140.114@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513069.314005:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690589294171264, offset 192 00000400:00000200:2.0:1616513069.314011:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.114@tcp1 00000400:00000200:2.0:1616513069.314018:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.114@tcp1 routed destination 00000400:00000200:2.0:1616513069.314033:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.114@tcp1(10.31.140.114@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.314040:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.314045:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.314048:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.314050:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.314055:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.314101:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.314112:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.314115:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.314116:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.314120:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.314122:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eeded0 00000800:00000200:8.0:1616513069.314124:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.314126:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.324288:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.324359:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.324369:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.324378:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.130@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.324387:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.130@tcp1 of length 224 into portal 26 MB=0x603fc0c4b2040 00000400:00000200:12.0:1616513069.324393:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.130@tcp1 of length 224/224 into md 0xf97c38e9 [1] + 0 00000400:00000200:12.0:1616513069.324398:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e990 00000400:00000200:12.0:1616513069.324400:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.324403:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513069.324408:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e990 00000800:00000200:12.0:1616513069.324411:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.324414:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.324416:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.324419:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.324429:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.324430:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513069.324450:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.130@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.324459:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.130@tcp1 of length 224 into portal 12 MB=0x603fc0c4b2100 00000400:00000200:20.0:1616513069.324465:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.130@tcp1 of length 224/224 into md 0x865b9f31 [1] + 29616 00000100:00000200:24.0:1616513069.324471:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230933155904 00000400:00000200:20.0:1616513069.324471:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.324473:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513069.324480:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513069.324482:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1693230933155904/t0(0) o400->cf55d745-c94a-f0f1-05e8-2ced41e5dc0e@10.31.138.130@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513069.324483:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.324485:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.324496:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230933155904, offset 192 00000400:00000200:24.0:1616513069.324502:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.130@tcp1 00000100:00000200:16.0:1616513069.324505:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230933156096 00000400:00000200:24.0:1616513069.324509:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.130@tcp1 routed destination 00010000:00000200:16.0:1616513069.324518:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce9b00 x1693230933156096/t0(0) o400->363525a1-1723-cb3b-51b3-d061e2b0cf72@10.31.138.130@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513069.324525:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.130@tcp1(10.31.138.130@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.324531:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:16.0:1616513069.324532:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230933156096, offset 192 00000800:00000200:24.0:1616513069.324537:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513069.324538:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.130@tcp1 00000800:00000200:24.0:1616513069.324540:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.324542:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513069.324546:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.130@tcp1 routed destination 00000800:00000200:24.0:1616513069.324547:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.324553:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.324554:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513069.324561:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.130@tcp1(10.31.138.130@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513069.324563:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513069.324567:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513069.324568:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513069.324570:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513069.324573:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:10.0:1616513069.324574:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513069.324576:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.324578:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513069.324578:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf24c8 00000800:00000200:10.0:1616513069.324581:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513069.324583:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513069.324583:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.324588:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513069.324668:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513069.324675:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513069.324678:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513069.324681:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513069.324685:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513069.324689:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a08 00000800:00000200:4.0:1616513069.324693:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.333598:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.333608:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.333612:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.333616:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.11@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.333621:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.11@tcp1 of length 224 into portal 26 MB=0x604a29bb42e40 00000400:00000200:8.0:1616513069.333624:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.11@tcp1 of length 224/224 into md 0xf97c38f9 [1] + 0 00000400:00000200:8.0:1616513069.333626:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f078 00000400:00000200:8.0:1616513069.333627:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.333629:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513069.333631:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f078 00000800:00000200:8.0:1616513069.333632:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.333634:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.333635:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.333636:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.333646:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513069.333708:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946303753792 00010000:00000200:24.0:1616513069.333718:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1693946303753792/t0(0) o400->ba3c2b42-5af9-d366-e087-4ac08cba0a35@10.31.141.11@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513069.333721:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.333731:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946303753792, offset 192 00000800:00000200:22.0:1616513069.333731:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513069.333737:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.11@tcp1 00000400:00000200:22.0:1616513069.333740:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.11@tcp1 : PUT - for me 00000400:00000200:24.0:1616513069.333744:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.11@tcp1 routed destination 00000400:00000200:22.0:1616513069.333748:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.11@tcp1 of length 224 into portal 12 MB=0x604a29bb431c0 00000400:00000200:22.0:1616513069.333755:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.11@tcp1 of length 224/224 into md 0xf96450b9 [1] + 3584 00000400:00000200:24.0:1616513069.333758:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.11@tcp1(10.31.141.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513069.333760:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.333762:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513069.333764:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513069.333769:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513069.333770:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.333772:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513069.333772:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513069.333774:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513069.333774:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513069.333777:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513069.333779:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.333785:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.333786:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.333793:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513069.333795:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513069.333797:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513069.333800:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513069.333803:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2a18 00000800:00000200:10.0:1616513069.333804:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.333806:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.333833:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946303754688 00010000:00000200:2.0:1616513069.333845:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5e80 x1693946303754688/t0(0) o400->383d11bf-f0e1-5d5f-3c70-a7592349db9e@10.31.141.11@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513069.333860:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946303754688, offset 192 00000400:00000200:2.0:1616513069.333866:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.11@tcp1 00000400:00000200:2.0:1616513069.333873:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.11@tcp1 routed destination 00000400:00000200:2.0:1616513069.333888:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.11@tcp1(10.31.141.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.333895:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.333901:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.333904:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.333906:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.333911:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.333915:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.333972:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.333979:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.333983:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.333990:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.333994:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec088 00000800:00000200:12.0:1616513069.333998:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.334001:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.343344:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513069.343398:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.343408:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513069.343417:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.189@tcp1 : PUT - for me 00000800:00000200:11.2:1616513069.343422:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513069.343427:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.189@tcp1 of length 224 into portal 26 MB=0x5ffd3357c9240 00000800:00000200:8.0:1616513069.343431:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.343439:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.343443:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.189@tcp1 : PUT - for me 00000400:00000200:18.0:1616513069.343444:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.189@tcp1 of length 224/224 into md 0xf97c3919 [1] + 0 00000400:00000200:8.0:1616513069.343448:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.189@tcp1 of length 224 into portal 12 MB=0x5ffd3357c9340 00000400:00000200:18.0:1616513069.343450:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e440 00000400:00000200:8.0:1616513069.343451:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.189@tcp1 of length 224/224 into md 0x865b9f31 [1] + 29840 00000800:00000200:22.0:1616513069.343452:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.343453:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.343453:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.343455:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:8.0:1616513069.343455:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.343458:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.343459:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513069.343460:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e440 00000800:00000200:8.0:1616513069.343460:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.343461:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513069.343463:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.343465:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.343467:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.343535:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657484091968 00000100:00000200:16.0:1616513069.343539:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657484092224 00010000:00000200:24.0:1616513069.343545:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688657484091968/t0(0) o400->7042e8b0-a90b-4f48-eccc-496868585275@10.31.138.189@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:16.0:1616513069.343552:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9f500 x1688657484092224/t0(0) o400->a3caea7d-a993-5956-cd5b-c7f168204b74@10.31.138.189@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513069.343560:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657484091968, offset 192 00000400:00000200:24.0:1616513069.343565:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.189@tcp1 00000100:00000200:16.0:1616513069.343568:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657484092224, offset 192 00000400:00000200:24.0:1616513069.343572:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.189@tcp1 routed destination 00000400:00000200:16.0:1616513069.343575:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.189@tcp1 00000400:00000200:16.0:1616513069.343583:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.189@tcp1 routed destination 00000400:00000200:24.0:1616513069.343586:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.189@tcp1(10.31.138.189@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.343593:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513069.343597:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.189@tcp1(10.31.138.189@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.343604:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.343604:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.343607:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.343609:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.343610:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.343613:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513069.343614:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.343616:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513069.343618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.343620:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513069.343621:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513069.343624:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513069.343626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513069.343627:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513069.343628:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513069.343632:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:14.0:1616513069.343633:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513069.343634:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2aa0 00000800:00000200:10.0:1616513069.343637:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513069.343680:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.343688:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.343690:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.343691:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.343697:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.343701:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43c28 00000800:00000200:12.0:1616513069.343705:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.352382:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513069.352448:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.352457:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513069.352466:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.82@tcp1 : PUT - for me 00000400:00000200:22.0:1616513069.352475:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.82@tcp1 of length 224 into portal 26 MB=0x60183d99dd040 00000400:00000200:22.0:1616513069.352482:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.82@tcp1 of length 224/224 into md 0xf97c3939 [1] + 0 00000400:00000200:22.0:1616513069.352487:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fcb0 00000400:00000200:22.0:1616513069.352490:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.352492:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513069.352497:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fcb0 00000800:00000200:22.0:1616513069.352500:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.352502:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.352503:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513069.352506:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.352556:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515663605824 00010000:00000200:24.0:1616513069.352565:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1690515663605824/t0(0) o400->c95b3461-b1be-3260-dce6-379b6893dc96@10.31.142.82@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.352577:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690515663605824, offset 192 00000400:00000200:24.0:1616513069.352582:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.82@tcp1 00000400:00000200:24.0:1616513069.352589:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.82@tcp1 routed destination 00000400:00000200:24.0:1616513069.352602:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.82@tcp1(10.31.142.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.352608:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.352613:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.352615:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.352617:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.352621:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.352626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.352627:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513069.352696:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.352702:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.352704:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513069.352704:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513069.352710:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.352715:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ed0 00000800:00000200:14.0:1616513069.352718:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.352791:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.352800:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.352804:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.352808:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.82@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.352812:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.82@tcp1 of length 224 into portal 12 MB=0x60183d99dd400 00000400:00000200:8.0:1616513069.352814:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.82@tcp1 of length 224/224 into md 0x865b9f31 [1] + 30064 00000400:00000200:8.0:1616513069.352816:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.352817:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.352820:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.352822:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.352822:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.352824:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.352887:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515663606784 00010000:00000200:16.0:1616513069.352900:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9e300 x1690515663606784/t0(0) o400->da06d8bc-83bb-9f4d-b8b7-ee315e4b8855@10.31.142.82@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.352914:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690515663606784, offset 192 00000400:00000200:16.0:1616513069.352920:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.82@tcp1 00000400:00000200:16.0:1616513069.352928:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.82@tcp1 routed destination 00000400:00000200:16.0:1616513069.352943:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.82@tcp1(10.31.142.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.352949:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.352955:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.352957:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.352960:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.352964:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.352969:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.352975:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.352977:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.352978:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.352980:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.352982:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ba0 00000800:00000200:8.0:1616513069.352983:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.352984:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513069.354689:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923440, portal 4 00000100:00000200:15.0:1616513069.354696:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923440, offset 0 00000400:00000200:15.0:1616513069.354703:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:15.0:1616513069.354710:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:15.0:1616513069.354723:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513069.354730:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:15.0:1616513069.354737:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513069.354741:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:15.0:1616513069.354743:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:15.0:1616513069.354749:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513069.354756:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513069.354763:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513069.354766:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513069.354767:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.354769:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513069.354773:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded6c00 x1693067725923440/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513069.354782:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77742a8 00000800:00000200:8.0:1616513069.354784:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513069.354785:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513069.354926:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513069.354932:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513069.354934:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513069.354937:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513069.354939:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626470 00000400:00000200:8.0:1616513069.354942:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d2621 [1] + 192 00000400:00000200:8.0:1616513069.354944:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.354946:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded6c00 x1693067725923440/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513069.354952:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513069.354953:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:8.0:1616513069.354954:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513069.354955:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:15.0:1616513069.354964:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded6c00 x1693067725923440/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513069.354977:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded6c00 x1693067725923440/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513069.354988:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774c38 00000800:00000200:11.2:1616513069.369984:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.369995:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.369998:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.370001:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.220@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.370005:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.220@tcp1 of length 224 into portal 26 MB=0x5ffd2f49d4100 00000400:00000200:8.0:1616513069.370008:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.220@tcp1 of length 224/224 into md 0xf97c3949 [1] + 0 00000400:00000200:8.0:1616513069.370010:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ec38 00000400:00000200:8.0:1616513069.370011:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.370012:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513069.370014:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ec38 00000800:00000200:8.0:1616513069.370015:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.370016:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.370017:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.370018:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.370082:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656395714816 00010000:00000200:24.0:1616513069.370092:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688656395714816/t0(0) o400->62b74559-906e-9f9a-aa14-6c294590b2ba@10.31.128.220@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.370104:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688656395714816, offset 192 00000400:00000200:24.0:1616513069.370110:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.220@tcp1 00000400:00000200:24.0:1616513069.370117:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.220@tcp1 routed destination 00000400:00000200:24.0:1616513069.370132:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.220@tcp1(10.31.128.220@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.370137:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.370143:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.370145:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.370147:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.370151:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.370157:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.370159:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.370179:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.370186:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513069.370188:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513069.370195:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.370202:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000800:00000200:18.0:1616513069.370205:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.370209:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.370214:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513069.370222:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.220@tcp1 : PUT - for me 00000400:00000200:18.0:1616513069.370230:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.220@tcp1 of length 224 into portal 12 MB=0x5ffd2f49d43c0 00000800:00000200:22.0:1616513069.370233:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.370237:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.220@tcp1 of length 224/224 into md 0xf96450b9 [1] + 3808 00000400:00000200:18.0:1616513069.370242:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.370244:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513069.370251:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.370254:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.370256:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.370316:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656395715520 00010000:00000200:2.0:1616513069.370327:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d3600 x1688656395715520/t0(0) o400->51b8770e-2309-9b21-1c47-31d6357ac38d@10.31.128.220@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513069.370340:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688656395715520, offset 192 00000400:00000200:2.0:1616513069.370346:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.220@tcp1 00000400:00000200:2.0:1616513069.370353:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.220@tcp1 routed destination 00000400:00000200:2.0:1616513069.370368:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.220@tcp1(10.31.128.220@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.370375:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.370380:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.370383:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.370385:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.370389:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.370395:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.370402:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.370404:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.370405:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.370408:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.370410:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eedd38 00000800:00000200:8.0:1616513069.370411:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.370413:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.379629:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.379638:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.379640:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.379643:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.21@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.379647:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.21@tcp1 of length 224 into portal 26 MB=0x601975e1fab80 00000400:00000200:8.0:1616513069.379650:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.21@tcp1 of length 224/224 into md 0xf97c3969 [1] + 0 00000400:00000200:8.0:1616513069.379651:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f430 00000400:00000200:8.0:1616513069.379653:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.379654:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513069.379656:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f430 00000800:00000200:8.0:1616513069.379657:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.379658:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.379659:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.379659:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.379662:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.379665:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.21@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.379667:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.21@tcp1 of length 224 into portal 12 MB=0x601975e1fac00 00000400:00000200:8.0:1616513069.379670:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.21@tcp1 of length 224/224 into md 0x865b9f31 [1] + 30288 00000400:00000200:8.0:1616513069.379671:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.379672:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.379675:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.379676:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.379676:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.379678:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.379736:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690599491087232 00010000:00000200:24.0:1616513069.379747:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1690599491087232/t0(0) o400->6934a1d5-2804-4b86-4fd5-00d046d241ca@10.31.132.21@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.379758:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690599491087360 00000100:00000200:24.0:1616513069.379761:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690599491087232, offset 192 00000400:00000200:24.0:1616513069.379767:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.21@tcp1 00010000:00000200:16.0:1616513069.379771:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9f980 x1690599491087360/t0(0) o400->28917d97-0a2c-521d-6f69-25aaa470ddd1@10.31.132.21@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513069.379774:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.21@tcp1 routed destination 00000100:00000200:16.0:1616513069.379785:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690599491087360, offset 192 00000400:00000200:24.0:1616513069.379789:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.21@tcp1(10.31.132.21@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513069.379791:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.21@tcp1 00000800:00000200:24.0:1616513069.379796:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513069.379798:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.21@tcp1 routed destination 00000400:00000200:16.0:1616513069.379813:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.21@tcp1(10.31.132.21@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.379820:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.379820:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.379823:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.379825:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.379825:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.379828:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.379830:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513069.379830:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.379835:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513069.379835:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.379837:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.379840:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.379852:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.379859:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.379859:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.379861:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.379868:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.379872:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ee0 00000800:00000200:12.0:1616513069.379876:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513069.379887:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.379893:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.379895:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.379901:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.379906:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:14.0:1616513069.379909:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.379912:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.392120:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513069.392132:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.392140:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513069.392148:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.223@tcp1 : PUT - for me 00000400:00000200:10.0:1616513069.392157:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.223@tcp1 of length 224 into portal 26 MB=0x6048d68e38300 00000400:00000200:10.0:1616513069.392163:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.223@tcp1 of length 224/224 into md 0xf97c3989 [1] + 0 00000400:00000200:10.0:1616513069.392167:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fc28 00000400:00000200:10.0:1616513069.392170:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513069.392172:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513069.392176:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fc28 00000800:00000200:10.0:1616513069.392178:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513069.392181:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513069.392182:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.392185:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.392239:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855256904448 00010000:00000200:24.0:1616513069.392250:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1693855256904448/t0(0) o400->7baf055e-a2d1-e8dc-fc44-3f5aa966d7ed@10.31.140.223@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.392264:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855256904448, offset 192 00000400:00000200:24.0:1616513069.392270:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.223@tcp1 00000400:00000200:24.0:1616513069.392277:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.223@tcp1 routed destination 00000400:00000200:24.0:1616513069.392293:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.223@tcp1(10.31.140.223@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.392299:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.392305:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.392310:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.392312:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.392316:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.392322:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.392323:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.392351:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513069.392375:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.392382:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.392385:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.392392:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.392399:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2908 00000800:00000200:14.0:1616513069.392403:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.392405:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513069.392406:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.392416:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513069.392425:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.223@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.392457:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.223@tcp1 of length 224 into portal 12 MB=0x6048d68e38500 00000400:00000200:20.0:1616513069.392464:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.223@tcp1 of length 224/224 into md 0x865b9f31 [1] + 30512 00000400:00000200:20.0:1616513069.392469:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513069.392470:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513069.392472:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513069.392479:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.392482:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.392484:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.392544:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855256904960 00010000:00000200:16.0:1616513069.392556:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8480 x1693855256904960/t0(0) o400->18dc9e86-a745-4d02-5806-f1c7728f1c21@10.31.140.223@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.392581:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855256904960, offset 192 00000400:00000200:16.0:1616513069.392587:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.223@tcp1 00000400:00000200:16.0:1616513069.392595:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.223@tcp1 routed destination 00000400:00000200:16.0:1616513069.392610:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.223@tcp1(10.31.140.223@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.392616:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.392622:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.392625:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.392627:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.392631:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.392637:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.392644:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.392647:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.392648:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.392651:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:12.0:1616513069.392652:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513069.392653:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a90 00000800:00000200:8.0:1616513069.392656:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.393289:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513069.393302:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.393310:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513069.393313:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513069.393318:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.178@tcp1 : PUT - for me 00000400:00000200:10.0:1616513069.393326:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.178@tcp1 of length 224 into portal 26 MB=0x604a366be2dc0 00000400:00000200:10.0:1616513069.393332:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.178@tcp1 of length 224/224 into md 0xf97c39a9 [1] + 0 00000400:00000200:10.0:1616513069.393336:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e7f8 00000400:00000200:10.0:1616513069.393338:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513069.393340:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513069.393344:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e7f8 00000800:00000200:10.0:1616513069.393347:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.393349:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.393351:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.393356:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949710183872 00010000:00000200:24.0:1616513069.393361:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1693949710183872/t0(0) o400->ee7ca7d7-512f-c07b-e14b-fff5afb1adb5@10.31.140.178@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.393372:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949710183872, offset 192 00000400:00000200:24.0:1616513069.393376:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.178@tcp1 00000400:00000200:24.0:1616513069.393382:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.178@tcp1 routed destination 00000400:00000200:24.0:1616513069.393394:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.178@tcp1(10.31.140.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.393400:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.393404:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.393406:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.393408:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.393411:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.393415:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.393416:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.393424:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.393428:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.393430:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.393435:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513069.393441:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513069.393446:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf37e8 00000800:00000200:14.0:1616513069.393449:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.393763:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513069.393775:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.393784:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513069.393785:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.393793:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.178@tcp1 : PUT - for me 00000400:00000200:18.0:1616513069.393802:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.178@tcp1 of length 224 into portal 12 MB=0x604a366be31c0 00000400:00000200:18.0:1616513069.393808:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.178@tcp1 of length 224/224 into md 0xf96450b9 [1] + 4032 00000400:00000200:18.0:1616513069.393813:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.393816:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513069.393822:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.393825:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.393827:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.393885:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949710184896 00010000:00000200:2.0:1616513069.393898:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d7980 x1693949710184896/t0(0) o400->47b11ab0-c40d-2bfa-1ae0-75e72861e043@10.31.140.178@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513069.393912:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949710184896, offset 192 00000400:00000200:2.0:1616513069.393918:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.178@tcp1 00000400:00000200:2.0:1616513069.393926:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.178@tcp1 routed destination 00000400:00000200:2.0:1616513069.393940:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.178@tcp1(10.31.140.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.393947:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.393953:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.393955:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.393957:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.393962:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.393967:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.394019:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.394027:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.394029:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.394035:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.394039:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec4c8 00000800:00000200:12.0:1616513069.394043:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.394046:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:6.0:1616513069.418594:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923456, portal 4 00000100:00000200:6.0:1616513069.418599:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923456, offset 0 00000400:00000200:6.0:1616513069.418604:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513069.418609:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:6.0:1616513069.418620:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513069.418625:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:6.0:1616513069.418630:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513069.418632:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:6.0:1616513069.418634:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:6.0:1616513069.418639:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513069.418688:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513069.418703:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513069.418710:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513069.418712:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.418719:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513069.418727:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cd9488d80 x1693067725923456/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513069.418740:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01a90 00000800:00000200:22.0:1616513069.418744:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513069.418747:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513069.418858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513069.418870:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513069.418879:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513069.418882:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513069.418888:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513069.418896:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626480 00000400:00000200:14.0:1616513069.418903:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c4019 [1] + 192 00000400:00000200:14.0:1616513069.418908:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.418913:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cd9488d80 x1693067725923456/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513069.418928:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513069.418931:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:6.0:1616513069.418932:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cd9488d80 x1693067725923456/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513069.418933:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:6.0:1616513069.418942:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cd9488d80 x1693067725923456/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513069.418950:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01078 00000800:00000200:7.2:1616513069.448265:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513069.448320:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.448338:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513069.448346:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.4@tcp1 : PUT - for me 00000400:00000200:22.0:1616513069.448356:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.4@tcp1 of length 224 into portal 12 MB=0x60182bb1fd680 00000400:00000200:22.0:1616513069.448362:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.4@tcp1 of length 224/224 into md 0xf96450b9 [1] + 4256 00000400:00000200:22.0:1616513069.448368:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.448370:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513069.448377:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.448380:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.448382:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513069.448386:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.448456:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510857066112 00010000:00000200:2.0:1616513069.448467:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0900 x1690510857066112/t0(0) o400->f88d852a-9147-af69-f3bd-78d4c4342cad@10.31.141.4@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513069.448469:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:2.0:1616513069.448481:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510857066112, offset 192 00000400:00000200:2.0:1616513069.448486:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.4@tcp1 00000400:00000200:2.0:1616513069.448493:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.4@tcp1 routed destination 00000400:00000200:2.0:1616513069.448507:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.4@tcp1(10.31.141.4@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.448513:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513069.448518:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.448521:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513069.448525:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513069.448529:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513069.448535:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513069.448535:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513069.448548:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513069.448549:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513069.448557:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.4@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.448566:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.4@tcp1 of length 224 into portal 26 MB=0x60182bb1fd540 00000400:00000200:12.0:1616513069.448573:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.4@tcp1 of length 224/224 into md 0xf97c39b9 [1] + 0 00000400:00000200:12.0:1616513069.448578:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f188 00000400:00000200:12.0:1616513069.448581:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.448583:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513069.448590:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f188 00000800:00000200:18.0:1616513069.448591:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513069.448593:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.448596:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513069.448597:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513069.448598:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513069.448599:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513069.448600:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.448605:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.448608:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec6e8 00000800:00000200:18.0:1616513069.448612:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.448654:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510857065792 00010000:00000200:24.0:1616513069.448665:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1690510857065792/t0(0) o400->ce9a659a-f22c-b04c-9b49-cf197bf14e74@10.31.141.4@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.448679:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510857065792, offset 192 00000400:00000200:24.0:1616513069.448685:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.4@tcp1 00000400:00000200:24.0:1616513069.448692:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.4@tcp1 routed destination 00000400:00000200:24.0:1616513069.448707:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.4@tcp1(10.31.141.4@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.448714:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.448720:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.448722:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.448725:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.448729:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.448735:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.448736:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513069.448745:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.448749:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.448751:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.448757:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513069.448758:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513069.448761:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2e58 00000800:00000200:12.0:1616513069.448764:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513069.466468:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923472, portal 4 00000100:00000200:18.0:1616513069.466472:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923472, offset 0 00000400:00000200:18.0:1616513069.466478:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:18.0:1616513069.466485:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:18.0:1616513069.466496:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513069.466501:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:18.0:1616513069.466506:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513069.466509:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:18.0:1616513069.466511:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:18.0:1616513069.466515:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513069.466563:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.466584:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513069.466590:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513069.466594:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.0:1616513069.466595:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:3.0:1616513069.466600:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923488, portal 4 00000400:00000200:27.0:1616513069.466601:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:3.0:1616513069.466606:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923488, offset 0 00000100:00000200:27.0:1616513069.466609:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cd9488d80 x1693067725923472/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513069.466613:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:3.0:1616513069.466620:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:27.0:1616513069.466622:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc6e8 00000400:00000200:3.0:1616513069.466636:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513069.466643:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:3.0:1616513069.466644:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000100:00000200:31.0:1616513069.466651:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923504, portal 4 00000800:00000200:3.0:1616513069.466651:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513069.466654:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000100:00000200:31.0:1616513069.466657:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923504, offset 0 00000800:00000200:3.0:1616513069.466657:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:31.0:1616513069.466663:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:3.0:1616513069.466663:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513069.466668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:31.0:1616513069.466670:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:27.0:1616513069.466677:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513069.466682:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:31.0:1616513069.466683:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:27.0:1616513069.466683:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513069.466689:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:3.0:1616513069.466689:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:31.0:1616513069.466690:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000100:00000200:27.0:1616513069.466694:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456e8d80 x1693067725923488/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513069.466697:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513069.466701:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:31.0:1616513069.466703:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000400:00000200:27.0:1616513069.466704:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2088 00000800:00000200:27.0:1616513069.466706:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:31.0:1616513069.466709:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:27.2:1616513069.466734:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513069.466742:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.466748:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513069.466751:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513069.466756:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:25.2:1616513069.466757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:27.2:1616513069.466759:0:6475:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:3.0:1616513069.466769:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513069.466770:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626490 00000400:00000200:27.0:1616513069.466777:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164c9d [1] + 192 00000800:00000200:3.0:1616513069.466778:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513069.466780:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513069.466782:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:3.0:1616513069.466785:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000100:00000200:27.0:1616513069.466786:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cd9488d80 x1693067725923472/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513069.466792:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6264a0 00000800:00000200:27.0:1616513069.466801:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:3.0:1616513069.466802:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164cad [1] + 192 00000800:00000200:27.0:1616513069.466803:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513069.466805:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:3.0:1616513069.466807:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.466809:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cd9488d80 x1693067725923472/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513069.466814:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000100:00000200:18.0:1616513069.466820:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cd9488d80 x1693067725923472/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513069.466822:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513069.466825:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513069.466828:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc000 00000400:00000200:14.0:1616513069.466832:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:3.0:1616513069.466833:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456e8d80 x1693067725923488/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:3.0:1616513069.466848:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:14.0:1616513069.466850:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bfd580 x1693067725923504/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:3.0:1616513069.466851:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:3.0:1616513069.466852:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:14.0:1616513069.466862:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c36d8 00000800:00000200:14.0:1616513069.466866:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513069.466869:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:7.0:1616513069.466870:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456e8d80 x1693067725923488/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513069.466883:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456e8d80 x1693067725923488/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513069.466893:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2000 00000800:00000200:25.2:1616513069.466934:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513069.466944:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513069.466950:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513069.466952:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513069.466960:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513069.466967:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6264b0 00000400:00000200:14.0:1616513069.466973:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c4069 [1] + 192 00000400:00000200:14.0:1616513069.466978:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.466983:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bfd580 x1693067725923504/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513069.466996:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513069.466999:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513069.467001:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:31.0:1616513069.467006:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bfd580 x1693067725923504/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513069.467018:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bfd580 x1693067725923504/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513069.467029:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2000 00000800:00000200:7.2:1616513069.470934:0:48332:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513069.470988:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.470997:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513069.471006:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.38@tcp1 : PUT - for me 00000400:00000200:14.0:1616513069.471015:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.38@tcp1 of length 224 into portal 26 MB=0x604a353a99040 00000400:00000200:14.0:1616513069.471022:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.38@tcp1 of length 224/224 into md 0xf97c39c9 [1] + 0 00000400:00000200:14.0:1616513069.471027:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f298 00000400:00000200:14.0:1616513069.471029:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.471032:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513069.471036:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f298 00000800:00000200:14.0:1616513069.471039:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.471040:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.471042:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.471045:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.471058:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949390065728 00010000:00000200:24.0:1616513069.471069:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1693949390065728/t0(0) o400->42dcd879-73bf-2a9b-3b34-452656b73924@10.31.141.38@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.471083:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949390065728, offset 192 00000400:00000200:24.0:1616513069.471090:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.38@tcp1 00000400:00000200:24.0:1616513069.471098:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.38@tcp1 routed destination 00000400:00000200:24.0:1616513069.471113:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.38@tcp1(10.31.141.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.471120:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.471126:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.471129:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.471131:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.471136:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.471140:0:48332:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.471143:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513069.471189:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.471195:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513069.471197:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513069.471199:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.471202:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.471206:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:18.0:1616513069.471209:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.471407:0:48332:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513069.471418:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.471426:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513069.471427:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513069.471448:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.38@tcp1 : PUT - for me 00000400:00000200:22.0:1616513069.471457:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.38@tcp1 of length 224 into portal 12 MB=0x604a353a99100 00000400:00000200:22.0:1616513069.471463:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.38@tcp1 of length 224/224 into md 0xf96450b9 [1] + 4480 00000400:00000200:22.0:1616513069.471469:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.471472:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513069.471479:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.471482:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.471483:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.471557:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949390065920 00010000:00000200:2.0:1616513069.471569:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad2400 x1693949390065920/t0(0) o400->2f57e993-ad66-d653-7661-fc6dab459373@10.31.141.38@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513069.471583:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949390065920, offset 192 00000400:00000200:2.0:1616513069.471590:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.38@tcp1 00000400:00000200:2.0:1616513069.471597:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.38@tcp1 routed destination 00000400:00000200:2.0:1616513069.471612:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.38@tcp1(10.31.141.38@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.471618:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.471624:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.471627:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.471629:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.471633:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.471682:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.471694:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.471697:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.471699:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.471702:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.471705:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed540 00000800:00000200:8.0:1616513069.471707:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.471708:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.477306:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513069.477361:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.477370:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513069.477372:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513069.477379:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.20@tcp1 : PUT - for me 00000800:00000200:18.0:1616513069.477380:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513069.477389:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.20@tcp1 of length 224 into portal 26 MB=0x60393e91dc540 00000400:00000200:18.0:1616513069.477390:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.20@tcp1 : PUT - for me 00000400:00000200:14.0:1616513069.477396:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.20@tcp1 of length 224/224 into md 0xf97c39d9 [1] + 0 00000400:00000200:18.0:1616513069.477397:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.20@tcp1 of length 224 into portal 12 MB=0x60393e91dc640 00000400:00000200:14.0:1616513069.477400:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f100 00000400:00000200:14.0:1616513069.477403:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513069.477404:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.20@tcp1 of length 224/224 into md 0xf96450b9 [1] + 4704 00000400:00000200:18.0:1616513069.477408:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.477409:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513069.477414:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f100 00000100:00000200:18.0:1616513069.477416:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513069.477417:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513069.477420:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513069.477422:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513069.477422:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513069.477424:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.477425:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513069.477427:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.477477:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783666382144 00010000:00000200:24.0:1616513069.477488:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1692783666382144/t0(0) o400->3db00cff-60ba-a80b-2a82-e5fb9cb7bdc3@10.31.140.20@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:2.0:1616513069.477499:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783666382400 00000100:00000200:24.0:1616513069.477502:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783666382144, offset 192 00000400:00000200:24.0:1616513069.477508:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.20@tcp1 00010000:00000200:2.0:1616513069.477512:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d3f00 x1692783666382400/t0(0) o400->4179e2ca-ba75-38d0-0032-d070c21a69a3@10.31.140.20@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513069.477516:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.20@tcp1 routed destination 00000100:00000200:2.0:1616513069.477526:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783666382400, offset 192 00000400:00000200:24.0:1616513069.477530:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.20@tcp1(10.31.140.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513069.477532:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.20@tcp1 00000800:00000200:24.0:1616513069.477537:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:2.0:1616513069.477540:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.20@tcp1 routed destination 00000800:00000200:24.0:1616513069.477543:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.477545:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.477548:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.477552:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513069.477555:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.20@tcp1(10.31.140.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513069.477557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.477559:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513069.477561:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.477567:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513069.477568:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513069.477570:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.477572:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513069.477576:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513069.477576:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513069.477579:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513069.477579:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.477585:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.477591:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a08 00000800:00000200:22.0:1616513069.477596:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.477623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.477680:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.477687:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513069.477690:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513069.477696:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513069.477700:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eeccc0 00000800:00000200:20.0:1616513069.477704:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.477707:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.507866:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.507877:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.507881:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.507885:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.59@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.507890:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.59@tcp1 of length 224 into portal 26 MB=0x6018445e02ac0 00000400:00000200:8.0:1616513069.507894:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.59@tcp1 of length 224/224 into md 0xf97c39f9 [1] + 0 00000400:00000200:8.0:1616513069.507896:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f5c8 00000400:00000200:8.0:1616513069.507898:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.507899:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513069.507902:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f5c8 00000800:00000200:8.0:1616513069.507903:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.507905:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.507906:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.507907:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.507910:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.507913:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.59@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.507916:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.59@tcp1 of length 224 into portal 12 MB=0x6018445e02b40 00000400:00000200:8.0:1616513069.507918:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.59@tcp1 of length 224/224 into md 0x865b9f31 [1] + 30736 00000400:00000200:8.0:1616513069.507920:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.507922:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.507925:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.507926:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.507926:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.507930:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.507977:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517479893696 00010000:00000200:24.0:1616513069.507987:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1690517479893696/t0(0) o400->291f73dd-b7d3-51f0-06fd-f01b5d8e5892@10.31.142.59@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.507999:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517479893824 00000100:00000200:24.0:1616513069.508002:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690517479893696, offset 192 00000400:00000200:24.0:1616513069.508010:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.59@tcp1 00010000:00000200:16.0:1616513069.508011:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9d100 x1690517479893824/t0(0) o400->c945ea9e-c58e-cb86-15ad-615a7435a9f6@10.31.142.59@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513069.508017:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.59@tcp1 routed destination 00000100:00000200:16.0:1616513069.508026:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690517479893824, offset 192 00000400:00000200:24.0:1616513069.508032:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.59@tcp1(10.31.142.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513069.508032:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.59@tcp1 00000800:00000200:24.0:1616513069.508039:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513069.508039:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.59@tcp1 routed destination 00000400:00000200:16.0:1616513069.508054:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.59@tcp1(10.31.142.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.508061:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.508062:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.508064:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.508066:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.508067:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.508070:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.508071:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513069.508072:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.508077:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.508078:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.508078:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513069.508082:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.508093:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.508099:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.508101:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513069.508103:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513069.508108:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.508112:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43cb0 00000800:00000200:12.0:1616513069.508115:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513069.508130:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.508137:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.508140:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.508146:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.508151:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:14.0:1616513069.508154:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.508157:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.510297:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513069.510306:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.510312:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513069.510317:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.219@tcp1 : PUT - for me 00000400:00000200:30.0:1616513069.510323:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.219@tcp1 of length 224 into portal 26 MB=0x5ffdb95859c00 00000400:00000200:30.0:1616513069.510328:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.219@tcp1 of length 224/224 into md 0xf97c3a09 [1] + 0 00000400:00000200:30.0:1616513069.510331:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e770 00000400:00000200:30.0:1616513069.510332:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513069.510334:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513069.510336:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e770 00000800:00000200:30.0:1616513069.510338:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513069.510340:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.510341:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513069.510343:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.510403:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693455035392 00010000:00000200:24.0:1616513069.510413:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688693455035392/t0(0) o400->28f9a612-a440-86e5-89e7-837644952324@10.31.140.219@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.510427:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693455035392, offset 192 00000400:00000200:24.0:1616513069.510446:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.219@tcp1 00000400:00000200:24.0:1616513069.510453:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.219@tcp1 routed destination 00000400:00000200:24.0:1616513069.510468:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.219@tcp1(10.31.140.219@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.510475:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.510481:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.510483:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.510485:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.510490:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.510495:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.510497:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513069.510549:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.510555:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.510558:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.510565:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.510570:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:14.0:1616513069.510574:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.510577:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.510603:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513069.510610:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.510617:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.510620:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513069.510628:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.219@tcp1 : PUT - for me 00000400:00000200:18.0:1616513069.510638:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.219@tcp1 of length 224 into portal 12 MB=0x5ffdb95859f00 00000400:00000200:18.0:1616513069.510644:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.219@tcp1 of length 224/224 into md 0xf96450b9 [1] + 4928 00000400:00000200:18.0:1616513069.510649:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.510651:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513069.510658:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.510660:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.510662:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.510718:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693455036160 00010000:00000200:2.0:1616513069.510730:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d0000 x1688693455036160/t0(0) o400->ef8be3f9-7695-a420-4841-6e40d265df59@10.31.140.219@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513069.510743:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693455036160, offset 192 00000400:00000200:2.0:1616513069.510750:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.219@tcp1 00000400:00000200:2.0:1616513069.510757:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.219@tcp1 routed destination 00000400:00000200:2.0:1616513069.510771:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.219@tcp1(10.31.140.219@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.510777:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.510782:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.510785:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.510787:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.510791:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.510838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.510849:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.510851:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.510853:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.510856:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.510858:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed6d8 00000800:00000200:8.0:1616513069.510860:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.510861:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.529229:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513069.529282:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.529291:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513069.529292:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513069.529298:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513069.529299:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.96@tcp1 : PUT - for me 00000400:00000200:30.0:1616513069.529303:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.96@tcp1 : PUT - for me 00000400:00000200:14.0:1616513069.529307:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.96@tcp1 of length 224 into portal 26 MB=0x5ffdd16e723c0 00000400:00000200:30.0:1616513069.529309:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.96@tcp1 of length 224 into portal 12 MB=0x5ffdd16e72780 00000400:00000200:30.0:1616513069.529312:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.96@tcp1 of length 224/224 into md 0xf96450b9 [1] + 5152 00000400:00000200:30.0:1616513069.529315:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.529318:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.96@tcp1 of length 224/224 into md 0xf97c3a29 [1] + 0 00000400:00000200:14.0:1616513069.529322:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e5d8 00000100:00000200:30.0:1616513069.529324:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513069.529325:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513069.529328:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000100:00000200:14.0:1616513069.529328:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:30.0:1616513069.529330:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513069.529331:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.529333:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513069.529333:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e5d8 00000800:00000200:14.0:1616513069.529336:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.529338:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.529340:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.529392:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699920656256 00010000:00000200:2.0:1616513069.529405:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cd948e300 x1688699920656256/t0(0) o400->d97d47e4-a269-96f9-bdf3-864fd54685c2@10.31.142.96@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513069.529412:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699920655296 00000100:00000200:2.0:1616513069.529419:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699920656256, offset 192 00010000:00000200:24.0:1616513069.529423:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688699920655296/t0(0) o400->028b26e5-2c63-a7ee-a4b4-c4b15f6b6b0a@10.31.142.96@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513069.529425:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.96@tcp1 00000400:00000200:2.0:1616513069.529432:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.96@tcp1 routed destination 00000100:00000200:24.0:1616513069.529448:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699920655296, offset 192 00000400:00000200:24.0:1616513069.529454:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.96@tcp1 00000400:00000200:2.0:1616513069.529459:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.96@tcp1(10.31.142.96@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513069.529462:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.96@tcp1 routed destination 00000800:00000200:2.0:1616513069.529466:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513069.529472:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.529475:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513069.529477:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.96@tcp1(10.31.142.96@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.529477:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513069.529481:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513069.529484:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513069.529487:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.529492:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.529494:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.529497:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513069.529498:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513069.529501:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513069.529504:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.529506:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.529506:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:24.0:1616513069.529509:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.529509:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513069.529512:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.529516:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eeddc0 00000800:00000200:18.0:1616513069.529519:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513069.529573:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513069.529580:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513069.529582:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513069.529589:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513069.529595:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000800:00000200:16.0:1616513069.529598:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513069.529601:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.536958:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.536967:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.536971:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.536975:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.536981:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x60399133b1680 00000400:00000200:8.0:1616513069.536984:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0x865b9f31 [1] + 30960 00000400:00000200:8.0:1616513069.536986:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.536988:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.536991:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.536993:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.536994:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.536995:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.537073:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805847783040 00010000:00000200:16.0:1616513069.537143:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9a400 x1692805847783040/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:695/0 lens 336/336 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.537158:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805847783040, offset 192 00000400:00000200:16.0:1616513069.537164:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:16.0:1616513069.537171:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:16.0:1616513069.537185:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.537192:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.537198:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.537200:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.537203:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.537207:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.537215:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.537268:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.537274:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.537277:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.537283:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.537288:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43e48 00000800:00000200:14.0:1616513069.537292:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.537295:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.550557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513069.550567:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.550571:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513069.550576:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.150@tcp1 : PUT - for me 00000400:00000200:30.0:1616513069.550581:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.150@tcp1 of length 224 into portal 26 MB=0x5ffd752370500 00000400:00000200:30.0:1616513069.550584:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.150@tcp1 of length 224/224 into md 0xf97c3a49 [1] + 0 00000400:00000200:30.0:1616513069.550587:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e550 00000400:00000200:30.0:1616513069.550588:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513069.550590:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513069.550592:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e550 00000800:00000200:30.0:1616513069.550594:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513069.550595:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.550596:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513069.550598:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.550660:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675145942272 00010000:00000200:24.0:1616513069.550671:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688675145942272/t0(0) o400->f32531ad-b02a-657c-ace7-cdb33ee96b0b@10.31.129.150@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.550685:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688675145942272, offset 192 00000400:00000200:24.0:1616513069.550691:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.150@tcp1 00000400:00000200:24.0:1616513069.550698:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.150@tcp1 routed destination 00000400:00000200:24.0:1616513069.550713:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.150@tcp1(10.31.129.150@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.550719:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.550725:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.550728:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.550730:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.550734:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.550741:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.550743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.550798:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513069.550805:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513069.550808:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513069.550815:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513069.550820:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:16.0:1616513069.550823:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513069.550827:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.551014:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.551022:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.551026:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.551030:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.150@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.551034:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.150@tcp1 of length 224 into portal 12 MB=0x5ffd752370a00 00000400:00000200:8.0:1616513069.551037:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.150@tcp1 of length 224/224 into md 0x865b9f31 [1] + 31296 00000400:00000200:8.0:1616513069.551039:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.551041:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.551044:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.551045:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.551046:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.551048:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.551066:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675145943552 00010000:00000200:16.0:1616513069.551076:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b99200 x1688675145943552/t0(0) o400->cf9bbd8f-51b3-a26f-7936-270b3864a919@10.31.129.150@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.551090:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688675145943552, offset 192 00000400:00000200:16.0:1616513069.551096:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.150@tcp1 00000400:00000200:16.0:1616513069.551103:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.150@tcp1 routed destination 00000400:00000200:16.0:1616513069.551117:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.150@tcp1(10.31.129.150@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.551124:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.551129:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.551132:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.551134:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.551140:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.551145:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.551197:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.551203:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.551206:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.551212:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.551216:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43320 00000800:00000200:14.0:1616513069.551219:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.551222:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.560694:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.560752:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.560761:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513069.560769:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.230@tcp1 : PUT - for me 00000400:00000200:16.0:1616513069.560777:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.230@tcp1 of length 224 into portal 26 MB=0x604f603c9c7c0 00000400:00000200:16.0:1616513069.560783:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.230@tcp1 of length 224/224 into md 0xf97c3a59 [1] + 0 00000400:00000200:16.0:1616513069.560788:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e3b8 00000400:00000200:16.0:1616513069.560790:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513069.560793:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513069.560797:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e3b8 00000800:00000200:16.0:1616513069.560800:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513069.560802:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513069.560804:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.560807:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.560813:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304532285376 00000800:00000200:8.0:1616513069.560816:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.560818:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00010000:00000200:24.0:1616513069.560821:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1694304532285376/t0(0) o400->f3ed97b4-e870-1624-f107-77ac40115765@10.31.140.230@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513069.560826:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.230@tcp1 : PUT - for me 00000100:00000200:24.0:1616513069.560833:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694304532285376, offset 192 00000400:00000200:12.0:1616513069.560835:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.230@tcp1 of length 224 into portal 12 MB=0x604f603c9c840 00000400:00000200:24.0:1616513069.560838:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.230@tcp1 00000400:00000200:12.0:1616513069.560843:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.230@tcp1 of length 224/224 into md 0x865b9f31 [1] + 31520 00000400:00000200:24.0:1616513069.560845:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.230@tcp1 routed destination 00000400:00000200:12.0:1616513069.560849:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.560851:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513069.560857:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.230@tcp1(10.31.140.230@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513069.560862:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.560863:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513069.560865:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.560867:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513069.560868:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.560871:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.560872:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.560876:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513069.560877:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304532285504 00000800:00000200:11.2:1616513069.560882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.560883:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:16.0:1616513069.560887:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa43f00 x1694304532285504/t0(0) o400->6bd87081-7695-e11a-8c1f-38a79ce82dc3@10.31.140.230@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.560900:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694304532285504, offset 192 00000400:00000200:16.0:1616513069.560905:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.230@tcp1 00000400:00000200:16.0:1616513069.560911:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.230@tcp1 routed destination 00000400:00000200:16.0:1616513069.560924:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.230@tcp1(10.31.140.230@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.560930:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513069.560934:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513069.560936:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.560938:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.560940:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513069.560943:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513069.560944:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513069.560945:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513069.560946:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513069.560952:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513069.560952:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513069.560958:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2b28 00000800:00000200:20.0:1616513069.560961:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513069.560963:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513069.560966:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513069.560968:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513069.560973:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513069.560975:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43650 00000800:00000200:30.0:1616513069.560978:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513069.560980:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.578613:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923520, portal 4 00000100:00000200:24.0:1616513069.578618:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923520, offset 0 00000400:00000200:24.0:1616513069.578625:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:24.0:1616513069.578630:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:24.0:1616513069.578643:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.578650:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:24.0:1616513069.578656:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.578659:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:24.0:1616513069.578661:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513069.578666:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513069.578705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513069.578717:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513069.578723:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513069.578725:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513069.578728:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:8.0:1616513069.578732:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513069.578738:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4557cee780 x1693067725923520/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513069.578750:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2990 00000800:00000200:8.0:1616513069.578754:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513069.578903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:24.0:1616513069.578914:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513069.578921:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513069.578925:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:24.0:1616513069.578928:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:24.0:1616513069.578935:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6264c0 00000400:00000200:24.0:1616513069.578940:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d2751 [1] + 192 00000400:00000200:24.0:1616513069.578945:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513069.578950:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4557cee780 x1693067725923520/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:24.0:1616513069.578965:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513069.578967:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513069.578969:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:28.0:1616513069.579019:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4557cee780 x1693067725923520/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513069.579044:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4557cee780 x1693067725923520/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513069.579054:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:11.2:1616513069.581752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.581809:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.581818:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.581819:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:11.2:1616513069.581826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513069.581828:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.151@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.581836:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.151@tcp1 of length 224 into portal 26 MB=0x5ffdfb84464c0 00000400:00000200:12.0:1616513069.581843:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.151@tcp1 of length 224/224 into md 0xf97c3a79 [1] + 0 00000400:00000200:12.0:1616513069.581849:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f8f8 00000400:00000200:12.0:1616513069.581852:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.581855:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513069.581859:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f8f8 00000800:00000200:12.0:1616513069.581862:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.581865:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.581867:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.581869:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.581876:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513069.581879:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513069.581883:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.151@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.581889:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.151@tcp1 of length 224 into portal 12 MB=0x5ffdfb8446580 00000400:00000200:12.0:1616513069.581894:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.151@tcp1 of length 224/224 into md 0x865b9f31 [1] + 31744 00000400:00000200:12.0:1616513069.581898:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.581900:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513069.581905:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.581907:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.581909:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.581920:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711217833152 00010000:00000200:24.0:1616513069.581930:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688711217833152/t0(0) o400->e45fd0e6-8267-c0bc-4b2b-bf36e42fca05@10.31.141.151@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.581944:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688711217833152, offset 192 00000400:00000200:24.0:1616513069.581949:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.151@tcp1 00000400:00000200:24.0:1616513069.581956:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.151@tcp1 routed destination 00000100:00000200:16.0:1616513069.581969:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711217833344 00000400:00000200:24.0:1616513069.581970:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.151@tcp1(10.31.141.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.581980:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:16.0:1616513069.581981:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa43180 x1688711217833344/t0(0) o400->565b089f-a256-9087-f7a3-7c6c7e3f8d02@10.31.141.151@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513069.581985:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.581988:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.581990:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.581994:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513069.581994:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688711217833344, offset 192 00000800:00000200:11.2:1616513069.581999:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.582000:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513069.582000:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.151@tcp1 00000800:00000200:8.0:1616513069.582006:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513069.582007:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.151@tcp1 routed destination 00000800:00000200:8.0:1616513069.582008:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.582009:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.582012:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513069.582013:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513069.582014:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3c28 00000800:00000200:8.0:1616513069.582016:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:16.0:1616513069.582023:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.151@tcp1(10.31.141.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.582028:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.582034:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.582036:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.582039:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.582043:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.582051:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.582106:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.582113:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.582116:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.582123:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.582127:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42000 00000800:00000200:14.0:1616513069.582131:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.582134:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.592477:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.592533:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.592542:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.592551:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.65@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.592559:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.65@tcp1 of length 224 into portal 12 MB=0x5ffdd99ddbfc0 00000400:00000200:12.0:1616513069.592566:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.65@tcp1 of length 224/224 into md 0x865b9f31 [1] + 31968 00000400:00000200:12.0:1616513069.592571:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.592573:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513069.592579:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.592582:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.592584:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.592586:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.592641:0:48346:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513069.592643:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702117855168 00000800:00000200:30.0:1616513069.592654:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:16.0:1616513069.592656:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa45e80 x1688702117855168/t0(0) o400->7c5493a0-2da1-a983-52f5-6ecbc16b057c@10.31.142.65@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:30.0:1616513069.592662:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000100:00000200:16.0:1616513069.592670:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702117855168, offset 192 00000400:00000200:30.0:1616513069.592672:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.65@tcp1 : PUT - for me 00000400:00000200:16.0:1616513069.592677:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.65@tcp1 00000400:00000200:30.0:1616513069.592680:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.65@tcp1 of length 224 into portal 26 MB=0x5ffdd99ddbc80 00000400:00000200:16.0:1616513069.592684:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.65@tcp1 routed destination 00000400:00000200:30.0:1616513069.592687:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.65@tcp1 of length 224/224 into md 0xf97c3a89 [1] + 0 00000400:00000200:30.0:1616513069.592692:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fe48 00000400:00000200:30.0:1616513069.592694:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513069.592697:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513069.592699:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.65@tcp1(10.31.142.65@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513069.592701:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:30.0:1616513069.592704:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513069.592705:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513069.592707:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513069.592708:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513069.592711:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513069.592712:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513069.592714:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.592716:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.592720:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.592725:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.592765:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702117854336 00010000:00000200:24.0:1616513069.592776:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688702117854336/t0(0) o400->07c05088-3296-d930-d526-462bcc46f37b@10.31.142.65@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513069.592782:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.592789:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:24.0:1616513069.592790:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702117854336, offset 192 00000400:00000200:20.0:1616513069.592792:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513069.592792:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513069.592796:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.65@tcp1 00000400:00000200:20.0:1616513069.592798:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513069.592802:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42880 00000400:00000200:24.0:1616513069.592804:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.65@tcp1 routed destination 00000800:00000200:20.0:1616513069.592809:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513069.592831:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.65@tcp1(10.31.142.65@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.592837:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.592843:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.592845:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.592847:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.592852:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.592856:0:48346:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.592858:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513069.592909:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.592915:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.592918:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.592925:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.592930:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:14.0:1616513069.592934:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.592937:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.610519:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923536, portal 4 00000100:00000200:2.0:1616513069.610524:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923536, offset 0 00000400:00000200:2.0:1616513069.610531:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:2.0:1616513069.610537:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:2.0:1616513069.610550:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.610556:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:2.0:1616513069.610563:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.610566:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:2.0:1616513069.610569:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:2.0:1616513069.610576:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513069.610624:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513069.610645:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513069.610652:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513069.610655:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513069.610663:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513069.610672:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cd948f980 x1693067725923536/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513069.610685:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eedf58 00000800:00000200:1.0:1616513069.610691:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513069.610695:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513069.610702:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513069.610711:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513069.610720:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6264d0 00000400:00000200:1.0:1616513069.610727:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff715 [1] + 192 00000400:00000200:1.0:1616513069.610732:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513069.610737:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cd948f980 x1693067725923536/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:9.0:1616513069.610748:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513069.610752:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513069.610755:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513069.610759:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:2.0:1616513069.610761:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cd948f980 x1693067725923536/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513069.610775:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cd948f980 x1693067725923536/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513069.610786:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed760 00000800:00000200:7.2:1616513069.619929:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513069.619986:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.619996:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513069.619997:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.620005:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.63@tcp1 : PUT - for me 00000400:00000200:18.0:1616513069.620014:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.63@tcp1 of length 224 into portal 12 MB=0x5ffe2b3361f40 00000400:00000200:18.0:1616513069.620021:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.63@tcp1 of length 224/224 into md 0xf96450b9 [1] + 5376 00000400:00000200:18.0:1616513069.620026:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.620029:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513069.620036:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.620038:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.620040:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.620049:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:2.0:1616513069.620104:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724017913664 00000800:00000200:12.0:1616513069.620105:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.620114:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00010000:00000200:2.0:1616513069.620117:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d4c80 x1688724017913664/t0(0) o400->dc42ba01-1665-cc3f-d0e6-e84feb1d3cd8@10.31.140.63@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513069.620122:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.63@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.620131:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.63@tcp1 of length 224 into portal 26 MB=0x5ffe2b3361d00 00000100:00000200:2.0:1616513069.620131:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688724017913664, offset 192 00000400:00000200:12.0:1616513069.620137:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.63@tcp1 of length 224/224 into md 0xf97c3aa9 [1] + 0 00000400:00000200:12.0:1616513069.620142:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ebb0 00000400:00000200:12.0:1616513069.620145:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513069.620147:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.63@tcp1 00000100:00000200:12.0:1616513069.620150:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513069.620154:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ebb0 00000400:00000200:2.0:1616513069.620155:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.63@tcp1 routed destination 00000800:00000200:12.0:1616513069.620157:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.620159:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.620161:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.620163:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513069.620170:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.63@tcp1(10.31.140.63@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.620176:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513069.620182:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.620184:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513069.620186:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513069.620191:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.620196:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.620205:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.620209:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.620211:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:24.0:1616513069.620216:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724017913088 00000400:00000200:8.0:1616513069.620216:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.620219:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed760 00000800:00000200:8.0:1616513069.620221:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.620223:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:24.0:1616513069.620226:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688724017913088/t0(0) o400->0429ab19-2ee9-872a-fd65-9f568a4654b8@10.31.140.63@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.620239:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688724017913088, offset 192 00000400:00000200:24.0:1616513069.620245:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.63@tcp1 00000400:00000200:24.0:1616513069.620252:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.63@tcp1 routed destination 00000400:00000200:24.0:1616513069.620266:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.63@tcp1(10.31.140.63@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.620272:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.620277:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.620280:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.620283:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.620288:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513069.620294:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513069.620298:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.620351:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.620357:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.620360:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.620367:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.620372:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2880 00000800:00000200:14.0:1616513069.620376:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.620379:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513069.626540:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923552, portal 4 00000100:00000200:30.0:1616513069.626545:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923552, offset 0 00000400:00000200:30.0:1616513069.626551:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:30.0:1616513069.626557:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:30.0:1616513069.626570:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513069.626576:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:30.0:1616513069.626583:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513069.626586:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:30.0:1616513069.626588:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:30.0:1616513069.626593:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513069.626644:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.626665:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513069.626672:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513069.626675:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513069.626681:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513069.626688:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56cbad7980 x1693067725923552/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513069.626700:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee990 00000800:00000200:27.0:1616513069.626704:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513069.626707:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513069.626810:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513069.626819:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.626826:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513069.626834:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513069.626848:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6264e0 00000400:00000200:27.0:1616513069.626857:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164cdd [1] + 192 00000400:00000200:27.0:1616513069.626862:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513069.626866:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56cbad7980 x1693067725923552/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:3.0:1616513069.626872:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513069.626880:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513069.626883:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513069.626885:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:30.0:1616513069.626888:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56cbad7980 x1693067725923552/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513069.626912:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56cbad7980 x1693067725923552/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513069.626920:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee660 00000100:00000200:1.0:1616513069.642652:0:6491:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923568, portal 4 00000100:00000200:1.0:1616513069.642658:0:6491:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923568, offset 0 00000400:00000200:1.0:1616513069.642666:0:6491:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:1.0:1616513069.642673:0:6491:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:1.0:1616513069.642687:0:6491:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:1.0:1616513069.642694:0:6491:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:1.0:1616513069.642702:0:6491:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513069.642705:0:6491:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:1.0:1616513069.642707:0:6491:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:1.0:1616513069.642713:0:6491:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513069.642759:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513069.642824:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513069.642831:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513069.642834:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.642841:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:12.0:1616513069.642848:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4558ad1f80 x1693067725923568/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513069.642860:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84bb0 00000800:00000200:12.0:1616513069.642865:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513069.642868:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513069.642941:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513069.642951:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513069.642958:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513069.642965:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513069.642972:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6264f0 00000400:00000200:8.0:1616513069.642977:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d27c1 [1] + 192 00000400:00000200:8.0:1616513069.642981:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.642985:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4558ad1f80 x1693067725923568/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513069.642998:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513069.643000:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:8.0:1616513069.643002:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513069.643004:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:1.0:1616513069.643010:0:6491:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4558ad1f80 x1693067725923568/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513069.643022:0:6491:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4558ad1f80 x1693067725923568/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513069.643033:0:6491:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84330 00000800:00000200:7.2:1616513069.678103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513069.678117:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.678127:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513069.678136:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.129@tcp1 : PUT - for me 00000400:00000200:30.0:1616513069.678146:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.129@tcp1 of length 224 into portal 26 MB=0x60184f13fc7c0 00000400:00000200:30.0:1616513069.678154:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.129@tcp1 of length 224/224 into md 0xf97c3ac9 [1] + 0 00000400:00000200:30.0:1616513069.678159:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e880 00000400:00000200:30.0:1616513069.678162:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513069.678164:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513069.678169:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e880 00000800:00000200:30.0:1616513069.678172:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.678172:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513069.678175:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.678177:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.678252:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690520355063744 00010000:00000200:24.0:1616513069.678264:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1690520355063744/t0(0) o400->827fa20b-4b40-acad-e4f7-b36c50fc6710@10.31.142.129@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.678278:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690520355063744, offset 192 00000400:00000200:24.0:1616513069.678284:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.129@tcp1 00000400:00000200:24.0:1616513069.678293:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.129@tcp1 routed destination 00000400:00000200:24.0:1616513069.678307:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.129@tcp1(10.31.142.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.678314:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.678320:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.678323:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.678325:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.678331:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.678338:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.678339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.678353:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.678360:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.678363:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.678369:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.678375:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:12.0:1616513069.678378:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.678381:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.678442:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.678457:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.678466:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.678475:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.129@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.678483:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.129@tcp1 of length 224 into portal 12 MB=0x60184f13fd580 00000400:00000200:8.0:1616513069.678490:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.129@tcp1 of length 224/224 into md 0x865b9f31 [1] + 32192 00000400:00000200:8.0:1616513069.678495:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.678498:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.678505:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.678508:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.678510:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.678513:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.678581:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690520355067264 00010000:00000200:16.0:1616513069.678594:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9de80 x1690520355067264/t0(0) o400->f249399c-c7f2-2bef-c340-12085911121a@10.31.142.129@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.678607:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690520355067264, offset 192 00000400:00000200:16.0:1616513069.678613:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.129@tcp1 00000400:00000200:16.0:1616513069.678620:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.129@tcp1 routed destination 00000400:00000200:16.0:1616513069.678634:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.129@tcp1(10.31.142.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.678640:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.678646:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.678648:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.678652:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.678657:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.678662:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.678717:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.678724:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.678727:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.678734:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.678740:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42dd0 00000800:00000200:14.0:1616513069.678744:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.678747:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.698305:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.698359:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.698368:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.698377:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.698386:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 600 into portal 12 MB=0x5fa07dd43ab58 00000400:00000200:12.0:1616513069.698393:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 600/600 into md 0x865b9f31 [1] + 32416 00000400:00000200:12.0:1616513069.698398:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.698401:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513069.698408:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.698411:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513069.698412:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.698417:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.698472:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467864 00010000:00000200:16.0:1616513069.698594:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa45100 x1682286567467864/t77331494629(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 600/592 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 301/0 00000100:00000200:16.0:1616513069.698609:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 560 bytes to portal 10, xid 1682286567467864, offset 192 00000400:00000200:16.0:1616513069.698615:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.698623:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.698638:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.698645:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 560 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.698651:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.698654:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.698656:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.698661:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.698666:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.698675:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.698680:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.698682:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.698688:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.698691:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a437e8 00000800:00000200:8.0:1616513069.698694:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.698697:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.704162:0:48358:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513069.704219:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.704228:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513069.704231:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513069.704237:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:18.0:1616513069.704253:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 448 into portal 23 MB=0x5fa07dd43ab5c 00000400:00000200:18.0:1616513069.704261:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 17 from 12345-10.242.110.165@tcp1 of length 448/448 into md 0xf33d3df9 [1] + 0 00000400:00000200:18.0:1616513069.704266:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.704271:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt_readpage 00000800:00000200:18.0:1616513069.704278:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.704282:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.704283:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513069.704343:0:170096:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467868 00000400:00000200:26.0:1616513069.704633:0:170096:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:26.0:1616513069.704641:0:170096:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:26.0:1616513069.704648:0:170096:0:(lib-move.c:1738:lnet_handle_send()) rspt_next_hop_nid = 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513069.704659:0:170096:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513069.704666:0:170096:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 8192 bytes in 2 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513069.704670:0:170096:0:(o2iblnd_cb.c:816:kiblnd_setup_rd_kiov()) niov 2 offset 0 nob 8192 00000800:00000200:26.0:1616513069.704676:0:170096:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513069.704679:0:170096:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513069.704681:0:170096:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513069.704686:0:170096:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.704691:0:48358:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:26.0:1616513069.704692:0:170096:0:(niobuf.c:262:ptlrpc_start_bulk_transfer()) Transferring 2 pages 8192 bytes via portal 14 id 12345-10.242.110.165@tcp1 mbits 0x5fa07dd43ab5c-0x5fa07dd43ab5c 00000800:00000200:14.0:1616513069.704745:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.704751:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.704754:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513069.704761:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.704761:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513069.704764:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.704769:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513069.704770:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.704776:0:48358:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.704786:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.704792:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.704793:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513069.704794:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513069.704800:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:18.0:1616513069.704802:0:6471:0:(events.c:450:server_bulk_callback()) event type 5, status 0, desc ffff8d747d773e00 00000800:00000200:18.0:1616513069.704806:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.705534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513069.705595:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513069.705604:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513069.705613:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : ACK - for me 00000400:00000200:28.0:1616513069.705624:0:6463:0:(lib-move.c:3970:lnet_parse_ack()) 10.242.46.233@o2ib1: ACK from 12345-10.242.110.165@tcp1 into md 0xf97c4199 00000400:00000200:28.0:1616513069.705627:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513069.705629:0:6463:0:(events.c:450:server_bulk_callback()) event type 4, status 0, desc ffff8d747d773e00 00000400:00000200:28.0:1616513069.705634:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d7460de9d40 00000800:00000200:28.0:1616513069.705638:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513069.705640:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513069.705642:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.705645:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:26.0:1616513069.705708:0:170096:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d1f80 x1682286567467868/t0(0) o37->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 448/440 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513069.705721:0:170096:0:(niobuf.c:85:ptl_send_buf()) Sending 408 bytes to portal 10, xid 1682286567467868, offset 192 00000400:00000200:26.0:1616513069.705726:0:170096:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:26.0:1616513069.705732:0:170096:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:26.0:1616513069.705745:0:170096:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513069.705751:0:170096:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 408 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513069.705756:0:170096:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513069.705758:0:170096:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513069.705760:0:170096:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513069.705763:0:170096:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.705769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.705779:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.705784:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.705785:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.705791:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.705794:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af760 00000800:00000200:8.0:1616513069.705797:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.705799:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.706341:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513069.706394:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.706403:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.706403:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513069.706412:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:14.0:1616513069.706421:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 584 into portal 12 MB=0x5fa07dd43ab60 00000400:00000200:14.0:1616513069.706427:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 584/584 into md 0xf96450b9 [1] + 5600 00000400:00000200:14.0:1616513069.706433:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513069.706447:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513069.706454:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.706457:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513069.706459:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513069.706520:0:188463:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467872 00010000:00000200:2.0:1616513069.706595:0:188463:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa0900 x1682286567467872/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 584/568 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 301/0 00000100:00000200:2.0:1616513069.706610:0:188463:0:(niobuf.c:85:ptl_send_buf()) Sending 536 bytes to portal 10, xid 1682286567467872, offset 192 00000400:00000200:2.0:1616513069.706616:0:188463:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:2.0:1616513069.706624:0:188463:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:2.0:1616513069.706644:0:188463:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.706650:0:188463:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 536 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513069.706656:0:188463:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513069.706659:0:188463:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513069.706661:0:188463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513069.706666:0:188463:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.706672:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.706725:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.706731:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.706734:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.706741:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.706745:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eecdd0 00000800:00000200:22.0:1616513069.706749:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.706751:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.707552:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513069.707606:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513069.707616:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513069.707632:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:28.0:1616513069.707641:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 632 into portal 12 MB=0x5fa07dd43ab64 00000400:00000200:28.0:1616513069.707647:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 632/632 into md 0x865b9f31 [1] + 33016 00000400:00000200:28.0:1616513069.707652:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513069.707655:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513069.707662:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513069.707665:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.707667:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513069.707671:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.707674:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467876 00010000:00000200:16.0:1616513069.707745:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8b600 x1682286567467876/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 632/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.707756:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467876, offset 192 00000400:00000200:16.0:1616513069.707761:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.707766:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.707778:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.707783:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.707788:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.707790:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.707792:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.707795:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.707800:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.707808:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.707812:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.707813:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.707818:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.707821:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43540 00000800:00000200:8.0:1616513069.707823:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.707825:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.730729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.730786:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.730796:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.730805:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.730813:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 632 into portal 12 MB=0x5fa07dd43ab6c 00000400:00000200:12.0:1616513069.730820:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 632/632 into md 0x865b9f31 [1] + 33648 00000400:00000200:12.0:1616513069.730825:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.730828:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513069.730834:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.730838:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513069.730840:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.730844:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.730899:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467884 00010000:00000200:16.0:1616513069.730997:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa44c80 x1682286567467884/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 632/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.731013:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467884, offset 192 00000400:00000200:16.0:1616513069.731021:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.731028:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.731043:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.731049:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.731055:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.731058:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.731060:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.731066:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.731072:0:48361:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.731084:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513069.731090:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513069.731093:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513069.731099:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513069.731103:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ff0 00000800:00000200:30.0:1616513069.731107:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513069.731109:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.732757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.732778:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.732785:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.732793:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.732799:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 640 into portal 12 MB=0x5fa07dd43ab74 00000400:00000200:8.0:1616513069.732805:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 640/640 into md 0x865b9f31 [1] + 34280 00000400:00000200:8.0:1616513069.732809:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.732812:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.732818:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.732821:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.732822:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.732825:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.732831:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467892 00010000:00000200:16.0:1616513069.732895:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b98900 x1682286567467892/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 640/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.732907:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467892, offset 192 00000400:00000200:16.0:1616513069.732912:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.732917:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.732931:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.732936:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.732941:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.732943:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.732945:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.732949:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.732954:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513069.733007:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513069.733015:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513069.733017:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513069.733024:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513069.733028:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a425d8 00000800:00000200:28.0:1616513069.733034:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.733037:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.733209:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513069.733264:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.733274:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513069.733283:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.211@tcp1 : PUT - for me 00000400:00000200:22.0:1616513069.733292:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.211@tcp1 of length 224 into portal 26 MB=0x5ffe390b1fdc0 00000400:00000200:22.0:1616513069.733299:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.211@tcp1 of length 224/224 into md 0xf97c3ad9 [1] + 0 00000400:00000200:22.0:1616513069.733304:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e198 00000400:00000200:22.0:1616513069.733307:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.733310:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513069.733323:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e198 00000800:00000200:22.0:1616513069.733326:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.733326:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.733329:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513069.733334:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513069.733337:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.733338:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513069.733346:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.211@tcp1 : PUT - for me 00000400:00000200:18.0:1616513069.733355:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.211@tcp1 of length 224 into portal 12 MB=0x5ffe390b20040 00000400:00000200:18.0:1616513069.733361:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.211@tcp1 of length 224/224 into md 0xf96450b9 [1] + 6184 00000400:00000200:18.0:1616513069.733367:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.733369:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513069.733376:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.733379:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727733796288 00000800:00000200:18.0:1616513069.733379:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513069.733381:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513069.733390:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688727733796288/t0(0) o400->d96e6452-525f-2f7b-081c-3023c25afd42@10.31.141.211@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.733406:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688727733796288, offset 192 00000400:00000200:24.0:1616513069.733412:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.211@tcp1 00000400:00000200:24.0:1616513069.733419:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.211@tcp1 routed destination 00000400:00000200:24.0:1616513069.733449:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.211@tcp1(10.31.141.211@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513069.733450:0:188423:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727733796928 00000800:00000200:24.0:1616513069.733456:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.733462:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:10.0:1616513069.733463:0:188423:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d5e80 x1688727733796928/t0(0) o400->6a825daa-68d9-08fe-0649-0a7d80feb299@10.31.141.211@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513069.733465:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513069.733467:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513069.733472:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.733476:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:10.0:1616513069.733477:0:188423:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688727733796928, offset 192 00000100:00000200:24.0:1616513069.733479:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:10.0:1616513069.733483:0:188423:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.211@tcp1 00000400:00000200:10.0:1616513069.733489:0:188423:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.211@tcp1 routed destination 00000400:00000200:10.0:1616513069.733503:0:188423:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.211@tcp1(10.31.141.211@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513069.733509:0:188423:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513069.733515:0:188423:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513069.733517:0:188423:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513069.733519:0:188423:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513069.733523:0:188423:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513069.733528:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513069.733529:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513069.733535:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:8.0:1616513069.733536:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513069.733537:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.733538:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513069.733539:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.733541:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.733544:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513069.733545:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513069.733546:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.733549:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82660 00000400:00000200:14.0:1616513069.733550:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:8.0:1616513069.733551:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513069.733554:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.734582:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513069.734649:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513069.734658:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513069.734667:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:28.0:1616513069.734676:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 632 into portal 12 MB=0x5fa07dd43ab7c 00000400:00000200:28.0:1616513069.734682:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 632/632 into md 0x865b9f31 [1] + 34920 00000400:00000200:28.0:1616513069.734687:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513069.734690:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513069.734697:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513069.734700:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513069.734702:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.734704:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.734707:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467900 00010000:00000200:16.0:1616513069.734761:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb88900 x1682286567467900/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 632/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.734772:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467900, offset 192 00000400:00000200:16.0:1616513069.734776:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.734782:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.734794:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.734800:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.734804:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.734807:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.734809:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.734812:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.734817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.734871:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.734878:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.734881:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.734887:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.734891:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42660 00000800:00000200:22.0:1616513069.734895:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.734898:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.736407:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513069.736465:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.736474:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513069.736475:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513069.736483:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.736491:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 632 into portal 12 MB=0x5fa07dd43ab84 00000400:00000200:20.0:1616513069.736498:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 632/632 into md 0x865b9f31 [1] + 35552 00000400:00000200:20.0:1616513069.736503:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.736505:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513069.736513:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.736517:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.736520:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.736521:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467908 00010000:00000200:16.0:1616513069.736575:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceb180 x1682286567467908/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 632/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.736585:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467908, offset 192 00000400:00000200:16.0:1616513069.736590:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.736595:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.736608:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.736613:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.736617:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.736619:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.736621:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.736624:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.736630:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513069.736683:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513069.736689:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513069.736692:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513069.736698:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513069.736702:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42e58 00000800:00000200:28.0:1616513069.736706:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.736709:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.738403:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.738414:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.738419:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.738425:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.738431:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 640 into portal 12 MB=0x5fa07dd43ab8c 00000400:00000200:8.0:1616513069.738443:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 640/640 into md 0x865b9f31 [1] + 36184 00000400:00000200:8.0:1616513069.738447:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.738449:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.738455:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.738457:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.738458:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.738460:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.738464:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467916 00010000:00000200:16.0:1616513069.738517:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9ba80 x1682286567467916/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 640/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.738527:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467916, offset 192 00000400:00000200:16.0:1616513069.738531:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.738536:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000100:00000200:30.0:1616513069.738538:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923584, portal 4 00000100:00000200:30.0:1616513069.738543:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923584, offset 0 00000400:00000200:30.0:1616513069.738549:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:16.0:1616513069.738549:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513069.738555:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:16.0:1616513069.738555:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.738559:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.738561:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.738562:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.738565:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513069.738567:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513069.738571:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.738572:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513069.738581:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513069.738584:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513069.738585:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513069.738589:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:30.0:1616513069.738597:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513069.738601:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513069.738603:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513069.738608:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513069.738611:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42bb0 00000800:00000200:30.0:1616513069.738613:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513069.738615:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:5.2:1616513069.738637:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513069.738696:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513069.738702:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513069.738705:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513069.738712:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513069.738718:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d2400 x1693067725923584/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513069.738730:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef760 00000800:00000200:28.0:1616513069.738734:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513069.738737:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513069.738779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513069.738787:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513069.738793:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513069.738797:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:8.0:1616513069.738798:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513069.738803:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626500 00000400:00000200:8.0:1616513069.738808:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d2871 [1] + 192 00000400:00000200:8.0:1616513069.738811:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.738815:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d2400 x1693067725923584/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513069.738827:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513069.738830:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513069.738831:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513069.738832:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d2400 x1693067725923584/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513069.738843:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d2400 x1693067725923584/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513069.738851:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee660 00000800:00000200:11.2:1616513069.740225:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513069.740282:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513069.740291:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513069.740299:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:28.0:1616513069.740308:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 632 into portal 12 MB=0x5fa07dd43ab94 00000400:00000200:28.0:1616513069.740314:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 632/632 into md 0x865b9f31 [1] + 36824 00000400:00000200:28.0:1616513069.740321:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513069.740324:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513069.740331:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.740333:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.740334:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513069.740336:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.740339:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467924 00010000:00000200:16.0:1616513069.740392:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8c800 x1682286567467924/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 632/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.740411:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467924, offset 192 00000400:00000200:16.0:1616513069.740416:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.740421:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.740433:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.740444:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.740448:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.740450:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.740452:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.740455:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.740460:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.740513:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.740521:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513069.740523:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513069.740523:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513069.740530:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513069.740534:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43870 00000800:00000200:20.0:1616513069.740538:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.741841:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.741896:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.741905:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.741914:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.741923:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 640 into portal 12 MB=0x5fa07dd43ab9c 00000400:00000200:12.0:1616513069.741929:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 640/640 into md 0x865b9f31 [1] + 37456 00000400:00000200:12.0:1616513069.741934:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.741937:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513069.741944:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.741947:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.741949:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.741951:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.741952:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467932 00010000:00000200:16.0:1616513069.742005:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa42880 x1682286567467932/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 640/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.742014:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467932, offset 192 00000400:00000200:16.0:1616513069.742019:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.742024:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.742036:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.742042:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.742046:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.742048:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.742049:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.742052:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.742059:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.742113:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.742120:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.742122:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.742129:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.742133:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43b18 00000800:00000200:22.0:1616513069.742137:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.742142:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.742148:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513069.742149:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.742150:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.742152:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.742154:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.743724:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.743735:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.743740:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.743747:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.743752:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 640 into portal 12 MB=0x5fa07dd43aba4 00000400:00000200:8.0:1616513069.743757:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 640/640 into md 0x865b9f31 [1] + 38096 00000400:00000200:8.0:1616513069.743761:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.743763:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.743767:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.743769:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.743770:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.743773:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.743777:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467940 00010000:00000200:16.0:1616513069.743829:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b99f80 x1682286567467940/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 640/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.743839:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467940, offset 192 00000400:00000200:16.0:1616513069.743843:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.743848:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.743861:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.743867:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.743871:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.743873:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.743875:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.743877:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.743883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.743936:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.743943:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.743946:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.743952:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.743956:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43430 00000800:00000200:12.0:1616513069.743960:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.743963:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.745618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513069.745629:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.745634:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513069.745640:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:8.0:1616513069.745646:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 640 into portal 12 MB=0x5fa07dd43abac 00000400:00000200:8.0:1616513069.745650:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 640/640 into md 0x865b9f31 [1] + 38736 00000400:00000200:8.0:1616513069.745654:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.745655:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.745660:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.745661:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.745663:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.745665:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.745669:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467948 00010000:00000200:16.0:1616513069.745730:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9d580 x1682286567467948/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 640/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.745739:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467948, offset 192 00000400:00000200:16.0:1616513069.745744:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.745748:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.745759:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.745764:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.745768:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.745770:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.745771:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.745774:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.745779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.745832:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.745839:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.745841:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513069.745842:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513069.745849:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.745853:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:14.0:1616513069.745856:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.747473:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.747542:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.747551:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.747560:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.747568:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 632 into portal 12 MB=0x5fa07dd43abb4 00000400:00000200:12.0:1616513069.747574:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 632/632 into md 0x865b9f31 [1] + 39376 00000400:00000200:12.0:1616513069.747579:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.747581:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513069.747587:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.747590:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.747592:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.747595:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.747596:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467956 00010000:00000200:16.0:1616513069.747645:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa40900 x1682286567467956/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 632/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.747654:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467956, offset 192 00000400:00000200:16.0:1616513069.747658:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.747663:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.747675:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.747680:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513069.747683:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.747685:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513069.747687:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513069.747689:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.747695:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513069.747703:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.747707:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.747709:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.747713:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513069.747716:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513069.747716:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a433a8 00000800:00000200:8.0:1616513069.747719:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.762879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.762936:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.762945:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.762954:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.762963:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 632 into portal 12 MB=0x5fa07dd43abbc 00000400:00000200:12.0:1616513069.762969:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 632/632 into md 0x865b9f31 [1] + 40008 00000400:00000200:12.0:1616513069.762974:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.762977:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513069.762984:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.762986:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.762988:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.762991:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.763046:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567467964 00010000:00000200:16.0:1616513069.763141:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa41680 x1682286567467964/t0(0) o101->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:695/0 lens 632/624 e 0 to 0 dl 1616513075 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.763156:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 592 bytes to portal 10, xid 1682286567467964, offset 192 00000400:00000200:16.0:1616513069.763163:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:16.0:1616513069.763170:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:16.0:1616513069.763185:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.763191:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 592 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.763198:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.763200:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.763203:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.763207:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.763215:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.763275:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.763282:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.763284:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.763291:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.763295:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42f68 00000800:00000200:22.0:1616513069.763299:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513069.763302:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.766828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513069.766882:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.766891:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513069.766892:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513069.766898:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.50@tcp1 : PUT - for me 00000400:00000200:20.0:1616513069.766907:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.50@tcp1 of length 224 into portal 26 MB=0x5ffe1a1d9f800 00000400:00000200:20.0:1616513069.766913:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.50@tcp1 of length 224/224 into md 0xf97c3af9 [1] + 0 00000400:00000200:20.0:1616513069.766917:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ee58 00000400:00000200:20.0:1616513069.766920:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.766922:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513069.766927:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ee58 00000800:00000200:20.0:1616513069.766930:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.766932:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.766934:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.766981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513069.766991:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513069.766992:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688719431694336 00000800:00000200:30.0:1616513069.766997:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00010000:00000200:24.0:1616513069.767003:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688719431694336/t0(0) o400->ed064001-7e74-69f4-5d5a-e3e82a52b047@10.31.140.50@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:30.0:1616513069.767005:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.50@tcp1 : PUT - for me 00000400:00000200:30.0:1616513069.767012:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.50@tcp1 of length 224 into portal 12 MB=0x5ffe1a1d9f8c0 00000400:00000200:30.0:1616513069.767018:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.50@tcp1 of length 224/224 into md 0xf96450b9 [1] + 6408 00000100:00000200:24.0:1616513069.767019:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688719431694336, offset 192 00000400:00000200:30.0:1616513069.767023:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513069.767025:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513069.767026:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.50@tcp1 00000800:00000200:30.0:1616513069.767033:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513069.767033:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.50@tcp1 routed destination 00000800:00000200:30.0:1616513069.767035:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.767036:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513069.767039:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513069.767048:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.50@tcp1(10.31.140.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.767054:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.767060:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.767063:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.767065:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.767070:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.767075:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.767076:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513069.767101:0:188423:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688719431694528 00010000:00000200:18.0:1616513069.767113:0:188423:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cd9489b00 x1688719431694528/t0(0) o400->695cea4f-4190-8682-af51-e29d09b4e3db@10.31.140.50@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513069.767126:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:18.0:1616513069.767128:0:188423:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688719431694528, offset 192 00000800:00000200:12.0:1616513069.767132:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513069.767134:0:188423:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.50@tcp1 00000400:00000200:12.0:1616513069.767134:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513069.767140:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:18.0:1616513069.767141:0:188423:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.50@tcp1 routed destination 00000400:00000200:12.0:1616513069.767145:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:12.0:1616513069.767149:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513069.767151:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513069.767156:0:188423:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.50@tcp1(10.31.140.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513069.767162:0:188423:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513069.767168:0:188423:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513069.767171:0:188423:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513069.767173:0:188423:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.767180:0:188423:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.767185:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513069.767238:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513069.767245:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.767248:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513069.767250:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513069.767254:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.767258:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc000 00000800:00000200:22.0:1616513069.767262:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.771766:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513069.771776:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.771781:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513069.771787:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.14@tcp1 : PUT - for me 00000400:00000200:30.0:1616513069.771794:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.14@tcp1 of length 224 into portal 26 MB=0x604a467e745c0 00000400:00000200:30.0:1616513069.771798:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.14@tcp1 of length 224/224 into md 0xf97c3b39 [1] + 0 00000400:00000200:30.0:1616513069.771802:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777edd0 00000400:00000200:30.0:1616513069.771804:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513069.771806:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513069.771809:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777edd0 00000800:00000200:30.0:1616513069.771811:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513069.771813:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.771815:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513069.771817:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.771871:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954024621504 00010000:00000200:24.0:1616513069.771882:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1693954024621504/t0(0) o400->4988e163-0783-6f8e-a522-58b19502bd4a@10.31.141.14@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.771896:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954024621504, offset 192 00000400:00000200:24.0:1616513069.771903:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.14@tcp1 00000400:00000200:24.0:1616513069.771916:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.14@tcp1 routed destination 00000400:00000200:24.0:1616513069.771931:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.14@tcp1(10.31.141.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.771938:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.771943:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.771946:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.771948:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.771953:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.771958:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.771959:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513069.771968:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.771973:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513069.771976:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.771981:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.771986:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:8.0:1616513069.771988:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.771991:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513069.772529:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513069.772582:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513069.772591:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513069.772600:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.14@tcp1 : PUT - for me 00000400:00000200:18.0:1616513069.772608:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.14@tcp1 of length 224 into portal 12 MB=0x604a467e74940 00000400:00000200:18.0:1616513069.772615:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.14@tcp1 of length 224/224 into md 0xf96450b9 [1] + 6632 00000400:00000200:18.0:1616513069.772620:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.772623:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513069.772631:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513069.772634:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513069.772636:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513069.772640:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513069.772694:0:188423:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954024622400 00010000:00000200:26.0:1616513069.772706:0:188423:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d7500 x1693954024622400/t0(0) o400->52d8abd3-076d-bc4f-d5cd-0dc064421529@10.31.141.14@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513069.772721:0:188423:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954024622400, offset 192 00000400:00000200:26.0:1616513069.772727:0:188423:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.14@tcp1 00000400:00000200:26.0:1616513069.772734:0:188423:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.14@tcp1 routed destination 00000400:00000200:26.0:1616513069.772749:0:188423:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.14@tcp1(10.31.141.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513069.772755:0:188423:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513069.772761:0:188423:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513069.772763:0:188423:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513069.772766:0:188423:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513069.772770:0:188423:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.772776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.772784:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513069.772788:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513069.772789:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513069.772793:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513069.772794:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513069.772796:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae000 00000800:00000200:30.0:1616513069.772798:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.773315:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.773370:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.773378:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.773387:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.196@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.773395:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.196@tcp1 of length 224 into portal 26 MB=0x605a072d0f880 00000400:00000200:12.0:1616513069.773402:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.196@tcp1 of length 224/224 into md 0xf97c3b69 [1] + 0 00000400:00000200:12.0:1616513069.773406:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777eb28 00000400:00000200:12.0:1616513069.773411:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.773413:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513069.773417:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eb28 00000800:00000200:12.0:1616513069.773420:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.773423:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513069.773424:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513069.773431:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.773448:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.773454:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513069.773462:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.196@tcp1 : PUT - for me 00000400:00000200:28.0:1616513069.773471:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.196@tcp1 of length 224 into portal 12 MB=0x605a072d0f940 00000400:00000200:28.0:1616513069.773478:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.196@tcp1 of length 224/224 into md 0x865b9f31 [1] + 40640 00000100:00000200:24.0:1616513069.773480:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036539467904 00000400:00000200:28.0:1616513069.773483:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513069.773486:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:24.0:1616513069.773490:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1695036539467904/t0(0) o400->8543633b-3c69-12ab-65b8-db4ef9a67807@10.31.129.196@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513069.773493:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513069.773496:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.773498:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.773504:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036539467904, offset 192 00000400:00000200:24.0:1616513069.773511:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.196@tcp1 00000400:00000200:24.0:1616513069.773518:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.196@tcp1 routed destination 00000400:00000200:24.0:1616513069.773535:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.196@tcp1(10.31.129.196@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.773541:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.773547:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.773550:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.773552:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513069.773555:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036539468096 00000800:00000200:24.0:1616513069.773557:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513069.773562:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.773563:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:16.0:1616513069.773566:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8da00 x1695036539468096/t0(0) o400->cf20ed29-f5c3-28cb-6c41-4fc006f4b2a1@10.31.129.196@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.773579:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036539468096, offset 192 00000400:00000200:16.0:1616513069.773585:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.196@tcp1 00000400:00000200:16.0:1616513069.773592:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.196@tcp1 routed destination 00000400:00000200:16.0:1616513069.773606:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.196@tcp1(10.31.129.196@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.773612:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513069.773615:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513069.773618:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.773620:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513069.773622:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513069.773622:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513069.773625:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513069.773625:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513069.773627:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.773631:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513069.773632:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513069.773637:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2b28 00000800:00000200:22.0:1616513069.773641:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513069.773641:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513069.773647:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513069.773649:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513069.773651:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513069.773656:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.773660:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42c38 00000800:00000200:22.0:1616513069.773664:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513069.805155:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.805211:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.805220:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.805228:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.180@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.805238:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.180@tcp1 of length 224 into portal 26 MB=0x6033cd6dbc3c0 00000400:00000200:12.0:1616513069.805245:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.180@tcp1 of length 224/224 into md 0xf97c3b89 [1] + 0 00000400:00000200:12.0:1616513069.805249:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ea18 00000400:00000200:12.0:1616513069.805252:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.805255:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513069.805259:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ea18 00000800:00000200:12.0:1616513069.805262:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513069.805264:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.805264:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.805271:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.805274:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.805275:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513069.805284:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.180@tcp1 : PUT - for me 00000400:00000200:28.0:1616513069.805294:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.180@tcp1 of length 224 into portal 12 MB=0x6033cd6dbc440 00000400:00000200:28.0:1616513069.805300:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.180@tcp1 of length 224/224 into md 0x865b9f31 [1] + 40864 00000400:00000200:28.0:1616513069.805305:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513069.805308:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513069.805315:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513069.805318:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.805320:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.805324:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692409697911744 00010000:00000200:24.0:1616513069.805336:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1692409697911744/t0(0) o400->d55efd6e-6d7a-0062-cc73-93ac23db6d34@10.31.138.180@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.805350:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692409697911744, offset 192 00000400:00000200:24.0:1616513069.805357:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.180@tcp1 00000400:00000200:24.0:1616513069.805365:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.180@tcp1 routed destination 00000100:00000200:16.0:1616513069.805378:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692409697911872 00000400:00000200:24.0:1616513069.805379:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.180@tcp1(10.31.138.180@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.805386:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:16.0:1616513069.805391:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8f500 x1692409697911872/t0(0) o400->336a528a-ec60-74d3-c8d0-3e0b26b4fad3@10.31.138.180@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513069.805392:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.805395:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.805397:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.805402:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513069.805404:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692409697911872, offset 192 00000800:00000200:11.2:1616513069.805407:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.805409:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513069.805410:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.180@tcp1 00000400:00000200:16.0:1616513069.805418:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.180@tcp1 routed destination 00000400:00000200:16.0:1616513069.805447:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.180@tcp1(10.31.138.180@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.805453:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.805460:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513069.805461:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513069.805462:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.805464:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513069.805468:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513069.805469:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513069.805471:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513069.805471:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513069.805477:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513069.805478:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513069.805483:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000800:00000200:20.0:1616513069.805487:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513069.805533:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513069.805549:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513069.805551:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513069.805553:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513069.805558:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513069.805562:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42110 00000800:00000200:14.0:1616513069.805566:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:3.0:1616513069.850539:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923600, portal 4 00000100:00000200:3.0:1616513069.850545:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923600, offset 0 00000400:00000200:3.0:1616513069.850552:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:3.0:1616513069.850560:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:3.0:1616513069.850574:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513069.850581:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:3.0:1616513069.850590:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513069.850593:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:3.0:1616513069.850596:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:3.0:1616513069.850602:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513069.850649:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513069.850669:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513069.850677:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513069.850680:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513069.850686:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513069.850693:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8f30a89200 x1693067725923600/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513069.850705:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2990 00000800:00000200:18.0:1616513069.850709:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513069.850712:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513069.850821:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513069.850833:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513069.850842:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:30.0:1616513069.850850:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513069.850859:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626510 00000400:00000200:30.0:1616513069.850866:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c4259 [1] + 192 00000400:00000200:30.0:1616513069.850871:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513069.850876:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8f30a89200 x1693067725923600/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513069.850886:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:30.0:1616513069.850890:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513069.850893:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:30.0:1616513069.850895:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:3.0:1616513069.850901:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8f30a89200 x1693067725923600/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513069.850917:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8f30a89200 x1693067725923600/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513069.850928:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:25.2:1616513069.853836:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513069.853893:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513069.853902:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513069.853911:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513069.853918:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 26 MB=0x5f9313e1f97c0 00000400:00000200:18.0:1616513069.853925:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf97c3b99 [1] + 0 00000400:00000200:18.0:1616513069.853930:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e000 00000400:00000200:18.0:1616513069.853932:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.853935:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513069.853940:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e000 00000800:00000200:18.0:1616513069.853943:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513069.853947:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513069.853948:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:18.0:1616513069.853954:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (23)-- 00000800:00000200:18.0:1616513069.853956:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513069.853958:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513069.853962:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513069.853966:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513069.853969:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513069.853976:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f98c0 00000400:00000200:18.0:1616513069.853976:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f97e0 00000400:00000200:18.0:1616513069.853981:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 6856 00000400:00000200:18.0:1616513069.853985:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.853988:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 7080 00000100:00000200:18.0:1616513069.853993:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:22.0:1616513069.853994:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.854000:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513069.854000:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (23)++ 00000800:00000200:18.0:1616513069.854002:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (24)-- 00000800:00000200:18.0:1616513069.854003:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (23)-- 00000800:00000200:18.0:1616513069.854005:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000100:00000200:24.0:1616513069.854008:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774524864 00000800:00000200:22.0:1616513069.854009:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (23)++ 00000800:00000200:18.0:1616513069.854009:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513069.854013:0:6472:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (24)++ 00000400:00000200:18.0:1616513069.854016:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:30.0:1616513069.854020:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000800:00000200:22.0:1616513069.854020:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (25)-- 00010000:00000200:24.0:1616513069.854021:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1681364774524864/t0(0) o400->1ae63833-db83-47b2-ef1b-80ab28744988@10.242.46.229@o2ib1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513069.854021:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9820 00000800:00000200:22.0:1616513069.854024:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (25)-- 00000800:00000200:30.0:1616513069.854026:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513069.854026:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 7304 00000800:00000200:14.0:1616513069.854026:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (24)++ 00000400:00000200:18.0:1616513069.854029:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.854031:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:30.0:1616513069.854033:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000100:00000200:24.0:1616513069.854035:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681364774524864, offset 192 00000800:00000200:14.0:1616513069.854038:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513069.854039:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (25)++ 00000400:00000200:30.0:1616513069.854040:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9800 00000800:00000200:22.0:1616513069.854040:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000400:00000200:24.0:1616513069.854041:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513069.854041:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (26)-- 00000800:00000200:18.0:1616513069.854042:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000800:00000200:22.0:1616513069.854044:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:30.0:1616513069.854045:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 7528 00000800:00000200:18.0:1616513069.854045:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000800:00000200:18.0:1616513069.854047:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (26)-- 00000400:00000200:14.0:1616513069.854047:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:24.0:1616513069.854049:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:18.0:1616513069.854049:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (25)-- 00000400:00000200:30.0:1616513069.854050:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513069.854051:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:18.0:1616513069.854051:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (24)++ 00000100:00000200:30.0:1616513069.854052:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513069.854055:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513069.854055:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f99c0 00000400:00000200:24.0:1616513069.854063:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513069.854065:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 7752 00000800:00000200:30.0:1616513069.854069:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (25)++ 00000800:00000200:24.0:1616513069.854069:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:14.0:1616513069.854070:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513069.854072:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (26)-- 00000400:00000200:22.0:1616513069.854072:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9880 00000100:00000200:14.0:1616513069.854072:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513069.854073:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:30.0:1616513069.854074:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (25)-- 00000800:00000200:24.0:1616513069.854075:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513069.854076:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (24)++ 00000400:00000200:18.0:1616513069.854078:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f98a0 00000800:00000200:30.0:1616513069.854079:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:24.0:1616513069.854080:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (25)++ 00000800:00000200:24.0:1616513069.854082:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (26)++ 00000800:00000200:14.0:1616513069.854083:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (26)++ 00000400:00000200:22.0:1616513069.854084:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 7976 00000400:00000200:30.0:1616513069.854087:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:24.0:1616513069.854087:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (28)-- 00000400:00000200:22.0:1616513069.854088:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513069.854088:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (28)-- 00000800:00000200:14.0:1616513069.854090:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000400:00000200:18.0:1616513069.854091:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 8200 00000400:00000200:30.0:1616513069.854092:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9840 00000800:00000200:14.0:1616513069.854092:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000100:00000200:22.0:1616513069.854094:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513069.854094:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513069.854095:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513069.854096:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513069.854103:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:22.0:1616513069.854107:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (26)++ 00000400:00000200:30.0:1616513069.854109:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 8424 00000800:00000200:22.0:1616513069.854109:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (27)-- 00000400:00000200:14.0:1616513069.854109:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f98e0 00000100:00000200:26.0:1616513069.854111:0:188423:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774524896 00000800:00000200:22.0:1616513069.854111:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000400:00000200:30.0:1616513069.854112:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513069.854112:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513069.854113:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000800:00000200:22.0:1616513069.854118:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513069.854122:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (26)++ 00010000:00000200:26.0:1616513069.854124:0:188423:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d3180 x1681364774524896/t0(0) o400->boslfs02-MDT0001-lwp-OST0018_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:22.0:1616513069.854124:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:18.0:1616513069.854124:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (27)-- 00000400:00000200:14.0:1616513069.854124:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 8648 00000800:00000200:18.0:1616513069.854125:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000800:00000200:18.0:1616513069.854127:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000100:00000200:30.0:1616513069.854128:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513069.854128:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513069.854129:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (26)-- 00000100:00000200:6.0:1616513069.854129:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525120 00000400:00000200:18.0:1616513069.854131:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513069.854136:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00010000:00000200:6.0:1616513069.854138:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad6780 x1681364774525120/t0(0) o400->boslfs02-MDT0001-lwp-OST0000_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:18.0:1616513069.854140:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:18.0:1616513069.854143:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (25)-- 00000800:00000200:18.0:1616513069.854145:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (24)++ 00000800:00000200:18.0:1616513069.854148:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:30.0:1616513069.854149:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (25)++ 00000100:00000200:14.0:1616513069.854149:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513069.854152:0:6469:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (26)++ 00000100:00000200:2.0:1616513069.854152:0:188413:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774524960 00000100:00000200:22.0:1616513069.854154:0:188418:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774524928 00000800:00000200:30.0:1616513069.854155:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (27)-- 00000400:00000200:18.0:1616513069.854155:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:30.0:1616513069.854156:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000800:00000200:30.0:1616513069.854158:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000800:00000200:30.0:1616513069.854161:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000100:00000200:26.0:1616513069.854164:0:188423:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774524896, offset 192 00010000:00000200:22.0:1616513069.854164:0:188418:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cd948f980 x1681364774524928/t0(0) o400->boslfs02-MDT0001-lwp-OST000c_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513069.854164:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (26)++ 00010000:00000200:2.0:1616513069.854164:0:188413:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d1200 x1681364774524960/t0(0) o400->boslfs02-MDT0001-lwp-OST0014_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513069.854165:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525120, offset 192 00000800:00000200:14.0:1616513069.854166:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (27)-- 00000400:00000200:30.0:1616513069.854167:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000100:00000200:18.0:1616513069.854168:0:170088:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525056 00000800:00000200:14.0:1616513069.854168:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000800:00000200:14.0:1616513069.854170:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000400:00000200:30.0:1616513069.854171:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9900 00000400:00000200:26.0:1616513069.854171:0:188423:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513069.854172:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (26)-- 00000800:00000200:14.0:1616513069.854175:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (25)-- 00000800:00000200:14.0:1616513069.854176:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (24)++ 00000100:00000200:10.0:1616513069.854176:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525376 00000400:00000200:6.0:1616513069.854177:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00010000:00000200:18.0:1616513069.854178:0:170088:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad3a80 x1681364774525056/t0(0) o400->boslfs02-MDT0001-lwp-OST0008_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:26.0:1616513069.854179:0:188423:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:30.0:1616513069.854180:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 8872 00000800:00000200:14.0:1616513069.854180:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:30.0:1616513069.854184:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513069.854184:0:188418:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774524928, offset 192 00000100:00000200:30.0:1616513069.854185:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:2.0:1616513069.854185:0:188413:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774524960, offset 192 00010000:00000200:10.0:1616513069.854187:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa0000 x1681364774525376/t0(0) o400->boslfs02-MDT0001-lwp-OST0028_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:26.0:1616513069.854193:0:188423:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513069.854196:0:188418:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:26.0:1616513069.854199:0:188423:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513069.854200:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:2.0:1616513069.854201:0:188413:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000100:00000200:18.0:1616513069.854203:0:170088:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525056, offset 192 00000100:00000200:10.0:1616513069.854204:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525376, offset 192 00000400:00000200:18.0:1616513069.854209:0:170088:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513069.854210:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513069.854214:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (25)++ 00000400:00000200:10.0:1616513069.854214:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513069.854216:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (26)-- 00000800:00000200:6.0:1616513069.854216:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513069.854217:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (25)-- 00000400:00000200:22.0:1616513069.854218:0:188418:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:30.0:1616513069.854219:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (24)++ 00000400:00000200:14.0:1616513069.854219:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:30.0:1616513069.854222:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513069.854231:0:188418:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513069.854237:0:188418:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:26.0:1616513069.854238:0:188423:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513069.854240:0:188413:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:26.0:1616513069.854241:0:188423:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (25)++ 00000800:00000200:26.0:1616513069.854243:0:188423:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (26)++ 00000800:00000200:26.0:1616513069.854248:0:188423:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (27)-- 00000400:00000200:2.0:1616513069.854253:0:188413:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513069.854260:0:188413:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:18.0:1616513069.854261:0:170088:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000100:00000200:26.0:1616513069.854268:0:188423:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525088 00010000:00000200:26.0:1616513069.854274:0:188423:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d3a80 x1681364774525088/t0(0) o400->boslfs02-MDT0001-lwp-OST0004_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:18.0:1616513069.854275:0:170088:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513069.854281:0:170088:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:10.0:1616513069.854283:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000100:00000200:26.0:1616513069.854284:0:188423:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525088, offset 192 00000400:00000200:26.0:1616513069.854290:0:188423:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:10.0:1616513069.854297:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513069.854303:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:6.0:1616513069.854303:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513069.854305:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:22.0:1616513069.854305:0:188418:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513069.854305:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (26)++ 00000400:00000200:26.0:1616513069.854306:0:188423:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:2.0:1616513069.854306:0:188413:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513069.854307:0:188418:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (27)++ 00000800:00000200:18.0:1616513069.854307:0:170088:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513069.854307:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (27)++ 00000800:00000200:18.0:1616513069.854309:0:170088:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (29)++ 00000800:00000200:2.0:1616513069.854309:0:188413:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (29)++ 00000800:00000200:6.0:1616513069.854310:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (29)-- 00000800:00000200:22.0:1616513069.854311:0:188418:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (30)++ 00000800:00000200:22.0:1616513069.854314:0:188418:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)-- 00000800:00000200:18.0:1616513069.854314:0:170088:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (31)++ 00000100:00000200:6.0:1616513069.854317:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774524992 00000800:00000200:18.0:1616513069.854318:0:170088:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)-- 00000800:00000200:2.0:1616513069.854318:0:188413:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (31)++ 00000400:00000200:26.0:1616513069.854319:0:188423:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00010000:00000200:6.0:1616513069.854321:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cd9488d80 x1681364774524992/t0(0) o400->boslfs02-MDT0001-lwp-OST0010_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513069.854323:0:188418:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525152 00000800:00000200:2.0:1616513069.854323:0:188413:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)-- 00000400:00000200:30.0:1616513069.854325:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9940 00000800:00000200:26.0:1616513069.854325:0:188423:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:14.0:1616513069.854325:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9980 00000800:00000200:10.0:1616513069.854326:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000100:00000200:18.0:1616513069.854328:0:170088:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525184 00000400:00000200:30.0:1616513069.854329:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 9096 00000800:00000200:26.0:1616513069.854329:0:188423:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513069.854329:0:188418:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5a00 x1681364774525152/t0(0) o400->boslfs02-MDT0001-lwp-OST001c_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:10.0:1616513069.854329:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (30)++ 00000800:00000200:10.0:1616513069.854331:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (31)++ 00000400:00000200:30.0:1616513069.854332:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513069.854332:0:188423:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)++ 00000100:00000200:6.0:1616513069.854332:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774524992, offset 192 00010000:00000200:18.0:1616513069.854334:0:170088:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b278480 x1681364774525184/t0(0) o400->boslfs02-MDT0001-lwp-OST0024_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513069.854335:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 9320 00000800:00000200:26.0:1616513069.854336:0:188423:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (33)++ 00000800:00000200:10.0:1616513069.854336:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (33)-- 00000400:00000200:14.0:1616513069.854338:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513069.854339:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513069.854339:0:188423:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (33)-- 00000800:00000200:30.0:1616513069.854348:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (32)++ 00000800:00000200:30.0:1616513069.854349:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (33)-- 00000400:00000200:6.0:1616513069.854349:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513069.854351:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (32)-- 00000100:00000200:22.0:1616513069.854351:0:188418:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525152, offset 192 00000100:00000200:14.0:1616513069.854351:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513069.854352:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (31)++ 00000400:00000200:6.0:1616513069.854353:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:30.0:1616513069.854355:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000100:00000200:18.0:1616513069.854359:0:170088:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525184, offset 192 00000400:00000200:22.0:1616513069.854361:0:188418:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000100:00000200:2.0:1616513069.854361:0:188410:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525248 00000400:00000200:6.0:1616513069.854362:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513069.854365:0:170088:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513069.854365:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (32)++ 00000800:00000200:6.0:1616513069.854366:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00010000:00000200:2.0:1616513069.854366:0:188410:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b27b180 x1681364774525248/t0(0) o400->boslfs02-MDT0001-lwp-OST0038_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513069.854367:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (33)-- 00000100:00000200:10.0:1616513069.854367:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525312 00000400:00000200:30.0:1616513069.854368:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513069.854369:0:188418:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:14.0:1616513069.854369:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (32)-- 00000800:00000200:14.0:1616513069.854371:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (31)++ 00010000:00000200:10.0:1616513069.854372:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa6c00 x1681364774525312/t0(0) o400->boslfs02-MDT0001-lwp-OST0020_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513069.854374:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513069.854376:0:170088:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000100:00000200:2.0:1616513069.854378:0:188410:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525248, offset 192 00000800:00000200:6.0:1616513069.854381:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513069.854383:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)++ 00000400:00000200:30.0:1616513069.854384:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f99a0 00000800:00000200:6.0:1616513069.854384:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (33)++ 00000400:00000200:2.0:1616513069.854384:0:188410:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513069.854385:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9860 00000100:00000200:10.0:1616513069.854385:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525312, offset 192 00000400:00000200:14.0:1616513069.854386:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:6.0:1616513069.854386:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (34)-- 00000400:00000200:30.0:1616513069.854388:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 9544 00000400:00000200:2.0:1616513069.854389:0:188410:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:30.0:1616513069.854391:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:10.0:1616513069.854393:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513069.854397:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 9768 00000400:00000200:18.0:1616513069.854397:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9920 00000400:00000200:14.0:1616513069.854397:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9960 00000400:00000200:10.0:1616513069.854399:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000100:00000200:30.0:1616513069.854401:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:22.0:1616513069.854401:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513069.854406:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (33)++ 00000800:00000200:30.0:1616513069.854408:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (34)-- 00000400:00000200:14.0:1616513069.854408:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 9992 00000800:00000200:30.0:1616513069.854409:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (33)-- 00000800:00000200:30.0:1616513069.854411:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (32)++ 00000400:00000200:14.0:1616513069.854412:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513069.854413:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (33)-- 00000400:00000200:18.0:1616513069.854414:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 10216 00000400:00000200:30.0:1616513069.854415:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:26.0:1616513069.854416:0:188410:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525344 00000100:00000200:22.0:1616513069.854418:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513069.854418:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513069.854419:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00010000:00000200:26.0:1616513069.854422:0:188410:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b27de80 x1681364774525344/t0(0) o400->boslfs02-MDT0001-lwp-OST002c_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:22.0:1616513069.854447:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (32)++ 00000100:00000200:14.0:1616513069.854447:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:2.0:1616513069.854449:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525024 00000800:00000200:22.0:1616513069.854450:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (33)-- 00000800:00000200:22.0:1616513069.854451:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (32)-- 00000100:00000200:18.0:1616513069.854453:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513069.854454:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (31)++ 00000800:00000200:14.0:1616513069.854454:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (31)++ 00010000:00000200:2.0:1616513069.854454:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad1200 x1681364774525024/t0(0) o400->boslfs02-MDT0001-lwp-OST003c_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:22.0:1616513069.854456:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (33)-- 00000800:00000200:14.0:1616513069.854456:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (33)-- 00000400:00000200:22.0:1616513069.854458:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513069.854458:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (31)-- 00000800:00000200:14.0:1616513069.854460:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (30)++ 00000400:00000200:30.0:1616513069.854462:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae5d8 00000800:00000200:14.0:1616513069.854462:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (31)-- 00000400:00000200:22.0:1616513069.854463:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:18.0:1616513069.854463:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (31)++ 00000400:00000200:14.0:1616513069.854463:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:6.0:1616513069.854463:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525280 00000100:00000200:26.0:1616513069.854465:0:188410:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525344, offset 192 00000100:00000200:2.0:1616513069.854465:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525024, offset 192 00000400:00000200:22.0:1616513069.854466:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01078 00000800:00000200:18.0:1616513069.854466:0:6471:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (31)++ 00010000:00000200:6.0:1616513069.854467:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa3f00 x1681364774525280/t0(0) o400->boslfs02-MDT0001-lwp-OST0034_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:30.0:1616513069.854468:0:6469:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513069.854469:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (32)-- 00000400:00000200:14.0:1616513069.854469:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:10.0:1616513069.854469:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525216 00000800:00000200:18.0:1616513069.854471:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (31)-- 00000400:00000200:26.0:1616513069.854472:0:188410:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513069.854473:0:6469:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513069.854473:0:6472:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513069.854473:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (30)++ 00000800:00000200:30.0:1616513069.854475:0:6469:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)++ 00000800:00000200:18.0:1616513069.854475:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (31)-- 00000400:00000200:18.0:1616513069.854476:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513069.854476:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513069.854477:0:6469:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (31)++ 00000400:00000200:26.0:1616513069.854477:0:188410:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:14.0:1616513069.854478:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ac38 00000800:00000200:30.0:1616513069.854480:0:6469:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:30.0:1616513069.854481:0:6469:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)-- 00000100:00000200:6.0:1616513069.854481:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525280, offset 192 00000400:00000200:18.0:1616513069.854482:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:22.0:1616513069.854484:0:6472:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00010000:00000200:10.0:1616513069.854484:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d7080 x1681364774525216/t0(0) o400->boslfs02-MDT0001-lwp-OST0030_UUID@10.242.46.229@o2ib1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:6.0:1616513069.854485:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:2.0:1616513069.854485:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:22.0:1616513069.854486:0:6472:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)++ 00000400:00000200:18.0:1616513069.854486:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd6d8 00000800:00000200:22.0:1616513069.854487:0:6472:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (32)++ 00000800:00000200:30.0:1616513069.854490:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (33)-- 00000800:00000200:22.0:1616513069.854491:0:6472:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:30.0:1616513069.854492:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (32)++ 00000800:00000200:22.0:1616513069.854492:0:6472:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)-- 00000800:00000200:14.0:1616513069.854492:0:6470:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513069.854493:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:30.0:1616513069.854495:0:6469:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000100:00000200:10.0:1616513069.854495:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525216, offset 192 00000800:00000200:30.0:1616513069.854496:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (32)-- 00000400:00000200:30.0:1616513069.854497:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513069.854497:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (31)-- 00000800:00000200:18.0:1616513069.854498:0:6471:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:22.0:1616513069.854499:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (30)++ 00000400:00000200:10.0:1616513069.854499:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513069.854501:0:6470:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513069.854502:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:22.0:1616513069.854502:0:6472:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:22.0:1616513069.854503:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (31)-- 00000800:00000200:18.0:1616513069.854503:0:6471:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513069.854503:0:6470:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)++ 00000400:00000200:30.0:1616513069.854504:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec330 00000400:00000200:10.0:1616513069.854504:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:22.0:1616513069.854505:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513069.854505:0:6471:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)++ 00000800:00000200:14.0:1616513069.854505:0:6470:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (31)++ 00000800:00000200:14.0:1616513069.854509:0:6470:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:30.0:1616513069.854510:0:6469:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513069.854510:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:18.0:1616513069.854510:0:6471:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (33)++ 00000800:00000200:14.0:1616513069.854510:0:6470:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (33)-- 00000800:00000200:14.0:1616513069.854512:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (33)-- 00000400:00000200:22.0:1616513069.854513:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82ff0 00000800:00000200:18.0:1616513069.854514:0:6471:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:14.0:1616513069.854514:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (32)++ 00000800:00000200:30.0:1616513069.854515:0:6469:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513069.854515:0:6471:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)-- 00000800:00000200:30.0:1616513069.854517:0:6469:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)++ 00000800:00000200:18.0:1616513069.854517:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (32)-- 00000800:00000200:22.0:1616513069.854518:0:6472:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513069.854518:0:6470:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:30.0:1616513069.854519:0:6469:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (32)++ 00000800:00000200:18.0:1616513069.854519:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (32)++ 00000800:00000200:14.0:1616513069.854519:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (32)-- 00000400:00000200:14.0:1616513069.854520:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513069.854522:0:6469:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:22.0:1616513069.854522:0:6472:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513069.854523:0:6469:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (33)-- 00000800:00000200:18.0:1616513069.854523:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.229@o2ib1 00000800:00000200:30.0:1616513069.854524:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (32)-- 00000800:00000200:22.0:1616513069.854524:0:6472:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)++ 00000400:00000200:14.0:1616513069.854525:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:30.0:1616513069.854527:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (32)++ 00000400:00000200:14.0:1616513069.854528:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeaa0 00000800:00000200:18.0:1616513069.854530:0:6471:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:22.0:1616513069.854531:0:6472:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (33)++ 00000800:00000200:18.0:1616513069.854531:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (33)++ 00000800:00000200:18.0:1616513069.854533:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (35)-- 00000800:00000200:14.0:1616513069.854533:0:6470:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513069.854534:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (34)-- 00000800:00000200:22.0:1616513069.854535:0:6472:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:22.0:1616513069.854535:0:6472:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (33)-- 00000800:00000200:18.0:1616513069.854536:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (33)++ 00000800:00000200:30.0:1616513069.854537:0:6469:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:14.0:1616513069.854537:0:6470:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513069.854538:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (33)-- 00000800:00000200:22.0:1616513069.854538:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (33)-- 00000400:00000200:30.0:1616513069.854539:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513069.854539:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.229@o2ib1 00000800:00000200:14.0:1616513069.854539:0:6470:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)++ 00000800:00000200:22.0:1616513069.854540:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (31)++ 00000800:00000200:14.0:1616513069.854541:0:6470:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (32)++ 00000400:00000200:30.0:1616513069.854544:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:14.0:1616513069.854544:0:6470:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:14.0:1616513069.854545:0:6470:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (34)-- 00000400:00000200:30.0:1616513069.854546:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf006e8 00000800:00000200:14.0:1616513069.854547:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (33)-- 00000800:00000200:18.0:1616513069.854548:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (33)++ 00000800:00000200:22.0:1616513069.854549:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (32)-- 00000800:00000200:14.0:1616513069.854549:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (32)++ 00000800:00000200:30.0:1616513069.854550:0:6469:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513069.854550:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (33)-- 00000800:00000200:22.0:1616513069.854551:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (33)-- 00000800:00000200:14.0:1616513069.854551:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (32)-- 00000800:00000200:18.0:1616513069.854552:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (31)-- 00000800:00000200:30.0:1616513069.854553:0:6469:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513069.854553:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513069.854554:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (29)++ 00000800:00000200:30.0:1616513069.854555:0:6469:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (29)++ 00000800:00000200:30.0:1616513069.854556:0:6469:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (31)++ 00000400:00000200:14.0:1616513069.854558:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:30.0:1616513069.854559:0:6469:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)-- 00000800:00000200:22.0:1616513069.854559:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (32)-- 00000800:00000200:30.0:1616513069.854560:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (31)-- 00000400:00000200:14.0:1616513069.854560:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b650 00000400:00000200:22.0:1616513069.854561:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513069.854562:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (29)-- 00000800:00000200:30.0:1616513069.854564:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (28)++ 00000400:00000200:22.0:1616513069.854566:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.854568:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccff0 00000800:00000200:30.0:1616513069.854569:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (29)-- 00000800:00000200:26.0:1616513069.854570:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (29)++ 00000400:00000200:30.0:1616513069.854571:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513069.854573:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (29)-- 00000800:00000200:22.0:1616513069.854573:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (29)-- 00000400:00000200:30.0:1616513069.854575:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:26.0:1616513069.854575:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513069.854575:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (28)++ 00000400:00000200:30.0:1616513069.854577:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec220 00000400:00000200:26.0:1616513069.854581:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:26.0:1616513069.854584:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e834b8 00000800:00000200:30.0:1616513069.854585:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (28)-- 00000800:00000200:22.0:1616513069.854585:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (28)-- 00000800:00000200:30.0:1616513069.854587:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (27)++ 00000400:00000200:22.0:1616513069.854587:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513069.854589:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (27)-- 00000800:00000200:26.0:1616513069.854589:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (27)-- 00000400:00000200:30.0:1616513069.854590:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513069.854591:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000400:00000200:22.0:1616513069.854592:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:30.0:1616513069.854594:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.854595:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afcb0 00000400:00000200:30.0:1616513069.854596:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eecdd0 00000800:00000200:22.0:1616513069.854602:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000800:00000200:2.0:1616513069.854603:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (26)-- 00000800:00000200:30.0:1616513069.854604:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (25)-- 00000400:00000200:2.0:1616513069.854605:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513069.854606:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (23)++ 00000400:00000200:2.0:1616513069.854611:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:2.0:1616513069.854614:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00d48 00000800:00000200:22.0:1616513069.854619:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (24)-- 00000800:00000200:2.0:1616513069.854619:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (24)-- 00000400:00000200:22.0:1616513069.854620:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513069.854621:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000400:00000200:22.0:1616513069.854625:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:22.0:1616513069.854627:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83d38 00000800:00000200:22.0:1616513069.854632:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (23)-- 00000800:00000200:30.0:1616513069.854634:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513069.854634:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.229@o2ib1 00000800:00000200:26.0:1616513069.854636:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (23)++ 00000800:00000200:26.0:1616513069.854638:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (24)-- 00000800:00000200:2.0:1616513069.854639:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (24)-- 00000800:00000200:30.0:1616513069.854640:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.229@o2ib1 00000800:00000200:30.0:1616513069.854642:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (23)++ 00000800:00000200:30.0:1616513069.854643:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:26.0:1616513069.854643:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (23)-- 00000800:00000200:30.0:1616513069.854646:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:27.0:1616513069.866510:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923616, portal 4 00000100:00000200:27.0:1616513069.866515:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923616, offset 0 00000400:00000200:27.0:1616513069.866521:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:27.0:1616513069.866527:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:27.0:1616513069.866539:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513069.866546:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:27.0:1616513069.866552:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513069.866555:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:27.0:1616513069.866557:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:27.0:1616513069.866562:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513069.866609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513069.866631:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513069.866637:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513069.866640:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513069.866647:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513069.866653:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63c800 x1693067725923616/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513069.866665:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248f68 00000800:00000200:1.0:1616513069.866670:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513069.866673:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513069.866782:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513069.866792:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513069.866798:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513069.866801:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513069.866807:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513069.866815:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626520 00000400:00000200:1.0:1616513069.866821:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff735 [1] + 192 00000400:00000200:1.0:1616513069.866827:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513069.866834:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63c800 x1693067725923616/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513069.866847:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513069.866850:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513069.866852:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:27.0:1616513069.866855:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63c800 x1693067725923616/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513069.866867:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63c800 x1693067725923616/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513069.866876:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249e48 00000800:00000200:7.2:1616513069.869527:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513069.869542:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513069.869550:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513069.869559:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.28@tcp1 : PUT - for me 00000400:00000200:2.0:1616513069.869567:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.28@tcp1 of length 224 into portal 26 MB=0x6048d6db9d300 00000400:00000200:2.0:1616513069.869573:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.28@tcp1 of length 224/224 into md 0xf97c3bc9 [1] + 0 00000400:00000200:2.0:1616513069.869578:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f540 00000400:00000200:2.0:1616513069.869581:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513069.869583:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513069.869587:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f540 00000800:00000200:2.0:1616513069.869590:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513069.869593:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513069.869594:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513069.869601:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513069.869604:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513069.869605:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513069.869614:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.28@tcp1 : PUT - for me 00000400:00000200:26.0:1616513069.869623:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.28@tcp1 of length 224 into portal 12 MB=0x6048d6db9d380 00000400:00000200:26.0:1616513069.869629:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.28@tcp1 of length 224/224 into md 0xf96450b9 [1] + 10440 00000400:00000200:26.0:1616513069.869635:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513069.869637:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513069.869644:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513069.869648:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513069.869649:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.869653:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855338058496 00010000:00000200:24.0:1616513069.869664:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1693855338058496/t0(0) o400->62b7be0c-2292-f95c-1c85-a6d7d8928a48@10.31.141.28@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513069.869666:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855338058624 00000100:00000200:24.0:1616513069.869678:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855338058496, offset 192 00010000:00000200:10.0:1616513069.869678:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c697500 x1693855338058624/t0(0) o400->7e0975bf-208d-e13a-34d8-52560df486df@10.31.141.28@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513069.869685:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.28@tcp1 00000400:00000200:24.0:1616513069.869692:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.28@tcp1 routed destination 00000100:00000200:10.0:1616513069.869692:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855338058624, offset 192 00000400:00000200:10.0:1616513069.869698:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.28@tcp1 00000400:00000200:10.0:1616513069.869706:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.28@tcp1 routed destination 00000400:00000200:24.0:1616513069.869707:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.28@tcp1(10.31.141.28@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.869714:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.869720:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:10.0:1616513069.869721:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.28@tcp1(10.31.141.28@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.869722:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.869724:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513069.869728:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513069.869729:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513069.869734:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:24.0:1616513069.869736:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513069.869736:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513069.869737:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513069.869739:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.869743:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.869748:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.869771:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513069.869777:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513069.869779:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513069.869780:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513069.869785:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513069.869789:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83ba0 00000800:00000200:30.0:1616513069.869792:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513069.869794:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513069.869801:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.869803:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513069.869810:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513069.869810:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513069.869816:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:12.0:1616513069.869819:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:19.0:1616513069.870619:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:19.0:1616513069.870627:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:7.2:1616513069.879101:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513069.879159:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.879168:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513069.879176:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.55@tcp1 : PUT - for me 00000800:00000200:11.2:1616513069.879185:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:10.0:1616513069.879186:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.55@tcp1 of length 224 into portal 26 MB=0x600194aaa9b00 00000400:00000200:10.0:1616513069.879192:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.55@tcp1 of length 224/224 into md 0xf97c3bf9 [1] + 0 00000400:00000200:10.0:1616513069.879196:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fa08 00000800:00000200:8.0:1616513069.879198:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513069.879199:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513069.879201:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513069.879205:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa08 00000800:00000200:8.0:1616513069.879205:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:10.0:1616513069.879208:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513069.879211:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513069.879211:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:8.0:1616513069.879213:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.55@tcp1 : PUT - for me 00000800:00000200:10.0:1616513069.879215:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513069.879222:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.55@tcp1 of length 224 into portal 12 MB=0x600194aaa9c00 00000400:00000200:8.0:1616513069.879227:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.55@tcp1 of length 224/224 into md 0x865b9f31 [1] + 41088 00000400:00000200:8.0:1616513069.879231:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.879234:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513069.879240:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.879242:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513069.879244:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.879246:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.879287:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688958487141120 00010000:00000200:24.0:1616513069.879297:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688958487141120/t0(0) o400->9a97b66c-c8db-1502-cf17-94c3a01a7a2e@10.31.133.55@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513069.879309:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688958487141376 00000100:00000200:24.0:1616513069.879311:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688958487141120, offset 192 00000400:00000200:24.0:1616513069.879318:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.55@tcp1 00010000:00000200:16.0:1616513069.879321:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9c800 x1688958487141376/t0(0) o400->2da0916f-4af8-08fe-a42f-4bf5367e468b@10.31.133.55@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513069.879326:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.55@tcp1 routed destination 00000100:00000200:16.0:1616513069.879336:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688958487141376, offset 192 00000400:00000200:24.0:1616513069.879341:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.55@tcp1(10.31.133.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513069.879342:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.55@tcp1 00000800:00000200:24.0:1616513069.879347:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513069.879350:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.55@tcp1 routed destination 00000800:00000200:24.0:1616513069.879358:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.879361:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.879363:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513069.879365:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.55@tcp1(10.31.133.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.879368:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513069.879372:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513069.879373:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.879374:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513069.879378:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.879381:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.879383:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.879388:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.879393:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513069.879426:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513069.879432:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513069.879447:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513069.879450:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513069.879451:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513069.879453:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513069.879457:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513069.879460:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513069.879461:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513069.879461:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf37e8 00000800:00000200:28.0:1616513069.879464:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513069.879466:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513069.879469:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42aa0 00000800:00000200:2.0:1616513069.879473:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:11.0:1616513069.882483:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923632, portal 4 00000100:00000200:11.0:1616513069.882488:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923632, offset 0 00000400:00000200:11.0:1616513069.882496:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:11.0:1616513069.882503:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:11.0:1616513069.882516:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513069.882523:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:11.0:1616513069.882529:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513069.882532:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:11.0:1616513069.882534:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:11.0:1616513069.882540:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513069.882586:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513069.882598:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513069.882601:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513069.882603:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.882608:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513069.882613:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be44800 x1693067725923632/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513069.882624:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a220 00000800:00000200:8.0:1616513069.882627:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513069.882629:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513069.882777:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:12.0:1616513069.882830:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513069.882839:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:12.0:1616513069.882848:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513069.882856:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626530 00000400:00000200:12.0:1616513069.882863:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d2981 [1] + 192 00000400:00000200:12.0:1616513069.882868:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.882876:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be44800 x1693067725923632/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513069.882883:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513069.882891:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513069.882894:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513069.882896:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:11.0:1616513069.882898:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be44800 x1693067725923632/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513069.882909:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be44800 x1693067725923632/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513069.882918:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3aaa0 00000800:00000200:7.2:1616513069.886205:0:48376:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513069.886260:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513069.886270:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513069.886279:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.233@tcp1 : PUT - for me 00000400:00000200:26.0:1616513069.886288:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.233@tcp1 of length 224 into portal 26 MB=0x603e102400280 00000400:00000200:26.0:1616513069.886294:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.233@tcp1 of length 224/224 into md 0xf97c3c19 [1] + 0 00000400:00000200:26.0:1616513069.886299:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e660 00000400:00000200:26.0:1616513069.886302:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513069.886304:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513069.886309:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e660 00000800:00000200:26.0:1616513069.886312:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513069.886314:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513069.886315:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513069.886322:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513069.886325:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.886326:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513069.886336:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.233@tcp1 : PUT - for me 00000400:00000200:10.0:1616513069.886345:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.233@tcp1 of length 224 into portal 12 MB=0x603e102400340 00000400:00000200:10.0:1616513069.886351:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.233@tcp1 of length 224/224 into md 0xf96450b9 [1] + 10664 00000400:00000200:10.0:1616513069.886357:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513069.886359:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513069.886367:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.886370:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.886372:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.886373:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114800538240 00010000:00000200:24.0:1616513069.886384:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1693114800538240/t0(0) o400->ad589bbf-2394-3af0-7a99-777c6c11aefe@10.31.136.233@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.886398:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114800538240, offset 192 00000400:00000200:24.0:1616513069.886404:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.233@tcp1 00000400:00000200:24.0:1616513069.886412:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.233@tcp1 routed destination 00000400:00000200:24.0:1616513069.886427:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.233@tcp1(10.31.136.233@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:14.0:1616513069.886432:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114800538432 00000800:00000200:24.0:1616513069.886448:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.886463:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:14.0:1616513069.886464:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bd9680 x1693114800538432/t0(0) o400->a969d1d0-4da8-db42-6a74-117dde9e4334@10.31.136.233@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513069.886465:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.886468:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.886472:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:14.0:1616513069.886478:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114800538432, offset 192 00000800:00000200:11.2:1616513069.886479:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513069.886480:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513069.886484:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.233@tcp1 00000800:00000200:8.0:1616513069.886488:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513069.886490:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513069.886492:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.233@tcp1 routed destination 00000400:00000200:8.0:1616513069.886492:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513069.886496:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.886499:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:8.0:1616513069.886501:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513069.886504:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513069.886506:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.233@tcp1(10.31.136.233@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513069.886513:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513069.886519:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513069.886521:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513069.886524:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513069.886528:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.886533:0:48376:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513069.886585:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513069.886592:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.886594:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513069.886594:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513069.886600:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513069.886604:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757650 00000800:00000200:2.0:1616513069.886608:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513069.930520:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923648, portal 4 00000100:00000200:22.0:1616513069.930526:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923648, offset 0 00000400:00000200:22.0:1616513069.930532:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:22.0:1616513069.930539:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:22.0:1616513069.930552:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513069.930559:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:22.0:1616513069.930565:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513069.930567:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:22.0:1616513069.930570:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:22.0:1616513069.930575:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513069.930620:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513069.930678:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513069.930685:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513069.930688:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513069.930695:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513069.930701:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b279b00 x1693067725923648/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513069.930713:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aaa0 00000800:00000200:28.0:1616513069.930717:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513069.930721:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513069.930828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513069.930839:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513069.930848:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513069.930857:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513069.930865:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626540 00000400:00000200:8.0:1616513069.930872:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d29a1 [1] + 192 00000400:00000200:8.0:1616513069.930877:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513069.930882:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b279b00 x1693067725923648/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513069.930893:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513069.930897:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513069.930900:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513069.930902:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:22.0:1616513069.930905:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b279b00 x1693067725923648/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513069.930930:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b279b00 x1693067725923648/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513076 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513069.930939:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba08 00000800:00000200:11.2:1616513069.966749:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513069.966805:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513069.966815:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513069.966824:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.231@tcp1 : PUT - for me 00000400:00000200:28.0:1616513069.966834:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.231@tcp1 of length 224 into portal 26 MB=0x601878ed28480 00000400:00000200:28.0:1616513069.966841:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.231@tcp1 of length 224/224 into md 0xf97c3c39 [1] + 0 00000400:00000200:28.0:1616513069.966845:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f7e8 00000400:00000200:28.0:1616513069.966849:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513069.966851:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513069.966856:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f7e8 00000800:00000200:28.0:1616513069.966859:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513069.966862:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.966863:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513069.966867:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513069.966921:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690531588637824 00000800:00000200:11.2:1616513069.966929:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:24.0:1616513069.966932:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1690531588637824/t0(0) o400->26c62256-4449-3020-769b-b1b5b86522a2@10.31.141.231@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.966946:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690531588637824, offset 192 00000400:00000200:24.0:1616513069.966952:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.231@tcp1 00000400:00000200:24.0:1616513069.966959:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.231@tcp1 routed destination 00000400:00000200:24.0:1616513069.966973:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.231@tcp1(10.31.141.231@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.966979:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.966985:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.966987:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.966987:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.966989:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513069.966994:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513069.966997:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513069.966998:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:24.0:1616513069.967001:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513069.967005:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.231@tcp1 : PUT - for me 00000800:00000200:8.0:1616513069.967005:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:8.0:1616513069.967007:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513069.967008:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513069.967014:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.231@tcp1 of length 224 into portal 12 MB=0x601878ed28680 00000400:00000200:8.0:1616513069.967014:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513069.967019:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ed0 00000800:00000200:8.0:1616513069.967022:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513069.967024:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.231@tcp1 of length 224/224 into md 0x865b9f31 [1] + 41312 00000400:00000200:20.0:1616513069.967029:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513069.967031:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513069.967039:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.967042:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513069.967044:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.967111:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690531588638336 00010000:00000200:16.0:1616513069.967124:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cee780 x1690531588638336/t0(0) o400->fcfbb072-cb22-22df-aac2-db09f15b3353@10.31.141.231@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.967139:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690531588638336, offset 192 00000400:00000200:16.0:1616513069.967145:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.231@tcp1 00000400:00000200:16.0:1616513069.967152:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.231@tcp1 routed destination 00000400:00000200:16.0:1616513069.967167:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.231@tcp1(10.31.141.231@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.967174:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.967180:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.967182:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.967186:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.967192:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.967198:0:48385:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513069.967253:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513069.967260:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513069.967263:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513069.967271:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513069.967275:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42440 00000800:00000200:26.0:1616513069.967279:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513069.967282:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.967954:0:48385:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513069.967966:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513069.967976:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513069.967985:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.44@tcp1 : PUT - for me 00000400:00000200:30.0:1616513069.967996:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.44@tcp1 of length 224 into portal 26 MB=0x60009f328e980 00000400:00000200:30.0:1616513069.968003:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.44@tcp1 of length 224/224 into md 0xf97c3c49 [1] + 0 00000400:00000200:30.0:1616513069.968007:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ed48 00000400:00000200:30.0:1616513069.968010:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513069.968013:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513069.968017:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ed48 00000800:00000200:30.0:1616513069.968020:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513069.968022:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513069.968023:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513069.968026:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.968030:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688892594514304 00010000:00000200:24.0:1616513069.968036:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688892594514304/t0(0) o400->1d4028ad-0c41-bb76-513b-de826b8ff92a@10.31.142.44@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513069.968041:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513069.968047:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688892594514304, offset 192 00000400:00000200:24.0:1616513069.968054:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.44@tcp1 00000400:00000200:24.0:1616513069.968060:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.44@tcp1 routed destination 00000400:00000200:24.0:1616513069.968074:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.44@tcp1(10.31.142.44@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.968079:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.968084:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.968087:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.968088:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513069.968091:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513069.968094:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513069.968095:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513069.968104:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513069.968105:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513069.968111:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513069.968113:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.44@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.968113:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513069.968114:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513069.968120:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513069.968122:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.44@tcp1 of length 224 into portal 12 MB=0x60009f328eb00 00000400:00000200:12.0:1616513069.968126:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2a18 00000800:00000200:12.0:1616513069.968129:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513069.968131:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.44@tcp1 of length 224/224 into md 0x865b9f31 [1] + 41536 00000400:00000200:28.0:1616513069.968136:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513069.968139:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513069.968146:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513069.968151:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513069.968153:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.968158:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688892594514688 00010000:00000200:16.0:1616513069.968165:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8f980 x1688892594514688/t0(0) o400->c4ac6f9e-8466-8ab9-0ed9-81dc78fca58c@10.31.142.44@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.968176:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688892594514688, offset 192 00000400:00000200:16.0:1616513069.968181:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.44@tcp1 00000400:00000200:16.0:1616513069.968187:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.44@tcp1 routed destination 00000400:00000200:16.0:1616513069.968200:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.44@tcp1(10.31.142.44@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.968206:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.968211:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.968213:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.968215:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.968226:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.968231:0:48385:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513069.968241:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513069.968245:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513069.968247:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513069.968251:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513069.968253:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513069.968255:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43d38 00000800:00000200:26.0:1616513069.968258:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513069.985235:0:48387:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513069.985291:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513069.985301:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513069.985303:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513069.985310:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.83@tcp1 : PUT - for me 00000400:00000200:2.0:1616513069.985320:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.83@tcp1 of length 224 into portal 26 MB=0x604a43a1d40c0 00000400:00000200:2.0:1616513069.985327:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.83@tcp1 of length 224/224 into md 0xf97c3c69 [1] + 0 00000400:00000200:2.0:1616513069.985332:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f3a8 00000400:00000200:2.0:1616513069.985337:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513069.985340:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513069.985344:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f3a8 00000800:00000200:2.0:1616513069.985347:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513069.985350:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513069.985352:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513069.985409:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693953256407232 00010000:00000200:24.0:1616513069.985420:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1693953256407232/t0(0) o400->81918792-84ae-a108-9a09-3facc7098a08@10.31.140.83@tcp1:26/0 lens 224/224 e 0 to 0 dl 1616513161 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513069.985447:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693953256407232, offset 192 00000400:00000200:24.0:1616513069.985453:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.83@tcp1 00000400:00000200:24.0:1616513069.985460:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.83@tcp1 routed destination 00000400:00000200:24.0:1616513069.985475:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.83@tcp1(10.31.140.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513069.985482:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513069.985488:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513069.985490:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513069.985492:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513069.985497:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513069.985504:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513069.985504:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513069.985559:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513069.985566:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513069.985569:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513069.985570:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513069.985575:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513069.985581:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ba0 00000800:00000200:20.0:1616513069.985584:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513069.985667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513069.985720:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513069.985730:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513069.985751:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.83@tcp1 : PUT - for me 00000400:00000200:12.0:1616513069.985758:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.83@tcp1 of length 224 into portal 12 MB=0x604a43a1d4b00 00000400:00000200:12.0:1616513069.985764:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.83@tcp1 of length 224/224 into md 0x865b9f31 [1] + 41760 00000400:00000200:12.0:1616513069.985769:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513069.985772:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513069.985778:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513069.985781:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.985781:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513069.985785:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513069.985842:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693953256409856 00010000:00000200:16.0:1616513069.985856:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa47500 x1693953256409856/t0(0) o400->4bcf395c-371a-086b-968b-d47e43b6d65d@10.31.140.83@tcp1:695/0 lens 224/224 e 0 to 0 dl 1616513075 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513069.985870:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693953256409856, offset 192 00000400:00000200:16.0:1616513069.985876:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.83@tcp1 00000400:00000200:16.0:1616513069.985883:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.83@tcp1 routed destination 00000400:00000200:16.0:1616513069.985898:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.83@tcp1(10.31.140.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513069.985905:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513069.985911:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513069.985914:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513069.985916:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513069.985921:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513069.985925:0:48387:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513069.985979:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513069.985986:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513069.985989:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513069.985995:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513069.985999:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43210 00000800:00000200:10.0:1616513069.986003:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513069.986006:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513070.010518:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923664, portal 4 00000100:00000200:18.0:1616513070.010524:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923664, offset 0 00000400:00000200:18.0:1616513070.010530:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:18.0:1616513070.010536:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:18.0:1616513070.010551:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513070.010558:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:18.0:1616513070.010564:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513070.010568:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:18.0:1616513070.010570:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:18.0:1616513070.010576:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513070.010623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513070.010642:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513070.010648:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513070.010651:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513070.010658:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513070.010664:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b27c800 x1693067725923664/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513070.010676:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc7f8 00000800:00000200:27.0:1616513070.010681:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513070.010684:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513070.010794:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513070.010803:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513070.010810:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513070.010814:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513070.010818:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513070.010826:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626550 00000400:00000200:27.0:1616513070.010833:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164d1d [1] + 192 00000400:00000200:27.0:1616513070.010838:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513070.010843:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b27c800 x1693067725923664/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513070.010857:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513070.010860:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513070.010862:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:18.0:1616513070.010868:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b27c800 x1693067725923664/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:18.0:1616513070.010881:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b27c800 x1693067725923664/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513070.010892:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd6d8 00000800:00000200:7.2:1616513070.040444:0:48393:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.040458:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.040467:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.040476:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.238@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.040487:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.238@tcp1 of length 224 into portal 26 MB=0x5fffbcce3c440 00000400:00000200:30.0:1616513070.040493:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.238@tcp1 of length 224/224 into md 0xf97c3c89 [1] + 0 00000400:00000200:30.0:1616513070.040510:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f760 00000400:00000200:30.0:1616513070.040513:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.040516:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513070.040519:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f760 00000800:00000200:2.0:1616513070.040519:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.040523:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.040525:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.040527:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513070.040586:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688831822906432 00010000:00000200:24.0:1616513070.040598:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688831822906432/t0(0) o400->3aab4823-bb32-d055-8954-bb5cbe0f1bfa@10.31.141.238@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513070.040612:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688831822906432, offset 192 00000400:00000200:24.0:1616513070.040619:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.238@tcp1 00000400:00000200:24.0:1616513070.040626:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.238@tcp1 routed destination 00000400:00000200:24.0:1616513070.040641:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.238@tcp1(10.31.141.238@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.040648:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:11.2:1616513070.040657:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513070.040662:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.040665:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513070.040667:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513070.040669:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513070.040672:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513070.040677:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:24.0:1616513070.040679:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:8.0:1616513070.040685:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.238@tcp1 : PUT - for me 00000400:00000200:8.0:1616513070.040694:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.238@tcp1 of length 224 into portal 12 MB=0x5fffbcce3c740 00000400:00000200:8.0:1616513070.040700:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.238@tcp1 of length 224/224 into md 0x865b9f31 [1] + 41984 00000400:00000200:8.0:1616513070.040705:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513070.040707:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513070.040714:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513070.040716:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513070.040718:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513070.040720:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513070.040723:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513070.040725:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513070.040731:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513070.040735:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3540 00000800:00000200:8.0:1616513070.040738:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513070.040740:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513070.040777:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688831822907200 00010000:00000200:16.0:1616513070.040789:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9b600 x1688831822907200/t0(0) o400->39f3fdc2-47f2-b051-04fb-a942d1307ef1@10.31.141.238@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513070.040803:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688831822907200, offset 192 00000400:00000200:16.0:1616513070.040811:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.238@tcp1 00000400:00000200:16.0:1616513070.040817:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.238@tcp1 routed destination 00000400:00000200:16.0:1616513070.040831:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.238@tcp1(10.31.141.238@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513070.040837:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513070.040843:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513070.040846:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513070.040848:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513070.040852:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.040859:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.040918:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.040925:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513070.040928:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513070.040935:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513070.040939:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42b28 00000800:00000200:10.0:1616513070.040943:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.040946:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513070.042603:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923680, portal 4 00000100:00000200:6.0:1616513070.042608:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923680, offset 0 00000400:00000200:6.0:1616513070.042615:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513070.042620:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:6.0:1616513070.042633:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513070.042639:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:6.0:1616513070.042646:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513070.042649:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:6.0:1616513070.042651:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:6.0:1616513070.042656:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513070.042701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513070.042716:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513070.042723:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513070.042726:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513070.042728:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513070.042733:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513070.042738:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d5a00 x1693067725923680/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513070.042750:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00dd0 00000800:00000200:2.0:1616513070.042754:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513070.042881:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513070.042942:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513070.042951:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:10.0:1616513070.042960:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:10.0:1616513070.042968:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626560 00000400:00000200:10.0:1616513070.042975:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c4429 [1] + 192 00000400:00000200:10.0:1616513070.042980:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.042984:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d5a00 x1693067725923680/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513070.042995:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513070.042999:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000100:00000200:6.0:1616513070.043004:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d5a00 x1693067725923680/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:10.0:1616513070.043005:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513070.043007:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:6.0:1616513070.043017:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d5a00 x1693067725923680/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513070.043027:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf016d8 00000800:00000200:11.2:1616513070.100679:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513070.100735:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.100745:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513070.100755:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.121@tcp1 : PUT - for me 00000400:00000200:12.0:1616513070.100764:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.121@tcp1 of length 224 into portal 26 MB=0x6037f10519680 00000400:00000200:12.0:1616513070.100772:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.121@tcp1 of length 224/224 into md 0xf97c3ca9 [1] + 0 00000400:00000200:12.0:1616513070.100777:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e220 00000400:00000200:12.0:1616513070.100780:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513070.100783:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513070.100788:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e220 00000800:00000200:20.0:1616513070.100791:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.100794:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.100797:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513070.100799:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513070.100801:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.100802:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513070.100806:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513070.100807:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513070.100812:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.81@tcp1 : PUT - for me 00000400:00000200:12.0:1616513070.100813:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.121@tcp1 : PUT - for me 00000400:00000200:12.0:1616513070.100823:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.121@tcp1 of length 224 into portal 12 MB=0x6037f10519700 00000400:00000200:20.0:1616513070.100824:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.81@tcp1 of length 224 into portal 26 MB=0x5ffe76de018c0 00000400:00000200:12.0:1616513070.100829:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.121@tcp1 of length 224/224 into md 0x865b9f31 [1] + 42208 00000400:00000200:20.0:1616513070.100831:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.81@tcp1 of length 224/224 into md 0xf97c3cb9 [1] + 0 00000400:00000200:12.0:1616513070.100833:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513070.100835:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513070.100836:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ef68 00000400:00000200:20.0:1616513070.100839:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513070.100841:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:20.0:1616513070.100842:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513070.100843:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513070.100844:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513070.100847:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ef68 00000800:00000200:20.0:1616513070.100850:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.100855:0:6464:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.100859:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513070.100861:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513070.100863:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692694129776256 00000800:00000200:7.2:1616513070.100863:0:48400:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513070.100864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513070.100873:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:24.0:1616513070.100875:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1692694129776256/t0(0) o400->1a385b23-e3e0-2184-7d28-3032f85d8d91@10.31.143.121@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:30.0:1616513070.100876:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513070.100878:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513070.100880:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513070.100882:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513070.100885:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:24.0:1616513070.100890:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692694129776256, offset 192 00000400:00000200:30.0:1616513070.100894:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.81@tcp1 : PUT - for me 00000400:00000200:24.0:1616513070.100896:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.121@tcp1 00000400:00000200:24.0:1616513070.100903:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.121@tcp1 routed destination 00000400:00000200:30.0:1616513070.100904:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.81@tcp1 of length 224 into portal 12 MB=0x5ffe76de01980 00000100:00000200:23.0F:1616513070.100907:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688744329484480 00000100:00000200:16.0:1616513070.100909:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692694129776384 00000400:00000200:30.0:1616513070.100910:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.81@tcp1 of length 224/224 into md 0xf96450b9 [1] + 10888 00000400:00000200:30.0:1616513070.100917:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513070.100919:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.121@tcp1(10.31.143.121@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:30.0:1616513070.100920:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:16.0:1616513070.100924:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa47980 x1692694129776384/t0(0) o400->e46ab10b-c0b4-d26b-e571-21ed2ec1d700@10.31.143.121@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:23.0:1616513070.100925:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688744329484480/t0(0) o400->42997430-72ad-d2ed-fdce-a95efae99f13@10.31.141.81@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513070.100926:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513070.100927:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.100930:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.100932:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513070.100932:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513070.100934:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513070.100934:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513070.100937:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513070.100939:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692694129776384, offset 192 00000800:00000200:24.0:1616513070.100941:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:23.0:1616513070.100941:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688744329484480, offset 192 00000400:00000200:16.0:1616513070.100946:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.121@tcp1 00000800:00000200:11.2:1616513070.100946:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:23.0:1616513070.100947:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.81@tcp1 00000100:00000200:24.0:1616513070.100949:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513070.100954:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.121@tcp1 routed destination 00000400:00000200:23.0:1616513070.100956:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.81@tcp1 routed destination 00000800:00000200:28.0:1616513070.100957:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.100964:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513070.100967:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513070.100968:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513070.100971:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.121@tcp1(10.31.143.121@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513070.100974:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:23.0:1616513070.100974:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.81@tcp1(10.31.141.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513070.100979:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513070.100980:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3d38 00000800:00000200:23.0:1616513070.100982:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513070.100984:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513070.100986:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.100989:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513070.100989:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.100991:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513070.100991:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.100994:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.100996:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513070.100996:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688744329484672 00000800:00000200:23.0:1616513070.100999:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513070.101000:0:48400:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513070.101004:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:14.0:1616513070.101009:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b27c800 x1688744329484672/t0(0) o400->06222c4a-5478-d3c7-d1d7-2f75fb8c24e0@10.31.141.81@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:23.0:1616513070.101011:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513070.101012:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.101016:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.101019:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513070.101019:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:14.0:1616513070.101024:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688744329484672, offset 192 00000400:00000200:12.0:1616513070.101025:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.101030:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1298 00000400:00000200:14.0:1616513070.101031:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.81@tcp1 00000800:00000200:12.0:1616513070.101033:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513070.101038:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.81@tcp1 routed destination 00000800:00000200:26.0:1616513070.101059:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513070.101063:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.81@tcp1(10.31.141.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513070.101066:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513070.101068:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513070.101070:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513070.101075:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:14.0:1616513070.101076:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:26.0:1616513070.101079:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42908 00000800:00000200:14.0:1616513070.101079:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513070.101081:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513070.101083:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.101086:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:14.0:1616513070.101086:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.101090:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.101099:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.101103:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.101107:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513070.101107:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513070.101113:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513070.101116:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756e58 00000800:00000200:20.0:1616513070.101119:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:6.0:1616513070.106468:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923696, portal 4 00000100:00000200:6.0:1616513070.106472:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923696, offset 0 00000400:00000200:6.0:1616513070.106476:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513070.106480:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:6.0:1616513070.106488:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513070.106492:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:6.0:1616513070.106496:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513070.106497:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:6.0:1616513070.106499:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:6.0:1616513070.106502:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513070.106548:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513070.106560:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:30.0:1616513070.106564:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:30.0:1616513070.106566:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513070.106570:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:30.0:1616513070.106571:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:30.0:1616513070.106576:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b279b00 x1693067725923696/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513070.106587:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00e58 00000800:00000200:30.0:1616513070.106591:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513070.106723:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513070.106777:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513070.106787:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:26.0:1616513070.106795:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513070.106803:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626570 00000400:00000200:26.0:1616513070.106810:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c4479 [1] + 192 00000400:00000200:26.0:1616513070.106815:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513070.106820:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b279b00 x1693067725923696/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:10.0:1616513070.106831:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513070.106835:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513070.106837:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:6.0:1616513070.106837:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b279b00 x1693067725923696/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513070.106839:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:6.0:1616513070.106846:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b279b00 x1693067725923696/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513070.106852:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf016d8 00000800:00000200:7.2:1616513070.112115:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513070.112172:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.112182:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.112182:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513070.112191:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.169@tcp1 : PUT - for me 00000400:00000200:2.0:1616513070.112200:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.169@tcp1 of length 224 into portal 12 MB=0x5ffee4074e480 00000400:00000200:2.0:1616513070.112207:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.169@tcp1 of length 224/224 into md 0xf96450b9 [1] + 11112 00000400:00000200:2.0:1616513070.112212:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513070.112215:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513070.112221:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.112224:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.112226:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.112265:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513070.112278:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513070.112285:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000100:00000200:14.0:1616513070.112286:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688773632255104 00000400:00000200:8.0:1616513070.112293:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.169@tcp1 : PUT - for me 00010000:00000200:14.0:1616513070.112299:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138f980 x1688773632255104/t0(0) o400->fc3d3ace-fb14-5c68-753d-c0aba56bd645@10.31.141.169@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:8.0:1616513070.112302:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.169@tcp1 of length 224 into portal 26 MB=0x5ffee4074e2c0 00000400:00000200:8.0:1616513070.112308:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.169@tcp1 of length 224/224 into md 0xf97c3cd9 [1] + 0 00000400:00000200:8.0:1616513070.112312:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e908 00000100:00000200:14.0:1616513070.112315:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688773632255104, offset 192 00000400:00000200:8.0:1616513070.112315:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513070.112317:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513070.112321:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e908 00000800:00000200:8.0:1616513070.112324:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513070.112326:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.169@tcp1 00000800:00000200:12.0:1616513070.112327:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513070.112327:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513070.112331:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513070.112334:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.169@tcp1 routed destination 00000400:00000200:14.0:1616513070.112348:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.169@tcp1(10.31.141.169@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.112355:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513070.112361:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.112363:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.112365:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513070.112370:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.112375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.112385:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:23.0:1616513070.112391:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688773632254656 00000800:00000200:10.0:1616513070.112392:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513070.112394:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513070.112401:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:23.0:1616513070.112403:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688773632254656/t0(0) o400->93aa3a46-b700-69aa-0f0e-b8402595a3d6@10.31.141.169@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:10.0:1616513070.112406:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757b18 00000800:00000200:10.0:1616513070.112409:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.112412:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.112417:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688773632254656, offset 192 00000400:00000200:23.0:1616513070.112424:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.169@tcp1 00000400:00000200:23.0:1616513070.112432:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.169@tcp1 routed destination 00000400:00000200:23.0:1616513070.112468:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.169@tcp1(10.31.141.169@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.112475:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.112481:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.112484:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.112486:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.112492:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.112496:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.112499:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513070.112549:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.112557:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513070.112560:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513070.112567:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513070.112573:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0880 00000800:00000200:28.0:1616513070.112577:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.112580:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:7.0:1616513070.154605:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923728, portal 4 00000100:00000200:7.0:1616513070.154610:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923728, offset 0 00000400:00000200:7.0:1616513070.154615:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000100:00000200:11.0:1616513070.154618:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923712, portal 4 00000400:00000200:7.0:1616513070.154620:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000100:00000200:11.0:1616513070.154622:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923712, offset 0 00000400:00000200:11.0:1616513070.154628:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:7.0:1616513070.154629:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000400:00000200:11.0:1616513070.154633:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:7.0:1616513070.154634:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000400:00000200:11.0:1616513070.154643:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513070.154648:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:7.0:1616513070.154649:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513070.154652:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:11.0:1616513070.154653:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513070.154653:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:11.0:1616513070.154656:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:7.0:1616513070.154657:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:11.0:1616513070.154658:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:11.0:1616513070.154662:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:5.2:1616513070.154719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.2:1616513070.154721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513070.154741:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513070.154746:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513070.154748:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513070.154751:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513070.154755:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513070.154761:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8006cdde80 x1693067725923712/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513070.154774:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a000 00000800:00000200:12.0:1616513070.154775:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:27.0:1616513070.154778:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:12.0:1616513070.154784:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513070.154787:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.154794:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:12.0:1616513070.154801:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600900 x1693067725923728/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513070.154813:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637e440 00000800:00000200:12.0:1616513070.154818:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513070.154821:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513070.154877:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:27.2:1616513070.154879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513070.154889:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:28.0:1616513070.154893:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513070.154895:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513070.154897:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:28.0:1616513070.154902:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:27.0:1616513070.154903:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513070.154912:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513070.154912:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626580 00000400:00000200:27.0:1616513070.154918:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37164d5d [1] + 192 00000400:00000200:28.0:1616513070.154920:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626590 00000400:00000200:27.0:1616513070.154924:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513070.154926:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d2a51 [1] + 192 00000100:00000200:27.0:1616513070.154928:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8006cdde80 x1693067725923712/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513070.154932:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.154937:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600900 x1693067725923728/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513070.154942:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513070.154945:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513070.154947:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:20.0:1616513070.154947:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:11.0:1616513070.154948:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8006cdde80 x1693067725923712/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513070.154952:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513070.154955:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513070.154956:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:7.0:1616513070.154957:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600900 x1693067725923728/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513070.154958:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8006cdde80 x1693067725923712/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513070.154965:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3aaa0 00000100:00000200:7.0:1616513070.154967:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600900 x1693067725923728/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513070.154975:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f5c8 00000100:00000200:2.0:1616513070.170483:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923744, portal 4 00000100:00000200:2.0:1616513070.170488:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923744, offset 0 00000400:00000200:2.0:1616513070.170495:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:2.0:1616513070.170501:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:2.0:1616513070.170514:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513070.170521:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:2.0:1616513070.170527:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513070.170532:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513070.170534:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513070.170539:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513070.170586:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513070.170599:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:30.0:1616513070.170605:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:30.0:1616513070.170607:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513070.170609:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:30.0:1616513070.170613:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:30.0:1616513070.170617:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b279b00 x1693067725923744/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513070.170628:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eedf58 00000800:00000200:30.0:1616513070.170631:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513070.170777:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513070.170787:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513070.170796:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:10.0:1616513070.170804:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:10.0:1616513070.170811:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6265a0 00000400:00000200:10.0:1616513070.170817:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c44b9 [1] + 192 00000400:00000200:10.0:1616513070.170822:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.170826:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b279b00 x1693067725923744/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513070.170839:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513070.170840:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:10.0:1616513070.170843:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513070.170845:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:2.0:1616513070.170846:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b279b00 x1693067725923744/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513070.170859:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b279b00 x1693067725923744/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513070.170869:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed760 00000800:00000200:11.2:1616513070.173705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513070.173769:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.173779:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513070.173788:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.91@tcp1 : PUT - for me 00000400:00000200:12.0:1616513070.173797:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.91@tcp1 of length 224 into portal 26 MB=0x602ad74698fc0 00000400:00000200:12.0:1616513070.173804:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.91@tcp1 of length 224/224 into md 0xf97c3ce9 [1] + 0 00000400:00000200:12.0:1616513070.173811:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f980 00000400:00000200:12.0:1616513070.173814:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513070.173817:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513070.173822:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f980 00000800:00000200:12.0:1616513070.173824:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.173827:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513070.173827:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513070.173831:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.173845:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:23.0:1616513070.173889:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691793865936832 00000800:00000200:2.0:1616513070.173900:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:23.0:1616513070.173901:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1691793865936832/t0(0) o400->84a25618-442c-c991-6cee-0113744b00e2@10.31.141.91@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513070.173910:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513070.173911:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513070.173916:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691793865936832, offset 192 00000400:00000200:2.0:1616513070.173919:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.91@tcp1 : PUT - for me 00000400:00000200:23.0:1616513070.173922:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.91@tcp1 00000400:00000200:2.0:1616513070.173928:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.91@tcp1 of length 224 into portal 12 MB=0x602ad74699040 00000400:00000200:23.0:1616513070.173932:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.91@tcp1 routed destination 00000400:00000200:2.0:1616513070.173935:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.91@tcp1 of length 224/224 into md 0xf96450b9 [1] + 11336 00000400:00000200:2.0:1616513070.173940:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513070.173943:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:23.0:1616513070.173949:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.91@tcp1(10.31.141.91@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513070.173954:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.173955:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513070.173957:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.173959:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:23.0:1616513070.173962:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.173965:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.173967:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:14.0:1616513070.173972:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691793865936960 00000800:00000200:23.0:1616513070.173973:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.173977:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.173980:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:14.0:1616513070.173985:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1388000 x1691793865936960/t0(0) o400->0c88d4e0-a1cb-9c3d-d821-08b90c3a73c5@10.31.141.91@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.173998:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691793865936960, offset 192 00000400:00000200:14.0:1616513070.174005:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.91@tcp1 00000400:00000200:14.0:1616513070.174012:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.91@tcp1 routed destination 00000400:00000200:14.0:1616513070.174027:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.91@tcp1(10.31.141.91@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.174033:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513070.174039:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513070.174039:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.174042:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513070.174045:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513070.174046:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513070.174049:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513070.174050:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513070.174050:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.174055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513070.174057:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513070.174065:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1760 00000800:00000200:26.0:1616513070.174068:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513070.174110:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.174117:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513070.174120:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513070.174125:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513070.174126:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513070.174130:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47573a8 00000800:00000200:20.0:1616513070.174135:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513070.194709:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513070.194765:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.194775:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513070.194784:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.137@tcp1 : PUT - for me 00000400:00000200:16.0:1616513070.194794:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.137@tcp1 of length 224 into portal 26 MB=0x602bfa7b139c0 00000400:00000200:16.0:1616513070.194800:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.137@tcp1 of length 224/224 into md 0xf97c3cf9 [1] + 0 00000400:00000200:16.0:1616513070.194805:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f6d8 00000400:00000200:16.0:1616513070.194808:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513070.194811:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513070.194818:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f6d8 00000800:00000200:12.0:1616513070.194819:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.194821:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.194824:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.194826:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.194900:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691872035682752 00010000:00000200:23.0:1616513070.194912:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1691872035682752/t0(0) o400->afb6c4ea-d7da-fb34-4ebb-482392e44fdc@10.31.142.137@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.194926:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691872035682752, offset 192 00000400:00000200:23.0:1616513070.194932:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.137@tcp1 00000400:00000200:23.0:1616513070.194940:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.137@tcp1 routed destination 00000400:00000200:23.0:1616513070.194956:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.137@tcp1(10.31.142.137@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.194963:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.194969:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.194972:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.194974:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.194979:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.194985:0:48409:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.194986:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513070.194996:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.195002:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.195005:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.195011:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.195016:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1e48 00000800:00000200:30.0:1616513070.195020:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.195023:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.195316:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513070.195370:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513070.195380:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513070.195389:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.137@tcp1 : PUT - for me 00000400:00000200:28.0:1616513070.195398:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.137@tcp1 of length 224 into portal 12 MB=0x602bfa7b14840 00000400:00000200:28.0:1616513070.195404:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.137@tcp1 of length 224/224 into md 0x865b9f31 [1] + 42432 00000400:00000200:28.0:1616513070.195411:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.195414:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513070.195422:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.195426:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.195427:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.195430:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513070.195460:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691872035686464 00010000:00000200:16.0:1616513070.195473:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8f080 x1691872035686464/t0(0) o400->94c7f278-5eba-fe95-5187-e322f04e53b1@10.31.142.137@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513070.195487:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691872035686464, offset 192 00000400:00000200:16.0:1616513070.195493:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.137@tcp1 00000400:00000200:16.0:1616513070.195501:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.137@tcp1 routed destination 00000400:00000200:16.0:1616513070.195516:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.137@tcp1(10.31.142.137@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513070.195523:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513070.195529:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513070.195532:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513070.195534:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.195538:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.195543:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.195552:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.195557:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.195560:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513070.195562:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513070.195566:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.195570:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43100 00000800:00000200:12.0:1616513070.195574:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.198489:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513070.198520:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513070.198543:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.198553:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513070.198562:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.99@tcp1 : PUT - for me 00000400:00000200:10.0:1616513070.198573:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.99@tcp1 of length 224 into portal 12 MB=0x5ffd9d6a6fa80 00000800:00000200:28.0:1616513070.198576:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513070.198580:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.99@tcp1 of length 224/224 into md 0xf96450b9 [1] + 11560 00000800:00000200:28.0:1616513070.198585:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:10.0:1616513070.198585:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.198587:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513070.198594:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.99@tcp1 : PUT - for me 00000800:00000200:10.0:1616513070.198594:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.198597:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513070.198599:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.198602:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513070.198603:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.99@tcp1 of length 224 into portal 26 MB=0x5ffd9d6a6fa00 00000400:00000200:28.0:1616513070.198611:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.99@tcp1 of length 224/224 into md 0xf97c3d09 [1] + 0 00000400:00000200:28.0:1616513070.198615:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22f68 00000100:00000200:14.0:1616513070.198616:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688685957806720 00000400:00000200:28.0:1616513070.198618:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.198621:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513070.198626:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22f68 00010000:00000200:14.0:1616513070.198628:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdc800 x1688685957806720/t0(0) o400->cbc7ea47-a59c-956c-a5c2-ef77650cd5d4@10.31.138.99@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513070.198629:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.198631:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513070.198633:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.198637:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513070.198643:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688685957806720, offset 192 00000400:00000200:14.0:1616513070.198649:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.99@tcp1 00000400:00000200:14.0:1616513070.198657:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.99@tcp1 routed destination 00000400:00000200:14.0:1616513070.198672:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.99@tcp1(10.31.138.99@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.198678:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513070.198684:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.198687:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.198689:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.198693:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688685957806592 00000800:00000200:14.0:1616513070.198694:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.198705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:23.0:1616513070.198708:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688685957806592/t0(0) o400->a24a157e-a6fb-292b-ef76-c359f26b768e@10.31.138.99@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:30.0:1616513070.198715:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.198719:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.198721:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:23.0:1616513070.198723:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688685957806592, offset 192 00000400:00000200:30.0:1616513070.198729:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:23.0:1616513070.198730:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.99@tcp1 00000400:00000200:30.0:1616513070.198733:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756ff0 00000800:00000200:30.0:1616513070.198736:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.198738:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:23.0:1616513070.198739:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.99@tcp1 routed destination 00000400:00000200:23.0:1616513070.198755:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.99@tcp1(10.31.138.99@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.198762:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.198770:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.198773:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.198775:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.198780:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.198784:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.198787:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513070.198838:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.198845:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.198848:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.198855:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.198861:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d03b8 00000800:00000200:16.0:1616513070.198865:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.198868:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:2.0:1616513070.202530:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923760, portal 4 00000100:00000200:2.0:1616513070.202536:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923760, offset 0 00000400:00000200:2.0:1616513070.202542:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:2.0:1616513070.202548:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:2.0:1616513070.202561:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513070.202567:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:2.0:1616513070.202573:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513070.202576:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:2.0:1616513070.202578:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:2.0:1616513070.202583:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513070.202634:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.202654:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513070.202660:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513070.202664:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513070.202671:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513070.202678:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b279b00 x1693067725923760/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513070.202690:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eeddc0 00000800:00000200:1.0:1616513070.202695:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.202698:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513070.202761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513070.202771:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.202777:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513070.202781:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513070.202785:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513070.202794:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6265b0 00000400:00000200:1.0:1616513070.202800:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff745 [1] + 192 00000400:00000200:1.0:1616513070.202805:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513070.202810:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b279b00 x1693067725923760/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513070.202825:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.202828:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.202829:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:2.0:1616513070.202835:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b279b00 x1693067725923760/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513070.202848:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b279b00 x1693067725923760/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513070.202861:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed760 00000100:00000200:15.0:1616513070.250652:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923776, portal 4 00000100:00000200:15.0:1616513070.250658:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923776, offset 0 00000400:00000200:15.0:1616513070.250666:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:15.0:1616513070.250671:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:15.0:1616513070.250683:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513070.250689:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000100:00000200:20.0:1616513070.250695:0:6484:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923792, portal 4 00000800:00000200:15.0:1616513070.250695:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513070.250698:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:15.0:1616513070.250700:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:20.0:1616513070.250701:0:6484:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923792, offset 0 00000800:00000200:15.0:1616513070.250705:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:20.0:1616513070.250708:0:6484:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:20.0:1616513070.250715:0:6484:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:20.0:1616513070.250728:0:6484:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513070.250735:0:6484:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:20.0:1616513070.250742:0:6484:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513070.250745:0:6484:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:20.0:1616513070.250747:0:6484:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.2:1616513070.250751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513070.250753:0:6484:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:20.0:1616513070.250771:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513070.250777:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513070.250780:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513070.250786:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513070.250791:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded7500 x1693067725923776/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513070.250793:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923808, portal 4 00000100:00000200:30.0:1616513070.250798:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923808, offset 0 00000800:00000200:1.2:1616513070.250800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:20.0:1616513070.250803:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775320 00000400:00000200:30.0:1616513070.250805:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:20.0:1616513070.250807:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513070.250810:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000400:00000200:30.0:1616513070.250811:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:1.0:1616513070.250818:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:30.0:1616513070.250823:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:1.0:1616513070.250824:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513070.250826:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513070.250829:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513070.250831:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513070.250834:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:30.0:1616513070.250835:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513070.250838:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:30.0:1616513070.250840:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (22)++ 00000100:00000200:1.0:1616513070.250841:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cb02e4380 x1693067725923792/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513070.250845:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.2:1616513070.250849:0:6467:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:5.0:1616513070.250858:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000400:00000200:1.0:1616513070.250860:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ee0 00000800:00000200:5.0:1616513070.250866:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (24)-- 00000400:00000200:5.0:1616513070.250869:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513070.250871:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513070.250874:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:5.0:1616513070.250876:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:5.0:1616513070.250883:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56cbad3600 x1693067725923808/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513070.250901:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee4c8 00000800:00000200:5.0:1616513070.250905:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:5.2:1616513070.250935:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513070.250948:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513070.250957:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:16.0:1616513070.250971:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513070.250980:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6265c0 00000800:00000200:1.2:1616513070.250981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:16.0:1616513070.250987:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d2a71 [1] + 192 00000800:00000200:1.0:1616513070.250990:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:16.0:1616513070.250993:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:1.0:1616513070.250997:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.232@o2ib1 00000100:00000200:16.0:1616513070.250998:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded7500 x1693067725923776/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513070.250999:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:5.0:1616513070.251005:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513070.251006:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:12.0:1616513070.251008:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:9.0:1616513070.251009:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:5.0:1616513070.251013:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513070.251014:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6265d0 00000400:00000200:5.0:1616513070.251021:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6265e0 00000400:00000200:1.0:1616513070.251021:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff755 [1] + 192 00000800:00000200:16.0:1616513070.251026:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:1.0:1616513070.251027:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513070.251029:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:5.0:1616513070.251029:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff765 [1] + 192 00000800:00000200:16.0:1616513070.251031:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000400:00000200:5.0:1616513070.251034:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513070.251035:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cb02e4380 x1693067725923792/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513070.251036:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded7500 x1693067725923776/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513070.251049:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded7500 x1693067725923776/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513070.251050:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000100:00000200:5.0:1616513070.251051:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56cbad3600 x1693067725923808/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513070.251053:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513070.251055:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:15.0:1616513070.251059:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774c38 00000800:00000200:5.0:1616513070.251065:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:20.0:1616513070.251067:0:6484:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cb02e4380 x1693067725923792/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513070.251068:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:5.0:1616513070.251069:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:20.0:1616513070.251081:0:6484:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cb02e4380 x1693067725923792/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513070.251090:0:6484:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23e48 00000100:00000200:30.0:1616513070.251095:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56cbad3600 x1693067725923808/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513070.251107:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56cbad3600 x1693067725923808/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513070.251117:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee660 00000800:00000200:7.2:1616513070.284628:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513070.284686:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.284698:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513070.284708:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.89@tcp1 : PUT - for me 00000400:00000200:10.0:1616513070.284718:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.89@tcp1 of length 224 into portal 26 MB=0x5f90941f3cfc0 00000400:00000200:10.0:1616513070.284725:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.89@tcp1 of length 224/224 into md 0xf97c3d19 [1] + 0 00000400:00000200:10.0:1616513070.284730:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e2a8 00000400:00000200:10.0:1616513070.284733:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.284736:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513070.284741:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e2a8 00000800:00000200:26.0:1616513070.284742:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.284745:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.284747:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.284749:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.284796:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:23.0:1616513070.284808:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681193040072640 00010000:00000200:23.0:1616513070.284819:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1681193040072640/t0(0) o400->cd8e2722-e70e-0eda-e2cc-00e1371267ca@10.31.131.89@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.284833:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681193040072640, offset 192 00000400:00000200:23.0:1616513070.284840:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.89@tcp1 00000400:00000200:23.0:1616513070.284848:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.89@tcp1 routed destination 00000800:00000200:20.0:1616513070.284851:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.284861:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:23.0:1616513070.284864:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.89@tcp1(10.31.131.89@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.284871:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513070.284871:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.89@tcp1 : PUT - for me 00000800:00000200:23.0:1616513070.284878:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.284881:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513070.284881:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.89@tcp1 of length 224 into portal 12 MB=0x5f90941f3d040 00000800:00000200:23.0:1616513070.284883:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513070.284887:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.89@tcp1 of length 224/224 into md 0x865b9f31 [1] + 42656 00000800:00000200:23.0:1616513070.284888:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513070.284893:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:7.2:1616513070.284893:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.284895:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:20.0:1616513070.284896:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513070.284903:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513070.284903:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.284904:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.284906:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513070.284908:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513070.284909:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.284913:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.284919:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.284925:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d14b8 00000800:00000200:30.0:1616513070.284928:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.284931:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513070.284981:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681193040072768 00010000:00000200:16.0:1616513070.284995:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8000 x1681193040072768/t0(0) o400->7e6a180f-e8c4-9dd0-1522-48799f40e282@10.31.131.89@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513070.285010:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681193040072768, offset 192 00000400:00000200:16.0:1616513070.285017:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.89@tcp1 00000400:00000200:16.0:1616513070.285025:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.89@tcp1 routed destination 00000400:00000200:16.0:1616513070.285042:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.89@tcp1(10.31.131.89@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513070.285048:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513070.285054:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513070.285058:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513070.285060:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.285065:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.285070:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.285122:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.285130:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.285133:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513070.285135:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513070.285139:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.285143:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a423b8 00000800:00000200:12.0:1616513070.285147:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513070.295358:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513070.295414:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513070.295424:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513070.295446:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.60@tcp1 : PUT - for me 00000400:00000200:28.0:1616513070.295455:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.60@tcp1 of length 224 into portal 26 MB=0x5ffd84bda3300 00000400:00000200:28.0:1616513070.295463:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.60@tcp1 of length 224/224 into md 0xf97c3d49 [1] + 0 00000400:00000200:28.0:1616513070.295467:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f870 00000400:00000200:28.0:1616513070.295471:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.295473:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513070.295478:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f870 00000800:00000200:28.0:1616513070.295482:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.295483:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.295485:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.295487:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.295544:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688679334163200 00010000:00000200:23.0:1616513070.295555:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688679334163200/t0(0) o400->6ec9e565-327c-58ed-29fa-98c6b92a85c0@10.31.128.60@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.295570:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688679334163200, offset 192 00000400:00000200:23.0:1616513070.295577:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.60@tcp1 00000400:00000200:23.0:1616513070.295585:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.60@tcp1 routed destination 00000400:00000200:23.0:1616513070.295601:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.60@tcp1(10.31.128.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.295607:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.295614:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.295616:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.295618:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.295623:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.295629:0:48420:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.295632:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513070.295684:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.295691:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513070.295695:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513070.295701:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513070.295707:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1100 00000800:00000200:26.0:1616513070.295712:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.295715:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.295759:0:48420:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.295770:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.295780:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.295789:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.60@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.295799:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.60@tcp1 of length 224 into portal 12 MB=0x5ffd84bda36c0 00000400:00000200:30.0:1616513070.295805:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.60@tcp1 of length 224/224 into md 0xf96450b9 [1] + 11784 00000400:00000200:30.0:1616513070.295811:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.295815:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513070.295822:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.295824:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.295825:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.295828:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.295886:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688679334164160 00010000:00000200:14.0:1616513070.295897:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b279b00 x1688679334164160/t0(0) o400->df0c222e-9456-64c6-41cc-9c3fc3992e0e@10.31.128.60@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.295912:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688679334164160, offset 192 00000400:00000200:14.0:1616513070.295918:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.60@tcp1 00000400:00000200:14.0:1616513070.295925:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.60@tcp1 routed destination 00000400:00000200:14.0:1616513070.295940:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.60@tcp1(10.31.128.60@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.295946:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513070.295951:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.295954:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513070.295956:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513070.295960:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.295965:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.296019:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.296026:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.296029:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.296035:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.296039:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756ff0 00000800:00000200:16.0:1616513070.296043:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.296046:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.298574:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923824, portal 4 00000100:00000200:24.0:1616513070.298581:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923824, offset 0 00000400:00000200:24.0:1616513070.298587:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:24.0:1616513070.298593:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:24.0:1616513070.298606:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.298612:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:24.0:1616513070.298618:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.298621:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:24.0:1616513070.298624:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:24.0:1616513070.298628:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513070.298676:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.298694:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513070.298700:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513070.298703:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513070.298710:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513070.298717:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d846aa42400 x1693067725923824/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513070.298730:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3d38 00000800:00000200:1.0:1616513070.298735:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.298737:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513070.298809:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513070.298817:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.298823:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513070.298832:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513070.298839:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6265f0 00000400:00000200:1.0:1616513070.298846:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff775 [1] + 192 00000400:00000200:1.0:1616513070.298851:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513070.298855:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d846aa42400 x1693067725923824/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513070.298868:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.298869:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513070.298872:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.298876:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:24.0:1616513070.298879:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d846aa42400 x1693067725923824/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:24.0:1616513070.298892:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d846aa42400 x1693067725923824/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513070.298905:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:15.2F:1616513070.303433:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:1.0:1616513070.303492:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:1.0:1616513070.303502:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.151@o2ib1 00000400:00000200:1.0:1616513070.303511:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.151@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513070.303520:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.151@o2ib1 of length 224 into portal 26 MB=0x5f9586dcfa4c0 00000400:00000200:1.0:1616513070.303527:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.151@o2ib1 of length 224/224 into md 0xf97c3d59 [1] + 0 00000400:00000200:1.0:1616513070.303532:0:6465:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fdc0 00000400:00000200:1.0:1616513070.303535:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513070.303537:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:1.0:1616513070.303542:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fdc0 00000800:00000200:9.0:1616513070.303544:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:1.0:1616513070.303545:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:1.0:1616513070.303548:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (24)-- 00000800:00000200:1.0:1616513070.303551:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (23)-- 00000800:00000200:9.0:1616513070.303554:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.151@o2ib1 00000800:00000200:1.0:1616513070.303554:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000400:00000200:9.0:1616513070.303563:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.151@o2ib1 : PUT - for me 00000400:00000200:9.0:1616513070.303571:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.151@o2ib1 of length 224 into portal 12 MB=0x5f9586dcfa540 00000400:00000200:9.0:1616513070.303578:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.151@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 34480 00000400:00000200:9.0:1616513070.303583:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:9.0:1616513070.303586:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:9.0:1616513070.303594:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:9.0:1616513070.303597:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:9.0:1616513070.303599:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000100:00000200:23.0:1616513070.303629:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681533078316224 00010000:00000200:23.0:1616513070.303638:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1681533078316224/t0(0) o400->9241ddb0-c18f-012c-8e38-794bc45f506e@10.242.46.151@o2ib1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.303652:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681533078316224, offset 192 00000400:00000200:23.0:1616513070.303658:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.151@o2ib1 00000400:00000200:23.0:1616513070.303665:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.151@o2ib1 local destination 00000100:00000200:25.0F:1616513070.303666:0:172819:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681533078316352 00000400:00000200:23.0:1616513070.303677:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.151@o2ib1(10.242.46.151@o2ib1:10.242.46.151@o2ib1) : PUT try# 0 00010000:00000200:25.0:1616513070.303682:0:172819:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4558ad6300 x1681533078316352/t0(0) o400->67eb3afe-7574-8f15-013a-b1df12d1e90c@10.242.46.151@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513070.303683:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.151@o2ib1 00000800:00000200:23.0:1616513070.303689:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4319f80] -> 10.242.46.151@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.303692:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:23.0:1616513070.303694:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87e8600] (21)++ 00000100:00000200:25.0:1616513070.303698:0:172819:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681533078316352, offset 192 00000800:00000200:23.0:1616513070.303698:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (22)-- 00000400:00000200:25.0:1616513070.303704:0:172819:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.151@o2ib1 00000800:00000200:15.2:1616513070.303704:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (21)++ 00000100:00000200:23.0:1616513070.303705:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:25.0:1616513070.303711:0:172819:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.151@o2ib1 local destination 00000800:00000200:1.0:1616513070.303712:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:1.0:1616513070.303718:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87e8600] (23)-- 00000400:00000200:1.0:1616513070.303720:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513070.303723:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000400:00000200:25.0:1616513070.303725:0:172819:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.151@o2ib1(10.242.46.151@o2ib1:10.242.46.151@o2ib1) : PUT try# 0 00000400:00000200:1.0:1616513070.303725:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.151@o2ib1: PUT: OK 00000400:00000200:1.0:1616513070.303730:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1f58 00000800:00000200:25.0:1616513070.303731:0:172819:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.151@o2ib1 00000800:00000200:1.0:1616513070.303733:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000800:00000200:25.0:1616513070.303737:0:172819:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4319f80] -> 10.242.46.151@o2ib1 (2) version: 12 00000800:00000200:25.0:1616513070.303740:0:172819:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:25.0:1616513070.303742:0:172819:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:25.0:1616513070.303747:0:172819:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:15.2:1616513070.303751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:9.0:1616513070.303759:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:9.0:1616513070.303763:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87e8600] (23)-- 00000400:00000200:9.0:1616513070.303765:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513070.303766:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000400:00000200:9.0:1616513070.303771:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.151@o2ib1: PUT: OK 00000400:00000200:9.0:1616513070.303774:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492673ba0 00000800:00000200:9.0:1616513070.303778:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000800:00000200:7.2:1616513070.308504:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513070.308560:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.308570:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513070.308579:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.25@tcp1 : PUT - for me 00000400:00000200:26.0:1616513070.308588:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.25@tcp1 of length 224 into portal 26 MB=0x6018cb645ecc0 00000400:00000200:26.0:1616513070.308595:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.25@tcp1 of length 224/224 into md 0xf97c3d69 [1] + 0 00000400:00000200:26.0:1616513070.308600:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777eaa0 00000400:00000200:26.0:1616513070.308603:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513070.308606:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513070.308611:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eaa0 00000800:00000200:26.0:1616513070.308614:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.308615:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.308617:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513070.308619:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.308677:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690553725349056 00010000:00000200:23.0:1616513070.308688:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1690553725349056/t0(0) o400->52b5d358-b101-319c-992a-a3c6a6002b4f@10.31.142.25@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.308702:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690553725349056, offset 192 00000400:00000200:23.0:1616513070.308708:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.25@tcp1 00000400:00000200:23.0:1616513070.308716:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.25@tcp1 routed destination 00000400:00000200:23.0:1616513070.308732:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.25@tcp1(10.31.142.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.308738:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.308744:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.308747:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.308749:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.308754:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.308759:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.308761:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513070.308813:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.308820:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.308824:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513070.308824:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513070.308830:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513070.308836:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0770 00000800:00000200:2.0:1616513070.308840:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.308949:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513070.308960:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.308968:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513070.308969:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513070.308977:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.25@tcp1 : PUT - for me 00000400:00000200:10.0:1616513070.308986:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.25@tcp1 of length 224 into portal 12 MB=0x6018cb645fa40 00000400:00000200:10.0:1616513070.308992:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.25@tcp1 of length 224/224 into md 0xf96450b9 [1] + 12008 00000400:00000200:10.0:1616513070.308997:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.309000:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513070.309007:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.309010:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.309012:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.309070:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690553725352512 00010000:00000200:14.0:1616513070.309081:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdec00 x1690553725352512/t0(0) o400->58e01dbd-4302-3970-4915-4d847119565a@10.31.142.25@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.309095:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690553725352512, offset 192 00000400:00000200:14.0:1616513070.309103:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.25@tcp1 00000400:00000200:14.0:1616513070.309110:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.25@tcp1 routed destination 00000400:00000200:14.0:1616513070.309124:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.25@tcp1(10.31.142.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.309130:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513070.309136:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.309138:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513070.309140:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513070.309145:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.309152:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.309207:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.309213:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513070.309216:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513070.309222:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513070.309223:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513070.309227:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47573a8 00000800:00000200:20.0:1616513070.309231:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.318679:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.318690:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.318698:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.318706:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.85@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.318716:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.85@tcp1 of length 224 into portal 26 MB=0x603705c54da80 00000400:00000200:30.0:1616513070.318723:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.11.85@tcp1 of length 224/224 into md 0xf97c3d79 [1] + 0 00000400:00000200:30.0:1616513070.318727:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e330 00000400:00000200:30.0:1616513070.318730:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.318733:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513070.318737:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e330 00000800:00000200:30.0:1616513070.318740:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.318742:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.318743:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.318746:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.318803:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692630980549248 00010000:00000200:23.0:1616513070.318814:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1692630980549248/t0(0) o400->ec8609f7-c204-68a3-3419-5ada61f72445@10.31.11.85@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.318827:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692630980549248, offset 192 00000400:00000200:23.0:1616513070.318834:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.85@tcp1 00000400:00000200:23.0:1616513070.318843:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.85@tcp1 routed destination 00000400:00000200:23.0:1616513070.318857:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.85@tcp1(10.31.11.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.318864:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.318869:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.318872:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.318874:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.318879:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.318883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.318885:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513070.318923:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513070.318937:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.318944:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513070.318947:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513070.318954:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513070.318960:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0b28 00000800:00000200:26.0:1616513070.318964:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.318967:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513070.318978:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.318988:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513070.318997:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.85@tcp1 : PUT - for me 00000400:00000200:16.0:1616513070.319006:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.85@tcp1 of length 224 into portal 12 MB=0x603705c54dd00 00000400:00000200:16.0:1616513070.319013:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.11.85@tcp1 of length 224/224 into md 0x865b9f31 [1] + 42880 00000400:00000200:16.0:1616513070.319018:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513070.319020:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513070.319030:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.319032:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.319033:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.319036:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513070.319095:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692630980549888 00010000:00000200:20.0:1616513070.319107:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e5e80 x1692630980549888/t0(0) o400->1e27a69f-11d8-a575-1ac7-d9e5b187f3dd@10.31.11.85@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513070.319122:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692630980549888, offset 192 00000400:00000200:20.0:1616513070.319128:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.85@tcp1 00000400:00000200:20.0:1616513070.319136:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.85@tcp1 routed destination 00000400:00000200:20.0:1616513070.319152:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.85@tcp1(10.31.11.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513070.319158:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513070.319164:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513070.319167:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513070.319171:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.319176:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.319181:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513070.319234:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.319241:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513070.319245:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513070.319245:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513070.319252:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513070.319256:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23e48 00000800:00000200:28.0:1616513070.319260:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.332162:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513070.332222:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.332232:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513070.332233:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.332241:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513070.332241:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.132@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.332248:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.132@tcp1 : PUT - for me 00000400:00000200:2.0:1616513070.332251:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.132@tcp1 of length 224 into portal 26 MB=0x6054e714673c0 00000400:00000200:30.0:1616513070.332256:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.132@tcp1 of length 224 into portal 12 MB=0x6054e71467440 00000400:00000200:2.0:1616513070.332258:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.132@tcp1 of length 224/224 into md 0xf97c3d89 [1] + 0 00000400:00000200:2.0:1616513070.332263:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ecc0 00000400:00000200:2.0:1616513070.332266:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.332267:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.132@tcp1 of length 224/224 into md 0xf96450b9 [1] + 12232 00000400:00000200:30.0:1616513070.332272:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513070.332272:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513070.332277:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ecc0 00000100:00000200:30.0:1616513070.332280:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513070.332280:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513070.332283:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513070.332285:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513070.332286:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513070.332289:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513070.332289:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.332291:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.332341:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694684326294464 00000100:00000200:14.0:1616513070.332350:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694684326294592 00010000:00000200:23.0:1616513070.332351:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1694684326294464/t0(0) o400->f6fb160c-b491-6ee4-532a-6a0b5a921e01@10.31.140.132@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:14.0:1616513070.332362:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38e780 x1694684326294592/t0(0) o400->3e89408a-e517-c326-66f6-f59d10e7d16e@10.31.140.132@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:23.0:1616513070.332365:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694684326294464, offset 192 00000400:00000200:23.0:1616513070.332371:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.132@tcp1 00000100:00000200:14.0:1616513070.332375:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694684326294592, offset 192 00000400:00000200:23.0:1616513070.332379:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.132@tcp1 routed destination 00000400:00000200:14.0:1616513070.332381:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.132@tcp1 00000400:00000200:14.0:1616513070.332389:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.132@tcp1 routed destination 00000400:00000200:23.0:1616513070.332405:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.132@tcp1(10.31.140.132@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513070.332405:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.132@tcp1(10.31.140.132@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.332412:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.332413:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513070.332418:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.332420:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.332421:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.332422:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.332423:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.332425:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513070.332427:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:23.0:1616513070.332429:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.332437:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.332449:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:23.0:1616513070.332455:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513070.332456:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513070.332459:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513070.332459:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513070.332466:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513070.332470:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513070.332471:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1ba0 00000800:00000200:10.0:1616513070.332475:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513070.332527:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.332534:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.332537:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.332543:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.332547:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757b18 00000800:00000200:12.0:1616513070.332551:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.332554:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.362740:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513070.362796:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.362805:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513070.362807:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513070.362813:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.84@tcp1 : PUT - for me 00000800:00000200:30.0:1616513070.362815:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513070.362822:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.84@tcp1 of length 224 into portal 26 MB=0x5ffdc4d9f5940 00000400:00000200:30.0:1616513070.362824:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.84@tcp1 : PUT - for me 00000400:00000200:2.0:1616513070.362829:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.84@tcp1 of length 224/224 into md 0xf97c3d99 [1] + 0 00000400:00000200:30.0:1616513070.362832:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.84@tcp1 of length 224 into portal 12 MB=0x5ffdc4d9f59c0 00000400:00000200:2.0:1616513070.362833:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f210 00000400:00000200:2.0:1616513070.362836:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.362837:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.84@tcp1 of length 224/224 into md 0xf96450b9 [1] + 12456 00000400:00000200:30.0:1616513070.362843:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513070.362843:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513070.362848:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f210 00000100:00000200:30.0:1616513070.362850:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513070.362851:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513070.362853:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513070.362856:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513070.362856:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.362859:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513070.362860:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.362861:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.362917:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696543730112 00000100:00000200:23.0:1616513070.362925:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696543729984 00010000:00000200:14.0:1616513070.362928:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38ec00 x1688696543730112/t0(0) o400->e20e24b8-de18-e35c-64d5-47cc77507493@10.31.141.84@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:23.0:1616513070.362935:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688696543729984/t0(0) o400->2a1a7baa-40b4-7e3b-c427-fc9bafb0974a@10.31.141.84@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:14.0:1616513070.362941:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696543730112, offset 192 00000100:00000200:23.0:1616513070.362948:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696543729984, offset 192 00000400:00000200:14.0:1616513070.362948:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.84@tcp1 00000400:00000200:14.0:1616513070.362954:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.84@tcp1 routed destination 00000400:00000200:23.0:1616513070.362955:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.84@tcp1 00000400:00000200:23.0:1616513070.362962:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.84@tcp1 routed destination 00000400:00000200:14.0:1616513070.362968:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.84@tcp1(10.31.141.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.362975:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:23.0:1616513070.362977:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.84@tcp1(10.31.141.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.362981:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.362983:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513070.362985:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.362987:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.362991:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.362991:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:23.0:1616513070.362993:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.362995:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513070.362996:0:48427:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.363000:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:23.0:1616513070.363006:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513070.363010:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.363016:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513070.363019:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513070.363021:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513070.363025:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513070.363028:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756e58 00000800:00000200:10.0:1616513070.363033:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.363043:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.363113:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.363120:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513070.363123:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513070.363129:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513070.363135:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1430 00000800:00000200:20.0:1616513070.363138:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.363141:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513070.369872:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513070.369930:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.369940:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513070.369949:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.11@tcp1 : PUT - for me 00000400:00000200:12.0:1616513070.369959:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.11@tcp1 of length 328 into portal 17 MB=0x603d0eaed2800 00000400:00000200:12.0:1616513070.369966:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 11 from 12345-10.31.21.11@tcp1 of length 328/328 into md 0x83e1be81 [1] + 1648 00000400:00000200:12.0:1616513070.369973:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513070.369976:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service ldlm_canceld 00000800:00000200:16.0:1616513070.369985:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.369985:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.369988:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513070.369990:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513070.370054:0:163458:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693045689755648 00000100:00000200:20.0:1616513070.370076:0:163458:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 18, xid 1693045689755648, offset 192 00000400:00000200:20.0:1616513070.370083:0:163458:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.11@tcp1 00000400:00000200:20.0:1616513070.370092:0:163458:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.11@tcp1 routed destination 00000400:00000200:20.0:1616513070.370107:0:163458:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.11@tcp1(10.31.21.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513070.370114:0:163458:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513070.370120:0:163458:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513070.370123:0:163458:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513070.370125:0:163458:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513070.370130:0:163458:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.370137:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.370193:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.370200:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513070.370203:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513070.370204:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513070.370209:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513070.370213:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22088 00000800:00000200:2.0:1616513070.370217:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.373093:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513070.373105:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.373111:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513070.373118:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.11@tcp1 : PUT - for me 00000400:00000200:20.0:1616513070.373126:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.11@tcp1 of length 328 into portal 17 MB=0x603d0eaed47c0 00000400:00000200:20.0:1616513070.373132:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 11 from 12345-10.31.21.11@tcp1 of length 328/328 into md 0x83e1be81 [1] + 1976 00000400:00000200:20.0:1616513070.373136:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82707e4c38 00000400:00000200:20.0:1616513070.373139:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513070.373142:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service ldlm_canceld 00000400:00000200:20.0:1616513070.373153:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82707e4c38 00000800:00000200:20.0:1616513070.373156:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.373159:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513070.373160:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.373165:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.373223:0:163458:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693045689763776 00000100:00000200:24.0:1616513070.373242:0:163458:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 18, xid 1693045689763776, offset 192 00000400:00000200:24.0:1616513070.373249:0:163458:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.11@tcp1 00000400:00000200:24.0:1616513070.373257:0:163458:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.11@tcp1 routed destination 00000400:00000200:24.0:1616513070.373272:0:163458:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.11@tcp1(10.31.21.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.373279:0:163458:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513070.373284:0:163458:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.373287:0:163458:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513070.373289:0:163458:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513070.373294:0:163458:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.373299:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513070.373309:0:163458:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 17 00000800:00000200:16.0:1616513070.373311:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.373319:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.373321:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.373328:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.373334:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:16.0:1616513070.373337:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.373340:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:29.2F:1616513070.377230:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa1000] (20)++ 00000800:00000200:20.0:1616513070.377245:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (21)++ 00000800:00000200:20.0:1616513070.377250:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.204@o2ib1 00000400:00000200:20.0:1616513070.377257:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.204@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513070.377262:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.204@o2ib1 of length 224 into portal 26 MB=0x60386ef461440 00000400:00000200:20.0:1616513070.377268:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.204@o2ib1 of length 224/224 into md 0xf97c3da9 [1] + 0 00000400:00000200:20.0:1616513070.377272:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e4c8 00000400:00000200:20.0:1616513070.377273:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513070.377276:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513070.377280:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e4c8 00000800:00000200:20.0:1616513070.377282:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa1000] (22)++ 00000800:00000200:20.0:1616513070.377284:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa1000] (23)-- 00000800:00000200:20.0:1616513070.377286:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (22)-- 00000800:00000200:20.0:1616513070.377288:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (21)++ 00000800:00000200:20.0:1616513070.377292:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.204@o2ib1 00000400:00000200:20.0:1616513070.377299:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.204@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513070.377303:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.204@o2ib1 of length 224 into portal 12 MB=0x60386ef461540 00000400:00000200:20.0:1616513070.377309:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.204@o2ib1 of length 224/224 into md 0x865b9f31 [1] + 43104 00000800:00000200:28.0:1616513070.377312:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (22)-- 00000400:00000200:20.0:1616513070.377312:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513070.377314:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513070.377319:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa1000] (21)++ 00000800:00000200:20.0:1616513070.377321:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa1000] (22)-- 00000800:00000200:20.0:1616513070.377322:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (21)-- 00000100:00000200:24.0:1616513070.377334:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692727935112512 00010000:00000200:24.0:1616513070.377343:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceba80 x1692727935112512/t0(0) o400->a639e6da-5738-1ba4-ca02-11389207dd66@10.242.46.204@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:23.0:1616513070.377348:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692727935112256 00000100:00000200:24.0:1616513070.377355:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692727935112512, offset 192 00010000:00000200:23.0:1616513070.377359:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1692727935112256/t0(0) o400->c7497b2f-7207-d905-9584-552f06d0affa@10.242.46.204@o2ib1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:24.0:1616513070.377360:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.204@o2ib1 00000400:00000200:24.0:1616513070.377366:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.204@o2ib1 local destination 00000100:00000200:23.0:1616513070.377374:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692727935112256, offset 192 00000400:00000200:24.0:1616513070.377378:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.204@o2ib1(10.242.46.204@o2ib1:10.242.46.204@o2ib1) : PUT try# 0 00000400:00000200:23.0:1616513070.377380:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.204@o2ib1 00000800:00000200:24.0:1616513070.377384:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.204@o2ib1 00000400:00000200:23.0:1616513070.377388:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.204@o2ib1 local destination 00000800:00000200:24.0:1616513070.377389:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6600] -> 10.242.46.204@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.377391:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa1000] (20)++ 00000800:00000200:24.0:1616513070.377393:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa1000] (21)++ 00000800:00000200:24.0:1616513070.377396:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa1000] (22)-- 00000800:00000200:29.2:1616513070.377401:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa1000] (21)++ 00000400:00000200:23.0:1616513070.377402:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.204@o2ib1(10.242.46.204@o2ib1:10.242.46.204@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.377409:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.204@o2ib1 00000800:00000200:23.0:1616513070.377416:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6600] -> 10.242.46.204@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.377418:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa1000] (22)++ 00000800:00000200:23.0:1616513070.377420:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa1000] (23)++ 00000800:00000200:23.0:1616513070.377426:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa1000] (24)-- 00000100:00000200:23.0:1616513070.377433:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513070.377456:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (23)++ 00000800:00000200:12.0:1616513070.377464:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa1000] (24)-- 00000800:00000200:28.0:1616513070.377467:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (23)++ 00000400:00000200:12.0:1616513070.377467:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.377473:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.204@o2ib1: PUT: OK 00000800:00000200:28.0:1616513070.377474:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa1000] (24)-- 00000400:00000200:28.0:1616513070.377476:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513070.377476:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (23)-- 00000400:00000200:12.0:1616513070.377477:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ba0 00000800:00000200:12.0:1616513070.377481:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (22)-- 00000400:00000200:28.0:1616513070.377483:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.204@o2ib1: PUT: OK 00000400:00000200:28.0:1616513070.377489:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0ff0 00000800:00000200:28.0:1616513070.377492:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (21)-- 00000800:00000200:11.2:1616513070.387148:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513070.387204:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.387214:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513070.387222:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:16.0:1616513070.387232:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d03229b140 00000400:00000200:16.0:1616513070.387238:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0x865b9f31 [1] + 43328 00000400:00000200:16.0:1616513070.387244:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513070.387247:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513070.387253:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.387256:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.387256:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.387260:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.387319:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042589937984 00010000:00000200:24.0:1616513070.387404:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e1b00 x1693042589937984/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:696/0 lens 440/368 e 0 to 0 dl 1616513076 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513070.387417:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042589937984, offset 192 00000400:00000200:24.0:1616513070.387423:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:24.0:1616513070.387432:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:24.0:1616513070.387458:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.387464:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513070.387470:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.387473:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513070.387474:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513070.387479:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.387487:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.387544:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.387551:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513070.387554:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513070.387561:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513070.387565:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2a18 00000800:00000200:26.0:1616513070.387569:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.387572:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.387671:0:48429:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.387681:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.387688:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.387695:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.11@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.387702:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.11@tcp1 of length 328 into portal 17 MB=0x603d0eaed6500 00000400:00000200:30.0:1616513070.387707:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 11 from 12345-10.31.21.11@tcp1 of length 328/328 into md 0xf40e28f9 [1] + 1320 00000400:00000200:30.0:1616513070.387712:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.387714:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service ldlm_canceld 00000800:00000200:30.0:1616513070.387720:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.387722:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.387724:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.387726:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513070.387746:0:6714:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693045689771264 00000100:00000200:10.0:1616513070.387764:0:6714:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 18, xid 1693045689771264, offset 192 00000400:00000200:10.0:1616513070.387771:0:6714:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.11@tcp1 00000400:00000200:10.0:1616513070.387778:0:6714:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.11@tcp1 routed destination 00000400:00000200:10.0:1616513070.387792:0:6714:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.11@tcp1(10.31.21.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513070.387799:0:6714:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513070.387804:0:6714:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513070.387807:0:6714:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513070.387809:0:6714:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513070.387814:0:6714:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.387819:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.387897:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.387913:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.387915:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.387922:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.387926:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82d48 00000800:00000200:12.0:1616513070.387930:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.387933:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.389256:0:48429:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513070.389309:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.389326:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513070.389327:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513070.389335:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.11@tcp1 : PUT - for me 00000400:00000200:26.0:1616513070.389349:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.11@tcp1 of length 328 into portal 17 MB=0x603d0eaed5740 00000400:00000200:26.0:1616513070.389356:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 11 from 12345-10.31.21.11@tcp1 of length 328/328 into md 0xf40e28f9 [1] + 1648 00000400:00000200:26.0:1616513070.389361:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513070.389364:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service ldlm_canceld 00000800:00000200:26.0:1616513070.389371:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.389374:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.389376:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513070.389381:0:6714:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693045689767744 00000100:00000200:10.0:1616513070.389390:0:6714:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 18, xid 1693045689767744, offset 192 00000400:00000200:10.0:1616513070.389395:0:6714:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.11@tcp1 00000400:00000200:10.0:1616513070.389400:0:6714:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.11@tcp1 routed destination 00000400:00000200:10.0:1616513070.389412:0:6714:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.11@tcp1(10.31.21.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513070.389417:0:6714:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513070.389421:0:6714:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513070.389424:0:6714:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513070.389425:0:6714:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.389428:0:6714:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.389432:0:48429:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.389444:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.389447:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.389449:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.389454:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.389456:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82770 00000800:00000200:30.0:1616513070.389459:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.389461:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.390978:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513070.391032:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.391041:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513070.391050:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.133@tcp1 : PUT - for me 00000400:00000200:20.0:1616513070.391059:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.133@tcp1 of length 224 into portal 26 MB=0x60219fb76f440 00000400:00000200:20.0:1616513070.391066:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.133@tcp1 of length 224/224 into md 0xf97c3db9 [1] + 0 00000400:00000200:20.0:1616513070.391071:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fed0 00000400:00000200:20.0:1616513070.391073:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513070.391076:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513070.391083:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fed0 00000800:00000200:16.0:1616513070.391085:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.391086:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.391089:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513070.391090:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.391143:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691160476578880 00010000:00000200:23.0:1616513070.391152:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1691160476578880/t0(0) o400->1221d4bc-adbd-6b0a-197d-e2e1cbe87cb2@10.31.140.133@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.391164:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691160476578880, offset 192 00000400:00000200:23.0:1616513070.391169:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.133@tcp1 00000400:00000200:23.0:1616513070.391176:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.133@tcp1 routed destination 00000800:00000200:7.2:1616513070.391188:0:48429:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:23.0:1616513070.391189:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.133@tcp1(10.31.140.133@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.391194:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513070.391199:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.391200:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.391202:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.391204:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513070.391204:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.391208:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513070.391211:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.133@tcp1 : PUT - for me 00000800:00000200:11.2:1616513070.391213:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.391215:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:10.0:1616513070.391219:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.133@tcp1 of length 224 into portal 12 MB=0x60219fb76f4c0 00000400:00000200:10.0:1616513070.391225:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.133@tcp1 of length 224/224 into md 0xf96450b9 [1] + 12680 00000400:00000200:10.0:1616513070.391230:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.391233:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513070.391238:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.391243:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513070.391244:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.391247:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513070.391270:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.391277:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.391291:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513070.391293:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.391299:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.391305:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0330 00000800:00000200:12.0:1616513070.391308:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513070.391324:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691160476579008 00010000:00000200:14.0:1616513070.391336:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bd9f80 x1691160476579008/t0(0) o400->f44ba8ed-ba10-f6db-ec3e-c3c1dad4240f@10.31.140.133@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.391351:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691160476579008, offset 192 00000400:00000200:14.0:1616513070.391357:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.133@tcp1 00000400:00000200:14.0:1616513070.391371:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.133@tcp1 routed destination 00000400:00000200:14.0:1616513070.391386:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.133@tcp1(10.31.140.133@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.391393:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513070.391399:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.391402:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.391404:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513070.391408:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.391415:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.391423:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.391426:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.391427:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.391432:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.391434:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757650 00000800:00000200:30.0:1616513070.391444:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.391446:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.395570:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513070.395625:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.395635:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513070.395636:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513070.395644:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.11@tcp1 : PUT - for me 00000400:00000200:26.0:1616513070.395653:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.11@tcp1 of length 328 into portal 17 MB=0x603d0eaed3900 00000400:00000200:26.0:1616513070.395659:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 11 from 12345-10.31.21.11@tcp1 of length 328/328 into md 0xf40e28f9 [1] + 1976 00000400:00000200:26.0:1616513070.395664:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82697b6d48 00000400:00000200:26.0:1616513070.395667:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513070.395669:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service ldlm_canceld 00000400:00000200:26.0:1616513070.395674:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82697b6d48 00000800:00000200:26.0:1616513070.395677:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.395680:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.395682:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513070.395686:0:6714:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693045689760000 00000100:00000200:10.0:1616513070.395695:0:6714:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 18, xid 1693045689760000, offset 192 00000400:00000200:10.0:1616513070.395700:0:6714:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.11@tcp1 00000400:00000200:10.0:1616513070.395705:0:6714:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.11@tcp1 routed destination 00000400:00000200:10.0:1616513070.395717:0:6714:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.11@tcp1(10.31.21.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513070.395722:0:6714:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513070.395727:0:6714:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513070.395729:0:6714:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513070.395730:0:6714:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513070.395733:0:6714:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.395741:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:10.0:1616513070.395744:0:6714:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 17 00000800:00000200:16.0:1616513070.395794:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.395801:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.395803:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.395810:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.395815:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83078 00000800:00000200:16.0:1616513070.395818:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.395821:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.399264:0:48430:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.399275:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.399281:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.399288:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.157@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.399310:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.157@tcp1 of length 224 into portal 26 MB=0x60183d6126f80 00000400:00000200:30.0:1616513070.399315:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.157@tcp1 of length 224/224 into md 0xf97c3dc9 [1] + 0 00000400:00000200:30.0:1616513070.399319:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fb18 00000400:00000200:30.0:1616513070.399320:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.399322:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513070.399326:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fb18 00000800:00000200:30.0:1616513070.399328:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.399330:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.399331:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.399333:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.399392:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515604139904 00010000:00000200:23.0:1616513070.399403:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1690515604139904/t0(0) o400->136f8dc8-1996-ce89-d5ef-eac6cd12050f@10.31.132.157@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.399417:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690515604139904, offset 192 00000400:00000200:23.0:1616513070.399423:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.157@tcp1 00000400:00000200:23.0:1616513070.399432:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.157@tcp1 routed destination 00000400:00000200:23.0:1616513070.399461:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.157@tcp1(10.31.132.157@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.399467:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.399474:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.399477:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.399479:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.399484:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.399488:0:48430:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.399491:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513070.399497:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.399501:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513070.399504:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513070.399509:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513070.399513:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0ee0 00000800:00000200:10.0:1616513070.399516:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.399518:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.399890:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513070.399902:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513070.399911:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513070.399920:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.157@tcp1 : PUT - for me 00000400:00000200:28.0:1616513070.399932:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.157@tcp1 of length 224 into portal 12 MB=0x60183d6127140 00000400:00000200:28.0:1616513070.399938:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.157@tcp1 of length 224/224 into md 0x865b9f31 [1] + 43768 00000400:00000200:28.0:1616513070.399944:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.399946:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513070.399953:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.399956:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.399958:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.399961:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.399965:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515604140352 00010000:00000200:24.0:1616513070.399972:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8d580 x1690515604140352/t0(0) o400->8042b538-35a2-7745-cfca-ec0361f3c70d@10.31.132.157@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513070.399982:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690515604140352, offset 192 00000400:00000200:24.0:1616513070.399987:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.157@tcp1 00000400:00000200:24.0:1616513070.399993:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.157@tcp1 routed destination 00000400:00000200:24.0:1616513070.400005:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.157@tcp1(10.31.132.157@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.400011:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513070.400015:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.400017:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513070.400019:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513070.400022:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.400027:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.400079:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.400086:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.400088:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.400095:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.400099:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ed0 00000800:00000200:16.0:1616513070.400104:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.400107:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.403135:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.403148:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513070.403149:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513070.403157:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.403166:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.164@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.403175:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.164@tcp1 of length 224 into portal 26 MB=0x5ffdc6cdbdf40 00000400:00000200:30.0:1616513070.403181:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.164@tcp1 of length 224/224 into md 0xf97c3dd9 [1] + 0 00000400:00000200:30.0:1616513070.403185:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f320 00000400:00000200:30.0:1616513070.403188:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.403191:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513070.403195:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f320 00000800:00000200:30.0:1616513070.403198:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.403201:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513070.403202:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513070.403203:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.403205:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513070.403211:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513070.403220:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.164@tcp1 : PUT - for me 00000400:00000200:28.0:1616513070.403237:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.164@tcp1 of length 224 into portal 12 MB=0x5ffdc6cdbe240 00000400:00000200:28.0:1616513070.403243:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.164@tcp1 of length 224/224 into md 0x865b9f31 [1] + 43992 00000400:00000200:28.0:1616513070.403249:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.403251:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513070.403257:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.403260:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:23.0:1616513070.403263:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697067790144 00000800:00000200:28.0:1616513070.403264:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.403267:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:23.0:1616513070.403274:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688697067790144/t0(0) o400->287f5d4f-24c1-db14-6fd1-550fb92932e3@10.31.141.164@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.403288:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697067790144, offset 192 00000400:00000200:23.0:1616513070.403294:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.164@tcp1 00000400:00000200:23.0:1616513070.403302:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.164@tcp1 routed destination 00000400:00000200:23.0:1616513070.403318:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.164@tcp1(10.31.141.164@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513070.403319:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697067790912 00000800:00000200:23.0:1616513070.403324:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:24.0:1616513070.403330:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8ba80 x1688697067790912/t0(0) o400->7d0e0437-0943-5ae3-4b94-06ba57c982ea@10.31.141.164@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513070.403331:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.403334:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.403336:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.403341:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513070.403342:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697067790912, offset 192 00000800:00000200:7.2:1616513070.403345:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513070.403347:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.164@tcp1 00000100:00000200:23.0:1616513070.403348:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513070.403353:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513070.403354:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.164@tcp1 routed destination 00000800:00000200:10.0:1616513070.403357:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513070.403359:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513070.403364:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:24.0:1616513070.403367:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.164@tcp1(10.31.141.164@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513070.403367:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0e58 00000800:00000200:10.0:1616513070.403369:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.403371:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513070.403373:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513070.403379:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.403381:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513070.403385:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513070.403389:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.403394:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.403449:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.403455:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.403458:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.403464:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.403468:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:16.0:1616513070.403471:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.403474:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513070.426578:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923840, portal 4 00000100:00000200:4.0:1616513070.426584:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923840, offset 0 00000400:00000200:4.0:1616513070.426590:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:4.0:1616513070.426597:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:4.0:1616513070.426610:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513070.426618:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:4.0:1616513070.426625:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513070.426628:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:4.0:1616513070.426630:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:4.0:1616513070.426636:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513070.426683:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.426703:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513070.426709:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513070.426712:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513070.426719:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513070.426727:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d846aa41200 x1693067725923840/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513070.426740:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000800:00000200:1.0:1616513070.426745:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.426748:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513070.426863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513070.426873:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.426888:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513070.426896:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513070.426905:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626600 00000400:00000200:1.0:1616513070.426914:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff795 [1] + 192 00000400:00000200:1.0:1616513070.426920:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513070.426924:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d846aa41200 x1693067725923840/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:9.0:1616513070.426932:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.426939:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.426942:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.426944:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:4.0:1616513070.426949:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d846aa41200 x1693067725923840/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513070.426962:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d846aa41200 x1693067725923840/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513070.426972:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbcb0 00000800:00000200:1.2:1616513070.427987:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513070.427995:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.428000:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513070.428007:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513070.428013:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 26 MB=0x5f931434ff0b0 00000400:00000200:1.0:1616513070.428019:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0xf97c3de9 [1] + 0 00000400:00000200:1.0:1616513070.428023:0:6465:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e088 00000400:00000200:1.0:1616513070.428025:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513070.428027:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:1.0:1616513070.428032:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:1.0:1616513070.428035:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513070.428037:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513070.428038:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.428040:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.428045:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513070.428046:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:1.0:1616513070.428051:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:9.0:1616513070.428056:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000400:00000200:1.0:1616513070.428056:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff0f0 00000800:00000200:5.0:1616513070.428058:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513070.428061:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 34704 00000800:00000200:9.0:1616513070.428062:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513070.428065:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:5.0:1616513070.428067:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000100:00000200:1.0:1616513070.428067:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:9.0:1616513070.428070:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:1.0:1616513070.428073:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (24)++ 00000400:00000200:5.0:1616513070.428075:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff110 00000800:00000200:1.0:1616513070.428075:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)-- 00000800:00000200:1.0:1616513070.428077:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)-- 00000400:00000200:9.0:1616513070.428078:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff130 00000800:00000200:1.0:1616513070.428079:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000400:00000200:5.0:1616513070.428082:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 34928 00000800:00000200:1.0:1616513070.428082:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:5.0:1616513070.428087:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:9.0:1616513070.428089:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 35152 00000400:00000200:1.0:1616513070.428089:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:9.0:1616513070.428094:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:5.0:1616513070.428094:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:23.0:1616513070.428095:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579440 00000400:00000200:1.0:1616513070.428095:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff170 00000100:00000200:9.0:1616513070.428102:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:5.0:1616513070.428103:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (24)++ 00010000:00000200:23.0:1616513070.428106:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1681364861579440/t0(0) o400->023d13a7-5961-d0c8-eb71-c4c0fa9348a4@10.242.46.232@o2ib1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:13.0:1616513070.428106:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000800:00000200:5.0:1616513070.428108:0:6468:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:5.0:1616513070.428112:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:13.0:1616513070.428113:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513070.428114:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (26)-- 00000800:00000200:9.0:1616513070.428115:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:5.0:1616513070.428116:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)++ 00000400:00000200:1.0:1616513070.428116:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 35376 00000100:00000200:23.0:1616513070.428119:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681364861579440, offset 192 00000800:00000200:9.0:1616513070.428119:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:5.0:1616513070.428119:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513070.428119:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:13.0:1616513070.428121:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:9.0:1616513070.428121:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (26)-- 00000100:00000200:1.0:1616513070.428121:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:9.0:1616513070.428123:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:9.0:1616513070.428125:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (26)-- 00000400:00000200:5.0:1616513070.428126:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:23.0:1616513070.428127:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:9.0:1616513070.428127:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000800:00000200:1.0:1616513070.428128:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)++ 00000400:00000200:13.0:1616513070.428129:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff1b0 00000800:00000200:9.0:1616513070.428129:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)-- 00000800:00000200:1.0:1616513070.428130:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:5.0:1616513070.428132:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff190 00000800:00000200:1.0:1616513070.428132:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:23.0:1616513070.428133:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:13.0:1616513070.428135:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 35600 00000800:00000200:1.2:1616513070.428135:0:6465:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:13.0:1616513070.428140:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:5.0:1616513070.428142:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 35824 00000800:00000200:9.0:1616513070.428143:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000400:00000200:23.0:1616513070.428146:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000100:00000200:13.0:1616513070.428146:0:6467:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:5.0:1616513070.428146:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:9.0:1616513070.428150:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:23.0:1616513070.428153:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:9.0:1616513070.428156:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000100:00000200:5.0:1616513070.428157:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:13.0:1616513070.428158:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (24)++ 00000800:00000200:23.0:1616513070.428159:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.428161:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:13.0:1616513070.428161:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:9.0:1616513070.428162:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff210 00000800:00000200:23.0:1616513070.428163:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:13.0:1616513070.428163:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000800:00000200:13.0:1616513070.428166:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:5.0:1616513070.428167:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (26)++ 00000800:00000200:23.0:1616513070.428168:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (26)-- 00000400:00000200:9.0:1616513070.428169:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 36048 00000800:00000200:13.0:1616513070.428170:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513070.428170:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (26)-- 00000800:00000200:5.0:1616513070.428171:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:9.0:1616513070.428173:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:5.0:1616513070.428173:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000100:00000200:23.0:1616513070.428174:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:9.0:1616513070.428175:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:13.0:1616513070.428176:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:5.0:1616513070.428177:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000100:00000200:17.0F:1616513070.428178:0:170084:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579504 00000400:00000200:13.0:1616513070.428182:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff0d0 00000400:00000200:5.0:1616513070.428183:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:9.0:1616513070.428184:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)++ 00000400:00000200:13.0:1616513070.428187:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 36272 00000800:00000200:9.0:1616513070.428187:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (26)-- 00000800:00000200:9.0:1616513070.428188:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:5.0:1616513070.428189:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff1f0 00000100:00000200:1.0:1616513070.428189:0:170083:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579536 00000400:00000200:13.0:1616513070.428190:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:9.0:1616513070.428190:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000100:00000200:29.0F:1616513070.428192:0:170519:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579568 00000100:00000200:21.0F:1616513070.428194:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579632 00010000:00000200:17.0:1616513070.428194:0:170084:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d853e8cda00 x1681364861579504/t0(0) o400->boslfs02-MDT0001-lwp-OST001b_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:9.0:1616513070.428194:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:5.0:1616513070.428194:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 36496 00000100:00000200:25.0:1616513070.428195:0:172819:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579664 00000100:00000200:13.0:1616513070.428197:0:6467:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:5.0:1616513070.428197:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:9.0:1616513070.428201:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00010000:00000200:1.0:1616513070.428201:0:170083:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843f7bbf00 x1681364861579536/t0(0) o400->boslfs02-MDT0001-lwp-OST0023_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:13.0:1616513070.428202:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)++ 00000100:00000200:5.0:1616513070.428202:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:13.0:1616513070.428204:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (26)-- 00010000:00000200:29.0:1616513070.428206:0:170519:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4558ad5100 x1681364861579568/t0(0) o400->boslfs02-MDT0001-lwp-OST0017_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:25.0:1616513070.428206:0:172819:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843f7bc380 x1681364861579664/t0(0) o400->boslfs02-MDT0001-lwp-OST002f_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:13.0:1616513070.428206:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:9.0:1616513070.428206:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff230 00000800:00000200:5.0:1616513070.428207:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:13.0:1616513070.428208:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00010000:00000200:21.0:1616513070.428209:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d853e8ce780 x1681364861579632/t0(0) o400->boslfs02-MDT0001-lwp-OST002b_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:17.0:1616513070.428209:0:170084:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579504, offset 192 00000800:00000200:5.0:1616513070.428210:0:6468:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (26)++ 00000400:00000200:9.0:1616513070.428211:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 36720 00000800:00000200:5.0:1616513070.428212:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:13.0:1616513070.428213:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (27)-- 00000400:00000200:9.0:1616513070.428214:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:13.0:1616513070.428215:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513070.428216:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:17.0:1616513070.428217:0:170084:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513070.428218:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000400:00000200:13.0:1616513070.428220:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513070.428220:0:170083:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579536, offset 192 00000100:00000200:9.0:1616513070.428221:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:5.0:1616513070.428221:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:17.0:1616513070.428225:0:170084:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:13.0:1616513070.428225:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1188 00000100:00000200:29.0:1616513070.428226:0:170519:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579568, offset 192 00000100:00000200:25.0:1616513070.428227:0:172819:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579664, offset 192 00000100:00000200:21.0:1616513070.428228:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579632, offset 192 00000400:00000200:1.0:1616513070.428230:0:170083:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:29.0:1616513070.428235:0:170519:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:17.0:1616513070.428238:0:170084:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000400:00000200:25.0:1616513070.428241:0:172819:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:9.0:1616513070.428243:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:13.0:1616513070.428244:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000800:00000200:17.0:1616513070.428245:0:170084:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:9.0:1616513070.428245:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (26)-- 00000400:00000200:21.0:1616513070.428246:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:5.0:1616513070.428247:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513070.428249:0:170083:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:9.0:1616513070.428253:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)-- 00000800:00000200:13.0:1616513070.428255:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000400:00000200:1.0:1616513070.428263:0:170083:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:13.0:1616513070.428264:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:1.0:1616513070.428269:0:170083:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:29.0:1616513070.428271:0:170519:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000100:00000200:9.0:1616513070.428283:0:175777:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579696 00000400:00000200:29.0:1616513070.428284:0:170519:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513070.428290:0:170519:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:25.0:1616513070.428293:0:172819:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00010000:00000200:9.0:1616513070.428293:0:175777:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1391200 x1681364861579696/t0(0) o400->boslfs02-MDT0001-lwp-OST0027_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:25.0:1616513070.428306:0:172819:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000100:00000200:9.0:1616513070.428307:0:175777:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579696, offset 192 00000800:00000200:25.0:1616513070.428313:0:172819:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:17.0:1616513070.428313:0:170084:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:9.0:1616513070.428314:0:175777:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:21.0:1616513070.428315:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:17.0:1616513070.428316:0:170084:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (24)++ 00000800:00000200:17.0:1616513070.428318:0:170084:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:17.0:1616513070.428323:0:170084:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (26)-- 00000400:00000200:21.0:1616513070.428329:0:170085:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000100:00000200:17.0:1616513070.428334:0:170084:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579792 00000800:00000200:21.0:1616513070.428335:0:170085:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:1.0:1616513070.428337:0:170083:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513070.428338:0:170519:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00010000:00000200:17.0:1616513070.428339:0:170084:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4558ad1f80 x1681364861579792/t0(0) o400->boslfs02-MDT0001-lwp-OST0003_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:13.0:1616513070.428339:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:25.0:1616513070.428340:0:172819:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:9.0:1616513070.428340:0:175777:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:1.0:1616513070.428340:0:170083:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:29.0:1616513070.428341:0:170519:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:25.0:1616513070.428342:0:172819:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (26)++ 00000800:00000200:1.0:1616513070.428342:0:170083:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (26)++ 00000800:00000200:1.0:1616513070.428345:0:170083:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (29)-- 00000800:00000200:29.0:1616513070.428346:0:170519:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:29.0:1616513070.428350:0:170519:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (29)-- 00000100:00000200:17.0:1616513070.428350:0:170084:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579792, offset 192 00000800:00000200:25.0:1616513070.428351:0:172819:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:9.0:1616513070.428353:0:175777:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000100:00000200:1.0:1616513070.428354:0:170083:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579472 00000800:00000200:25.0:1616513070.428355:0:172819:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (29)-- 00000400:00000200:17.0:1616513070.428355:0:170084:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:9.0:1616513070.428359:0:175777:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:5.0:1616513070.428360:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff150 00010000:00000200:1.0:1616513070.428360:0:170083:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1391b00 x1681364861579472/t0(0) o400->boslfs02-MDT0001-lwp-OST003f_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:29.0:1616513070.428361:0:170519:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579760 00000800:00000200:21.0:1616513070.428361:0:170085:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:17.0:1616513070.428362:0:170084:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:21.0:1616513070.428364:0:170085:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (28)++ 00000400:00000200:5.0:1616513070.428364:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 36944 00000100:00000200:25.0:1616513070.428365:0:172819:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579824 00000800:00000200:21.0:1616513070.428366:0:170085:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (29)++ 00010000:00000200:29.0:1616513070.428367:0:170519:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843f7b9200 x1681364861579760/t0(0) o400->boslfs02-MDT0001-lwp-OST0007_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:5.0:1616513070.428367:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:5.0:1616513070.428369:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:25.0:1616513070.428370:0:172819:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4558ad0000 x1681364861579824/t0(0) o400->boslfs02-MDT0001-lwp-OST001f_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:21.0:1616513070.428370:0:170085:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)-- 00000100:00000200:1.0:1616513070.428373:0:170083:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579472, offset 192 00000400:00000200:17.0:1616513070.428375:0:170084:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000400:00000200:1.0:1616513070.428378:0:170083:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000100:00000200:29.0:1616513070.428380:0:170519:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579760, offset 192 00000800:00000200:5.0:1616513070.428380:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:17.0:1616513070.428381:0:170084:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:13.0:1616513070.428381:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff270 00000100:00000200:25.0:1616513070.428382:0:172819:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579824, offset 192 00000800:00000200:9.0:1616513070.428382:0:175777:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513070.428382:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:5.0:1616513070.428383:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)-- 00000800:00000200:9.0:1616513070.428384:0:175777:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:1.0:1616513070.428384:0:170083:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:29.0:1616513070.428385:0:170519:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513070.428385:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (28)++ 00000800:00000200:9.0:1616513070.428386:0:175777:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:13.0:1616513070.428389:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 37168 00000800:00000200:5.0:1616513070.428389:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:9.0:1616513070.428390:0:175777:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (31)-- 00000100:00000200:21.0:1616513070.428391:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579600 00000400:00000200:13.0:1616513070.428393:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:25.0:1616513070.428395:0:172819:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:1.0:1616513070.428396:0:170083:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00010000:00000200:21.0:1616513070.428397:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843f7bad00 x1681364861579600/t0(0) o400->boslfs02-MDT0001-lwp-OST0013_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:13.0:1616513070.428397:0:6467:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:17.0:1616513070.428402:0:170084:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513070.428402:0:170083:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:29.0:1616513070.428404:0:170519:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:17.0:1616513070.428405:0:170084:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:17.0:1616513070.428406:0:170084:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (31)++ 00000100:00000200:21.0:1616513070.428408:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579600, offset 192 00000800:00000200:17.0:1616513070.428409:0:170084:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:25.0:1616513070.428412:0:172819:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:5.0:1616513070.428412:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:21.0:1616513070.428413:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000100:00000200:9.0:1616513070.428413:0:170518:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579888 00000800:00000200:13.0:1616513070.428416:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:13.0:1616513070.428418:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)-- 00010000:00000200:9.0:1616513070.428418:0:170518:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1395e80 x1681364861579888/t0(0) o400->boslfs02-MDT0001-lwp-OST000b_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:1.0:1616513070.428419:0:170083:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513070.428420:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)-- 00000400:00000200:21.0:1616513070.428421:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:1.0:1616513070.428421:0:170083:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:13.0:1616513070.428422:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:1.0:1616513070.428424:0:170083:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (32)++ 00000400:00000200:5.0:1616513070.428427:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff1d0 00000800:00000200:1.0:1616513070.428427:0:170083:0:(o2iblnd_cb.c:894:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: posted enough 00000100:00000200:9.0:1616513070.428429:0:170518:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579888, offset 192 00000800:00000200:1.0:1616513070.428429:0:170083:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (33)-- 00000800:00000200:13.0:1616513070.428431:0:6467:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000400:00000200:5.0:1616513070.428432:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 37392 00000800:00000200:13.0:1616513070.428433:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:5.0:1616513070.428446:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:9.0:1616513070.428449:0:170518:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000100:00000200:5.0:1616513070.428450:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:9.0:1616513070.428454:0:170518:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:13.0:1616513070.428455:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:13.0:1616513070.428457:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:5.0:1616513070.428458:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:13.0:1616513070.428461:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513070.428462:0:6468:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:5.0:1616513070.428464:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:5.0:1616513070.428465:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:5.0:1616513070.428467:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)++ 00000100:00000200:9.0:1616513070.428469:0:175777:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579728 00000400:00000200:13.0:1616513070.428471:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:5.0:1616513070.428471:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00010000:00000200:9.0:1616513070.428474:0:175777:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843f7bd100 x1681364861579728/t0(0) o400->boslfs02-MDT0001-lwp-OST0037_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:1.0:1616513070.428474:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)++ 00000400:00000200:13.0:1616513070.428476:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff290 00000400:00000200:5.0:1616513070.428478:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:1.0:1616513070.428478:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:13.0:1616513070.428481:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 37616 00000400:00000200:5.0:1616513070.428484:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff250 00000400:00000200:13.0:1616513070.428485:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:9.0:1616513070.428485:0:175777:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579728, offset 192 00000400:00000200:1.0:1616513070.428485:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:5.0:1616513070.428488:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 37840 00000400:00000200:1.0:1616513070.428490:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff2b0 00000100:00000200:13.0:1616513070.428492:0:6467:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:5.0:1616513070.428492:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:13.0:1616513070.428498:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)++ 00000800:00000200:13.0:1616513070.428500:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (33)-- 00000400:00000200:9.0:1616513070.428500:0:175777:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513070.428502:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:1.0:1616513070.428503:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 38064 00000800:00000200:13.0:1616513070.428504:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)++ 00000400:00000200:9.0:1616513070.428504:0:175777:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:13.0:1616513070.428506:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:13.0:1616513070.428507:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:5.0:1616513070.428507:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:1.0:1616513070.428507:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:13.0:1616513070.428512:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513070.428516:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:5.0:1616513070.428517:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)++ 00000100:00000200:9.0:1616513070.428518:0:175777:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579920 00000800:00000200:5.0:1616513070.428519:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:5.0:1616513070.428521:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)-- 00010000:00000200:9.0:1616513070.428523:0:175777:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1390480 x1681364861579920/t0(0) o400->boslfs02-MDT0001-lwp-OST003b_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:5.0:1616513070.428523:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:5.0:1616513070.428525:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (31)-- 00000400:00000200:5.0:1616513070.428527:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:13.0:1616513070.428530:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3e4c8 00000800:00000200:1.0:1616513070.428530:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (30)++ 00000100:00000200:9.0:1616513070.428532:0:175777:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579920, offset 192 00000400:00000200:5.0:1616513070.428532:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:1.0:1616513070.428532:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:1.0:1616513070.428533:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000100:00000200:17.0:1616513070.428535:0:170083:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579856 00000800:00000200:13.0:1616513070.428535:0:6467:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:1.0:1616513070.428535:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:5.0:1616513070.428536:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84330 00000800:00000200:1.0:1616513070.428538:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (30)-- 00000400:00000200:1.0:1616513070.428539:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:13.0:1616513070.428540:0:6467:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:9.0:1616513070.428540:0:175777:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00010000:00000200:17.0:1616513070.428542:0:170083:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843f7bec00 x1681364861579856/t0(0) o400->boslfs02-MDT0001-lwp-OST0033_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:5.0:1616513070.428542:0:6468:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513070.428543:0:6467:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:1.0:1616513070.428544:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:13.0:1616513070.428545:0:6467:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (30)++ 00000400:00000200:9.0:1616513070.428545:0:175777:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000100:00000200:21.0:1616513070.428547:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861579952 00000400:00000200:1.0:1616513070.428547:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2330 00000800:00000200:13.0:1616513070.428548:0:6467:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:13.0:1616513070.428550:0:6467:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:5.0:1616513070.428551:0:6468:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000100:00000200:17.0:1616513070.428552:0:170083:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579856, offset 192 00000800:00000200:13.0:1616513070.428552:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00010000:00000200:21.0:1616513070.428553:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d853e8c8d80 x1681364861579952/t0(0) o400->boslfs02-MDT0001-lwp-OST000f_UUID@10.242.46.232@o2ib1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:5.0:1616513070.428553:0:6468:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:1.0:1616513070.428553:0:6465:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513070.428554:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:5.0:1616513070.428555:0:6468:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (30)++ 00000400:00000200:17.0:1616513070.428557:0:170083:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:1.0:1616513070.428557:0:6465:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513070.428558:0:6468:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:5.0:1616513070.428559:0:6468:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:1.0:1616513070.428559:0:6465:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)++ 00000800:00000200:13.0:1616513070.428562:0:6467:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000400:00000200:17.0:1616513070.428563:0:170083:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:13.0:1616513070.428563:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:1.0:1616513070.428563:0:6465:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (32)++ 00000100:00000200:21.0:1616513070.428564:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861579952, offset 192 00000400:00000200:13.0:1616513070.428564:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513070.428566:0:6465:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:5.0:1616513070.428567:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:1.0:1616513070.428568:0:6465:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:21.0:1616513070.428569:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:13.0:1616513070.428569:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:5.0:1616513070.428569:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:1.0:1616513070.428571:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)-- 00000400:00000200:13.0:1616513070.428572:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492673540 00000800:00000200:5.0:1616513070.428573:0:6468:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:1.0:1616513070.428573:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)++ 00000400:00000200:21.0:1616513070.428574:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:5.0:1616513070.428574:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (30)-- 00000400:00000200:5.0:1616513070.428576:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513070.428577:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.232@o2ib1 00000800:00000200:13.0:1616513070.428580:0:6467:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:5.0:1616513070.428580:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:1.0:1616513070.428581:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:1.0:1616513070.428582:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)-- 00000400:00000200:5.0:1616513070.428583:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50ff0 00000800:00000200:1.0:1616513070.428584:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:13.0:1616513070.428585:0:6467:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513070.428585:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:13.0:1616513070.428587:0:6467:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:5.0:1616513070.428587:0:6468:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513070.428588:0:6467:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:1.0:1616513070.428588:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (30)-- 00000400:00000200:1.0:1616513070.428589:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:13.0:1616513070.428591:0:6467:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:5.0:1616513070.428592:0:6468:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513070.428593:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:5.0:1616513070.428594:0:6468:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00000400:00000200:1.0:1616513070.428594:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:13.0:1616513070.428595:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:5.0:1616513070.428595:0:6468:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (30)++ 00000400:00000200:1.0:1616513070.428597:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35100 00000800:00000200:5.0:1616513070.428599:0:6468:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:5.0:1616513070.428600:0:6468:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:13.0:1616513070.428602:0:6467:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:5.0:1616513070.428602:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:1.0:1616513070.428602:0:6465:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513070.428603:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (31)-- 00000400:00000200:13.0:1616513070.428604:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513070.428604:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:1.0:1616513070.428606:0:6465:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513070.428608:0:6468:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:1.0:1616513070.428608:0:6465:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00000400:00000200:13.0:1616513070.428609:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:5.0:1616513070.428609:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:1.0:1616513070.428610:0:6465:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (31)++ 00000400:00000200:13.0:1616513070.428611:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3e440 00000400:00000200:5.0:1616513070.428611:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513070.428613:0:6465:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:1.0:1616513070.428614:0:6465:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (31)-- 00000400:00000200:5.0:1616513070.428615:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:13.0:1616513070.428616:0:6467:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:1.0:1616513070.428616:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000400:00000200:5.0:1616513070.428618:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85f58 00000800:00000200:1.0:1616513070.428618:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:13.0:1616513070.428620:0:6467:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513070.428622:0:6467:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:5.0:1616513070.428622:0:6468:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:1.0:1616513070.428622:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.232@o2ib1 00000800:00000200:13.0:1616513070.428625:0:6467:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:1.0:1616513070.428625:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:5.0:1616513070.428626:0:6468:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513070.428626:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:13.0:1616513070.428627:0:6467:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (33)-- 00000800:00000200:1.0:1616513070.428628:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:13.0:1616513070.428629:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:5.0:1616513070.428629:0:6468:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:5.0:1616513070.428631:0:6468:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:1.0:1616513070.428631:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:5.0:1616513070.428634:0:6468:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:5.0:1616513070.428636:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:1.0:1616513070.428636:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (31)-- 00000400:00000200:1.0:1616513070.428637:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:13.0:1616513070.428638:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:1.0:1616513070.428642:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:1.0:1616513070.428644:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b3c28 00000800:00000200:13.0:1616513070.428654:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:1.0:1616513070.428654:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000400:00000200:13.0:1616513070.428655:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:17.0:1616513070.428656:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (28)++ 00000400:00000200:13.0:1616513070.428660:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:13.0:1616513070.428662:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84926722a8 00000800:00000200:17.0:1616513070.428663:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (29)-- 00000800:00000200:5.0:1616513070.428663:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:17.0:1616513070.428665:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:13.0:1616513070.428666:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)-- 00000800:00000200:5.0:1616513070.428666:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (29)-- 00000800:00000200:13.0:1616513070.428668:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (27)++ 00000400:00000200:5.0:1616513070.428668:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:17.0:1616513070.428671:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:5.0:1616513070.428673:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:17.0:1616513070.428674:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50880 00000800:00000200:13.0:1616513070.428674:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.232@o2ib1 00000800:00000200:13.0:1616513070.428677:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (28)++ 00000400:00000200:5.0:1616513070.428677:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca34c38 00000800:00000200:1.0:1616513070.428677:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (28)++ 00000800:00000200:17.0:1616513070.428678:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:13.0:1616513070.428678:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:1.0:1616513070.428679:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:5.0:1616513070.428680:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (27)-- 00000400:00000200:1.0:1616513070.428680:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:13.0:1616513070.428682:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (26)-- 00000800:00000200:17.0:1616513070.428684:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)++ 00000400:00000200:1.0:1616513070.428685:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:1.0:1616513070.428687:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35540 00000800:00000200:17.0:1616513070.428688:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (26)-- 00000800:00000200:1.0:1616513070.428689:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (26)-- 00000400:00000200:17.0:1616513070.428690:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513070.428691:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000400:00000200:17.0:1616513070.428695:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:1.0:1616513070.428695:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:1.0:1616513070.428697:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:17.0:1616513070.428698:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca34dd0 00000800:00000200:13.0:1616513070.428700:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000400:00000200:1.0:1616513070.428701:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:17.0:1616513070.428703:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000800:00000200:13.0:1616513070.428703:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:1.0:1616513070.428704:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3ea18 00000800:00000200:17.0:1616513070.428705:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000400:00000200:13.0:1616513070.428705:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513070.428707:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)-- 00000800:00000200:17.0:1616513070.428709:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513070.428709:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:13.0:1616513070.428710:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:17.0:1616513070.428711:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:13.0:1616513070.428712:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50330 00000800:00000200:17.0:1616513070.428713:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:13.0:1616513070.428715:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:17.0:1616513070.428717:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:7.2:1616513070.435415:0:48434:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.435428:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.435442:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.435451:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.58@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.435461:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.58@tcp1 of length 224 into portal 12 MB=0x5ffe2b9eef3c0 00000400:00000200:30.0:1616513070.435468:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.58@tcp1 of length 224/224 into md 0xf96450b9 [1] + 12904 00000400:00000200:30.0:1616513070.435474:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.435476:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513070.435482:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.435484:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.435487:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.435489:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.435550:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724130689984 00010000:00000200:14.0:1616513070.435562:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38d100 x1688724130689984/t0(0) o400->bb797345-a586-1a6c-0fe0-f01105acdb73@10.31.140.58@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.435577:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688724130689984, offset 192 00000400:00000200:14.0:1616513070.435583:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.58@tcp1 00000400:00000200:14.0:1616513070.435591:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.58@tcp1 routed destination 00000400:00000200:14.0:1616513070.435606:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.58@tcp1(10.31.140.58@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513070.435607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513070.435613:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513070.435619:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.435622:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.435624:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513070.435629:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.435633:0:48434:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513070.435662:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513070.435671:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513070.435680:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.58@tcp1 : PUT - for me 00000800:00000200:10.0:1616513070.435688:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513070.435689:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.58@tcp1 of length 224 into portal 26 MB=0x5ffe2b9eef340 00000800:00000200:10.0:1616513070.435695:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513070.435696:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.58@tcp1 of length 224/224 into md 0xf97c3e09 [1] + 0 00000400:00000200:10.0:1616513070.435698:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513070.435701:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f4b8 00000400:00000200:28.0:1616513070.435704:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:10.0:1616513070.435704:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:28.0:1616513070.435707:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513070.435712:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f4b8 00000800:00000200:28.0:1616513070.435715:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:10.0:1616513070.435715:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757870 00000800:00000200:12.0:1616513070.435717:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.435718:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.435721:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513070.435721:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.435724:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.435777:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724130689856 00010000:00000200:23.0:1616513070.435788:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688724130689856/t0(0) o400->cfc96df0-7739-e3a7-d5f2-d6864a3f6e69@10.31.140.58@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.435803:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688724130689856, offset 192 00000400:00000200:23.0:1616513070.435809:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.58@tcp1 00000400:00000200:23.0:1616513070.435818:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.58@tcp1 routed destination 00000400:00000200:23.0:1616513070.435833:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.58@tcp1(10.31.140.58@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.435840:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.435846:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.435849:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.435851:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.435856:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.435861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.435863:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513070.435914:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.435922:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.435925:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.435932:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.435938:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1d38 00000800:00000200:16.0:1616513070.435941:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.435944:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.438286:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513070.438343:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.438352:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513070.438353:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513070.438361:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.80@tcp1 : PUT - for me 00000400:00000200:2.0:1616513070.438371:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.80@tcp1 of length 224 into portal 12 MB=0x5ffdf21fd1700 00000400:00000200:2.0:1616513070.438377:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.80@tcp1 of length 224/224 into md 0xf96450b9 [1] + 13128 00000400:00000200:2.0:1616513070.438383:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513070.438385:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513070.438392:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.438395:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.438397:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.438422:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708696577792 00000800:00000200:11.2:1616513070.438428:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:14.0:1616513070.438447:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1388480 x1688708696577792/t0(0) o400->2a5553da-e158-66cf-b6ee-acfafdc49263@10.31.141.80@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.438472:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688708696577792, offset 192 00000400:00000200:14.0:1616513070.438478:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.80@tcp1 00000800:00000200:12.0:1616513070.438485:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513070.438486:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.80@tcp1 routed destination 00000800:00000200:12.0:1616513070.438494:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:14.0:1616513070.438502:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.80@tcp1(10.31.141.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513070.438503:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.80@tcp1 : PUT - for me 00000800:00000200:14.0:1616513070.438508:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513070.438512:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.80@tcp1 of length 224 into portal 26 MB=0x5ffdf21fd1540 00000800:00000200:14.0:1616513070.438514:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.438517:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.438519:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513070.438519:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.80@tcp1 of length 224/224 into md 0xf97c3e29 [1] + 0 00000400:00000200:12.0:1616513070.438523:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fba0 00000800:00000200:14.0:1616513070.438524:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513070.438526:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513070.438529:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:7.2:1616513070.438529:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513070.438536:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:28.0:1616513070.438538:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.438539:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.438542:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513070.438543:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513070.438544:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.438551:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513070.438553:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513070.438560:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513070.438564:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756880 00000800:00000200:10.0:1616513070.438568:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.438570:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.438600:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708696577344 00010000:00000200:23.0:1616513070.438611:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688708696577344/t0(0) o400->40fd6c7e-941b-5c94-869e-6a7816911eab@10.31.141.80@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.438626:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688708696577344, offset 192 00000400:00000200:23.0:1616513070.438632:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.80@tcp1 00000400:00000200:23.0:1616513070.438640:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.80@tcp1 routed destination 00000400:00000200:23.0:1616513070.438656:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.80@tcp1(10.31.141.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.438663:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.438671:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.438674:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.438676:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.438681:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.438685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.438688:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513070.438739:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.438746:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.438750:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.438757:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.438763:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0000 00000800:00000200:16.0:1616513070.438767:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.438770:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513070.456016:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513070.456037:0:48436:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.456048:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.456054:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.456062:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.191@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.456070:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.191@tcp1 of length 224 into portal 12 MB=0x5ffddbbda6840 00000800:00000200:28.0:1616513070.456071:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513070.456076:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.191@tcp1 of length 224/224 into md 0xf96450b9 [1] + 13352 00000400:00000200:30.0:1616513070.456080:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:28.0:1616513070.456080:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:30.0:1616513070.456083:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513070.456087:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513070.456089:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.191@tcp1 : PUT - for me 00000800:00000200:30.0:1616513070.456090:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.456091:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.456094:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513070.456098:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.191@tcp1 of length 224 into portal 26 MB=0x5ffddbbda67c0 00000400:00000200:28.0:1616513070.456105:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.191@tcp1 of length 224/224 into md 0xf97c3e39 [1] + 0 00000400:00000200:28.0:1616513070.456110:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fd38 00000400:00000200:28.0:1616513070.456113:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.456116:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513070.456121:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fd38 00000800:00000200:28.0:1616513070.456124:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.456127:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.456129:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.456131:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513070.456171:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702688061504 00010000:00000200:14.0:1616513070.456184:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38d580 x1688702688061504/t0(0) o400->2cc49ba6-d7a6-eb04-eb38-2af5562337c3@10.31.132.191@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:23.0:1616513070.456186:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702688061376 00010000:00000200:23.0:1616513070.456198:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688702688061376/t0(0) o400->e18f364f-834f-a17f-17fe-d934da083ac3@10.31.132.191@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:14.0:1616513070.456199:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702688061504, offset 192 00000400:00000200:14.0:1616513070.456206:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.191@tcp1 00000100:00000200:23.0:1616513070.456212:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702688061376, offset 192 00000400:00000200:14.0:1616513070.456213:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.191@tcp1 routed destination 00000400:00000200:23.0:1616513070.456218:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.191@tcp1 00000400:00000200:23.0:1616513070.456227:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.191@tcp1 routed destination 00000400:00000200:14.0:1616513070.456229:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.191@tcp1(10.31.132.191@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.456236:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:23.0:1616513070.456243:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.191@tcp1(10.31.132.191@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.456243:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.456245:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.456248:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.456250:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513070.456252:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:23.0:1616513070.456257:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:7.2:1616513070.456258:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.456260:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.456262:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.456267:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.456271:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.456274:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513070.456321:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.456337:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513070.456339:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:10.0:1616513070.456340:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513070.456346:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513070.456347:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.456351:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513070.456351:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756cc0 00000800:00000200:10.0:1616513070.456355:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.456357:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513070.456358:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.456364:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0f68 00000800:00000200:16.0:1616513070.456367:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.456370:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513070.458534:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923856, portal 4 00000100:00000200:30.0:1616513070.458540:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923856, offset 0 00000400:00000200:30.0:1616513070.458545:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513070.458551:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513070.458563:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513070.458569:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513070.458575:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513070.458578:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513070.458580:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513070.458584:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513070.458634:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513070.458693:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513070.458699:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513070.458702:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513070.458708:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513070.458714:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d5a00 x1693067725923856/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513070.458729:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efba0 00000800:00000200:28.0:1616513070.458733:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513070.458736:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513070.458812:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513070.458867:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513070.458876:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:16.0:1616513070.458885:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513070.458893:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626610 00000400:00000200:16.0:1616513070.458899:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d2b41 [1] + 192 00000400:00000200:16.0:1616513070.458907:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513070.458912:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d5a00 x1693067725923856/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513070.458924:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513070.458927:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:16.0:1616513070.458929:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513070.458931:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513070.458932:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d5a00 x1693067725923856/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513070.458944:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d5a00 x1693067725923856/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513070.458953:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee660 00000800:00000200:7.2:1616513070.463700:0:48437:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.463711:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.463716:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.463723:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.94@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.463730:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.94@tcp1 of length 224 into portal 26 MB=0x604a35e2d5cc0 00000400:00000200:30.0:1616513070.463737:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.94@tcp1 of length 224/224 into md 0xf97c3e49 [1] + 0 00000400:00000200:30.0:1616513070.463741:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f650 00000400:00000200:30.0:1616513070.463743:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.463746:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513070.463750:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f650 00000800:00000200:30.0:1616513070.463752:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.463754:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.463756:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.463758:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.463816:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949566475456 00010000:00000200:23.0:1616513070.463827:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1693949566475456/t0(0) o400->7e837180-ffbf-96ee-f1bb-35338d78598c@10.31.143.94@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.463841:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949566475456, offset 192 00000400:00000200:23.0:1616513070.463848:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.94@tcp1 00000400:00000200:23.0:1616513070.463856:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.94@tcp1 routed destination 00000800:00000200:7.2:1616513070.463867:0:48437:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:23.0:1616513070.463871:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.94@tcp1(10.31.143.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.463878:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.463890:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.463893:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.463895:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:23.0:1616513070.463900:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:23.0:1616513070.463909:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513070.463931:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.463940:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513070.463949:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.94@tcp1 : PUT - for me 00000400:00000200:10.0:1616513070.463958:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.94@tcp1 of length 224 into portal 12 MB=0x604a35e2d6100 00000400:00000200:10.0:1616513070.463965:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.94@tcp1 of length 224/224 into md 0xf96450b9 [1] + 13576 00000400:00000200:10.0:1616513070.463970:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.463972:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513070.463979:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:10.0:1616513070.463982:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:10.0:1616513070.463984:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513070.463986:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.463995:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.463997:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513070.463998:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513070.464005:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513070.464013:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d04c8 00000800:00000200:26.0:1616513070.464017:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.464039:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949566476544 00010000:00000200:14.0:1616513070.464051:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bda400 x1693949566476544/t0(0) o400->29ac1d90-2ab6-9c3e-3797-d158a42ffdb4@10.31.143.94@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.464065:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949566476544, offset 192 00000400:00000200:14.0:1616513070.464071:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.94@tcp1 00000400:00000200:14.0:1616513070.464078:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.94@tcp1 routed destination 00000400:00000200:14.0:1616513070.464092:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.94@tcp1(10.31.143.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.464100:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513070.464106:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.464108:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513070.464110:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513070.464115:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.464122:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513070.464176:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.464183:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513070.464186:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513070.464192:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513070.464196:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757f58 00000800:00000200:28.0:1616513070.464200:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.464203:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513070.474564:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923872, portal 4 00000100:00000200:22.0:1616513070.474569:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923872, offset 0 00000400:00000200:22.0:1616513070.474576:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513070.474582:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:22.0:1616513070.474594:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513070.474601:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:22.0:1616513070.474606:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513070.474610:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513070.474612:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513070.474617:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513070.474663:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513070.474720:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513070.474727:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513070.474730:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513070.474736:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513070.474742:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d2400 x1693067725923872/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513070.474754:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b320 00000800:00000200:2.0:1616513070.474758:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513070.474761:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513070.474833:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513070.474845:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513070.474853:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:26.0:1616513070.474856:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:30.0:1616513070.474861:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513070.474868:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626620 00000400:00000200:30.0:1616513070.474874:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c4729 [1] + 192 00000400:00000200:30.0:1616513070.474879:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.474884:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d2400 x1693067725923872/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513070.474898:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513070.474900:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:30.0:1616513070.474902:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:22.0:1616513070.474905:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d2400 x1693067725923872/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513070.474917:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d2400 x1693067725923872/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513070.474927:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba08 00000800:00000200:11.2:1616513070.503895:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513070.503952:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.503961:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513070.503971:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.208@tcp1 : PUT - for me 00000400:00000200:20.0:1616513070.503980:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.208@tcp1 of length 224 into portal 26 MB=0x5ffd4c5a24200 00000400:00000200:20.0:1616513070.503987:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.208@tcp1 of length 224/224 into md 0xf97c3e69 [1] + 0 00000400:00000200:20.0:1616513070.503992:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777eee0 00000400:00000200:20.0:1616513070.503995:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513070.503998:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513070.504003:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eee0 00000800:00000200:20.0:1616513070.504006:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.504006:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.504009:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513070.504010:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.504066:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664197448192 00010000:00000200:23.0:1616513070.504076:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688664197448192/t0(0) o400->9e357f4e-cddc-4cad-bf78-5a8c44e99b1a@10.31.128.208@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.504089:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664197448192, offset 192 00000400:00000200:23.0:1616513070.504096:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.208@tcp1 00000400:00000200:23.0:1616513070.504105:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.208@tcp1 routed destination 00000400:00000200:23.0:1616513070.504119:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.208@tcp1(10.31.128.208@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.504125:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.504132:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.504134:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.504136:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.504141:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513070.504147:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513070.504148:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.504201:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.504208:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513070.504212:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513070.504218:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513070.504224:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d06e8 00000800:00000200:2.0:1616513070.504227:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.504232:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.504541:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513070.504594:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513070.504603:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513070.504611:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.208@tcp1 : PUT - for me 00000400:00000200:28.0:1616513070.504620:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.208@tcp1 of length 224 into portal 12 MB=0x5ffd4c5a243c0 00000400:00000200:28.0:1616513070.504626:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.208@tcp1 of length 224/224 into md 0x865b9f31 [1] + 44216 00000400:00000200:28.0:1616513070.504631:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.504633:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513070.504639:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.504642:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.504644:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.504647:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.504704:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664197448640 00010000:00000200:24.0:1616513070.504716:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89b00 x1688664197448640/t0(0) o400->1261186f-5d35-c065-7293-aa750db90564@10.31.128.208@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513070.504729:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664197448640, offset 192 00000400:00000200:24.0:1616513070.504735:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.208@tcp1 00000400:00000200:24.0:1616513070.504742:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.208@tcp1 routed destination 00000400:00000200:24.0:1616513070.504757:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.208@tcp1(10.31.128.208@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.504763:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513070.504769:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.504771:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513070.504773:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513070.504778:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.504782:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.504793:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.504798:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.504800:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513070.504803:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513070.504807:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.504811:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf37e8 00000800:00000200:16.0:1616513070.504815:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.527918:0:48444:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513070.527979:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.527989:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513070.527991:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513070.527998:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.209@tcp1 : PUT - for me 00000400:00000200:26.0:1616513070.528008:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.209@tcp1 of length 224 into portal 26 MB=0x5ffdb525c06c0 00000400:00000200:26.0:1616513070.528015:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.209@tcp1 of length 224/224 into md 0xf97c3e79 [1] + 0 00000400:00000200:26.0:1616513070.528019:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2550 00000400:00000200:26.0:1616513070.528022:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513070.528025:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513070.528030:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2550 00000800:00000200:26.0:1616513070.528033:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.528036:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.528038:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.528096:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692328236736 00010000:00000200:23.0:1616513070.528106:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688692328236736/t0(0) o400->02778714-4e55-2c08-d74c-a21c739c571b@10.31.140.209@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.528120:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692328236736, offset 192 00000400:00000200:23.0:1616513070.528127:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.209@tcp1 00000400:00000200:23.0:1616513070.528134:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.209@tcp1 routed destination 00000400:00000200:23.0:1616513070.528150:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.209@tcp1(10.31.140.209@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.528157:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.528164:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.528167:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.528169:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.528174:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.528178:0:48444:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.528181:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513070.528191:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.528198:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513070.528201:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513070.528208:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513070.528215:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0220 00000800:00000200:2.0:1616513070.528218:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.528221:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.528697:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.528707:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.528714:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513070.528717:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513070.528723:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.209@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.528731:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.209@tcp1 of length 224 into portal 12 MB=0x5ffdb525c0a00 00000400:00000200:30.0:1616513070.528737:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.209@tcp1 of length 224/224 into md 0xf96450b9 [1] + 13800 00000400:00000200:30.0:1616513070.528742:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.528745:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513070.528751:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.528754:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.528756:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.528778:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692328237568 00010000:00000200:14.0:1616513070.528790:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38ba80 x1688692328237568/t0(0) o400->72616914-918b-b8e7-284a-0647a7f5f214@10.31.140.209@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.528804:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692328237568, offset 192 00000400:00000200:14.0:1616513070.528810:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.209@tcp1 00000400:00000200:14.0:1616513070.528817:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.209@tcp1 routed destination 00000400:00000200:14.0:1616513070.528831:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.209@tcp1(10.31.140.209@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.528838:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513070.528843:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.528846:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513070.528848:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513070.528852:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.528861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.528916:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.528922:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.528925:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.528932:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.528936:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756660 00000800:00000200:12.0:1616513070.528940:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.528942:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:21.0:1616513070.538779:0:6492:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923888, portal 4 00000100:00000200:21.0:1616513070.538784:0:6492:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923888, offset 0 00000400:00000200:21.0:1616513070.538791:0:6492:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:21.0:1616513070.538797:0:6492:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:21.0:1616513070.538810:0:6492:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:21.0:1616513070.538816:0:6492:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:21.0:1616513070.538824:0:6492:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513070.538827:0:6492:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:21.0:1616513070.538829:0:6492:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:21.0:1616513070.538834:0:6492:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513070.538883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.538901:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513070.538906:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513070.538909:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513070.538916:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513070.538925:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdd689680 x1693067725923888/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513070.538937:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51078 00000800:00000200:1.0:1616513070.538941:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.538943:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513070.539051:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513070.539060:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.539066:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:17.0:1616513070.539069:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513070.539074:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513070.539082:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626630 00000400:00000200:1.0:1616513070.539089:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff8a5 [1] + 192 00000400:00000200:1.0:1616513070.539094:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513070.539098:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdd689680 x1693067725923888/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513070.539112:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513070.539114:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513070.539116:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:21.0:1616513070.539119:0:6492:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdd689680 x1693067725923888/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:21.0:1616513070.539132:0:6492:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdd689680 x1693067725923888/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:21.0:1616513070.539141:0:6492:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50088 00000800:00000200:7.2:1616513070.588842:0:48451:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513070.588848:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513070.588898:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.588909:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513070.588916:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513070.588919:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.46@tcp1 : PUT - for me 00000800:00000200:20.0:1616513070.588926:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513070.588929:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.46@tcp1 of length 224 into portal 26 MB=0x603678986f340 00000400:00000200:20.0:1616513070.588935:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.46@tcp1 : PUT - for me 00000400:00000200:2.0:1616513070.588936:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.46@tcp1 of length 224/224 into md 0xf97c3e99 [1] + 0 00000400:00000200:2.0:1616513070.588941:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3870 00000400:00000200:2.0:1616513070.588944:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513070.588945:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.46@tcp1 of length 224 into portal 12 MB=0x603678986f3c0 00000100:00000200:2.0:1616513070.588947:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513070.588951:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.46@tcp1 of length 224/224 into md 0x865b9f31 [1] + 44440 00000400:00000200:2.0:1616513070.588952:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3870 00000800:00000200:10.0:1616513070.588953:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.588956:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513070.588957:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513070.588959:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513070.588960:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513070.588961:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513070.588967:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.588971:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.588973:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.588977:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.589018:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692593084101440 00010000:00000200:23.0:1616513070.589029:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1692593084101440/t0(0) o400->ed3ca774-7f9c-1ce6-8693-d8e23d82e597@10.31.131.46@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513070.589035:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692593084101568 00000100:00000200:23.0:1616513070.589043:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692593084101440, offset 192 00010000:00000200:24.0:1616513070.589049:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05a00 x1692593084101568/t0(0) o400->48a4bc60-13cf-7791-093b-e0c2eef0d832@10.31.131.46@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:23.0:1616513070.589051:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.46@tcp1 00000400:00000200:23.0:1616513070.589060:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.46@tcp1 routed destination 00000100:00000200:24.0:1616513070.589064:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692593084101568, offset 192 00000400:00000200:24.0:1616513070.589071:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.46@tcp1 00000400:00000200:23.0:1616513070.589076:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.46@tcp1(10.31.131.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513070.589080:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.46@tcp1 routed destination 00000800:00000200:23.0:1616513070.589083:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.589090:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.589093:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.589095:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513070.589096:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.46@tcp1(10.31.131.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.589100:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513070.589102:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513070.589104:0:48451:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.589107:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513070.589109:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.589111:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513070.589114:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513070.589119:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.589123:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513070.589159:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.589167:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.589171:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513070.589171:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513070.589177:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513070.589179:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513070.589183:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1980 00000800:00000200:12.0:1616513070.589186:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513070.589187:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513070.589189:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.589196:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.589199:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:12.0:1616513070.589204:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.589207:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513070.598247:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513070.598265:0:48452:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513070.598278:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.598288:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513070.598297:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.194@tcp1 : PUT - for me 00000400:00000200:10.0:1616513070.598307:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.194@tcp1 of length 224 into portal 26 MB=0x5ffdb4b45dd00 00000800:00000200:16.0:1616513070.598309:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513070.598314:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.194@tcp1 of length 224/224 into md 0xf97c3ea9 [1] + 0 00000400:00000200:10.0:1616513070.598323:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2d48 00000800:00000200:16.0:1616513070.598328:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:10.0:1616513070.598328:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.598331:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513070.598336:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2d48 00000400:00000200:16.0:1616513070.598337:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.194@tcp1 : PUT - for me 00000800:00000200:10.0:1616513070.598340:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.598343:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513070.598344:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:16.0:1616513070.598347:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.194@tcp1 of length 224 into portal 12 MB=0x5ffdb4b45df00 00000800:00000200:10.0:1616513070.598348:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513070.598353:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.194@tcp1 of length 224/224 into md 0x865b9f31 [1] + 44664 00000400:00000200:16.0:1616513070.598358:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513070.598361:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513070.598367:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.598370:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.598372:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.598375:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.598402:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692209343744 00010000:00000200:23.0:1616513070.598413:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688692209343744/t0(0) o400->9df8eccf-9bcd-cf7d-ba02-ff59d2c79ab5@10.31.140.194@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.598428:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692209343744, offset 192 00000100:00000200:24.0:1616513070.598446:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692209344256 00000400:00000200:23.0:1616513070.598448:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.194@tcp1 00000400:00000200:23.0:1616513070.598458:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.194@tcp1 routed destination 00010000:00000200:24.0:1616513070.598459:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e1680 x1688692209344256/t0(0) o400->ba6818bb-82f5-79f7-8787-c2712b6aed8d@10.31.140.194@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513070.598473:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692209344256, offset 192 00000400:00000200:23.0:1616513070.598473:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.194@tcp1(10.31.140.194@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.598480:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513070.598482:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.194@tcp1 00000800:00000200:23.0:1616513070.598487:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.598490:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513070.598491:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.194@tcp1 routed destination 00000800:00000200:23.0:1616513070.598492:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.598497:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.598501:0:48452:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.598505:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513070.598506:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.194@tcp1(10.31.140.194@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513070.598511:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513070.598513:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513070.598517:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513070.598519:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513070.598520:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513070.598522:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513070.598524:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513070.598526:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513070.598529:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513070.598531:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0c38 00000800:00000200:11.2:1616513070.598533:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513070.598535:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.598537:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513070.598590:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.598597:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.598600:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.598606:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.598610:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:12.0:1616513070.598614:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.598617:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513070.614537:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513070.614592:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.614601:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513070.614610:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.52@tcp1 : PUT - for me 00000400:00000200:20.0:1616513070.614619:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.52@tcp1 of length 224 into portal 26 MB=0x601990ba09880 00000400:00000200:20.0:1616513070.614627:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.52@tcp1 of length 224/224 into md 0xf97c3ec9 [1] + 0 00000400:00000200:20.0:1616513070.614633:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf27f8 00000400:00000200:20.0:1616513070.614636:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513070.614639:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513070.614643:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf27f8 00000800:00000200:16.0:1616513070.614645:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.614646:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.614649:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.614651:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.614709:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690606696962176 00010000:00000200:23.0:1616513070.614720:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1690606696962176/t0(0) o400->4f7a6f50-0f78-1ce4-8e61-1841072f2564@10.31.141.52@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.614734:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690606696962176, offset 192 00000400:00000200:23.0:1616513070.614740:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.52@tcp1 00000400:00000200:23.0:1616513070.614748:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.52@tcp1 routed destination 00000400:00000200:23.0:1616513070.614764:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.52@tcp1(10.31.141.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.614770:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.614779:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.614781:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.614784:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.614789:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513070.614795:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513070.614795:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.614850:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.614857:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513070.614860:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513070.614867:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513070.614873:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0198 00000800:00000200:2.0:1616513070.614877:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.614880:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.615315:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.615325:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.615340:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.615349:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.52@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.615357:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.52@tcp1 of length 224 into portal 12 MB=0x601990ba0a4c0 00000400:00000200:30.0:1616513070.615363:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.52@tcp1 of length 224/224 into md 0xf96450b9 [1] + 14024 00000400:00000200:30.0:1616513070.615368:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.615370:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513070.615385:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.615385:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.615388:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.615390:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.615465:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690606696965312 00010000:00000200:14.0:1616513070.615478:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38da00 x1690606696965312/t0(0) o400->8873f302-b511-c29c-767c-93b1da968ff9@10.31.141.52@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.615493:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690606696965312, offset 192 00000400:00000200:14.0:1616513070.615499:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.52@tcp1 00000400:00000200:14.0:1616513070.615507:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.52@tcp1 routed destination 00000400:00000200:14.0:1616513070.615522:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.52@tcp1(10.31.141.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.615529:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513070.615535:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.615538:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513070.615540:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513070.615545:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.615549:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.615601:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.615609:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.615611:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.615618:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.615622:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757d38 00000800:00000200:12.0:1616513070.615626:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.615629:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.626419:0:48455:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513070.626475:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.626485:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513070.626488:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513070.626493:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.108@tcp1 : PUT - for me 00000400:00000200:2.0:1616513070.626504:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.108@tcp1 of length 224 into portal 26 MB=0x60195bdd7b340 00000400:00000200:2.0:1616513070.626511:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.108@tcp1 of length 224/224 into md 0xf97c3ee9 [1] + 0 00000400:00000200:2.0:1616513070.626521:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2000 00000400:00000200:2.0:1616513070.626524:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513070.626527:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513070.626532:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2000 00000800:00000200:2.0:1616513070.626535:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.626538:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.626539:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.626595:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690592507048768 00010000:00000200:23.0:1616513070.626606:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1690592507048768/t0(0) o400->c2fa5898-efcc-ce62-6eaf-05b387e1e5b1@10.31.140.108@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.626619:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690592507048768, offset 192 00000400:00000200:23.0:1616513070.626625:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.108@tcp1 00000400:00000200:23.0:1616513070.626634:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.108@tcp1 routed destination 00000400:00000200:23.0:1616513070.626648:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.108@tcp1(10.31.140.108@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.626655:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513070.626655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.626660:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.626663:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.626665:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.626670:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.626674:0:48455:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.626676:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513070.626709:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.626718:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513070.626728:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.108@tcp1 : PUT - for me 00000800:00000200:26.0:1616513070.626729:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.626737:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:16.0:1616513070.626737:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.108@tcp1 of length 224 into portal 12 MB=0x60195bdd7b5c0 00000800:00000200:30.0:1616513070.626740:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513070.626740:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.626744:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.108@tcp1 of length 224/224 into md 0x865b9f31 [1] + 44888 00000400:00000200:26.0:1616513070.626747:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.626749:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513070.626753:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1c28 00000100:00000200:16.0:1616513070.626753:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513070.626756:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513070.626760:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.626762:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.626763:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.626766:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.626825:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690592507049408 00010000:00000200:24.0:1616513070.626838:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e7500 x1690592507049408/t0(0) o400->6726f997-aac8-a05b-2f6c-9f720354a204@10.31.140.108@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513070.626853:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690592507049408, offset 192 00000400:00000200:24.0:1616513070.626859:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.108@tcp1 00000400:00000200:24.0:1616513070.626867:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.108@tcp1 routed destination 00000400:00000200:24.0:1616513070.626882:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.108@tcp1(10.31.140.108@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.626888:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513070.626894:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.626897:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513070.626899:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513070.626904:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.626909:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.626973:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.626980:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.626983:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.626989:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.626995:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000800:00000200:12.0:1616513070.626999:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.627002:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.642636:0:48457:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513070.642703:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.642712:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513070.642721:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.188@tcp1 : PUT - for me 00000400:00000200:10.0:1616513070.642729:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.188@tcp1 of length 224 into portal 26 MB=0x5ffdcbdc78840 00000400:00000200:10.0:1616513070.642735:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.188@tcp1 of length 224/224 into md 0xf97c3f19 [1] + 0 00000400:00000200:10.0:1616513070.642740:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3298 00000400:00000200:10.0:1616513070.642743:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.642745:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513070.642749:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3298 00000800:00000200:10.0:1616513070.642752:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.642755:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513070.642755:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513070.642759:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.642814:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698425411648 00010000:00000200:23.0:1616513070.642824:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688698425411648/t0(0) o400->c4b8001a-32fb-1359-c9fb-c94cb1c2c4a1@10.31.141.188@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.642837:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698425411648, offset 192 00000400:00000200:23.0:1616513070.642843:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.188@tcp1 00000800:00000200:11.2:1616513070.642846:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:23.0:1616513070.642850:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.188@tcp1 routed destination 00000400:00000200:23.0:1616513070.642865:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.188@tcp1(10.31.141.188@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.642871:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.642877:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.642880:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.642882:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.642887:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.642891:0:48457:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.642894:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513070.642900:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513070.642902:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.642908:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513070.642909:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:30.0:1616513070.642910:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.642917:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513070.642918:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.188@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.642922:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1a90 00000800:00000200:30.0:1616513070.642926:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513070.642927:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.188@tcp1 of length 224 into portal 12 MB=0x5ffdcbdc78ac0 00000800:00000200:30.0:1616513070.642928:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513070.642933:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.188@tcp1 of length 224/224 into md 0x865b9f31 [1] + 45112 00000400:00000200:20.0:1616513070.642938:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513070.642941:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513070.642947:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.642950:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513070.642952:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.642955:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.643015:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698425412288 00010000:00000200:24.0:1616513070.643027:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05e80 x1688698425412288/t0(0) o400->bee00828-b035-104d-41f7-782ac89b9bd8@10.31.141.188@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513070.643042:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698425412288, offset 192 00000400:00000200:24.0:1616513070.643048:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.188@tcp1 00000400:00000200:24.0:1616513070.643056:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.188@tcp1 routed destination 00000400:00000200:24.0:1616513070.643071:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.188@tcp1(10.31.141.188@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.643078:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513070.643084:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.643089:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513070.643091:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513070.643096:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.643101:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.643154:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.643160:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.643162:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.643168:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.643172:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2b28 00000800:00000200:12.0:1616513070.643176:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.643178:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.690893:0:48462:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513070.690949:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.690965:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513070.690974:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.27.196@tcp1 : PUT - for me 00000400:00000200:2.0:1616513070.690984:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.27.196@tcp1 of length 224 into portal 12 MB=0x5ffe9c77b1c40 00000400:00000200:2.0:1616513070.690991:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.27.196@tcp1 of length 224/224 into md 0xf96450b9 [1] + 14248 00000400:00000200:2.0:1616513070.690997:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513070.691000:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513070.691004:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.691007:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.691010:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.691012:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.691076:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688754422750272 00010000:00000200:14.0:1616513070.691089:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1388900 x1688754422750272/t0(0) o400->3459ec93-8524-14ad-74a5-97b5a008dba8@10.31.27.196@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.691104:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688754422750272, offset 192 00000400:00000200:14.0:1616513070.691110:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.27.196@tcp1 00000400:00000200:14.0:1616513070.691119:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.27.196@tcp1 routed destination 00000400:00000200:14.0:1616513070.691134:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.27.196@tcp1(10.31.27.196@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.691141:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513070.691147:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.691150:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.691152:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513070.691157:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.691162:0:48462:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.691174:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.691180:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.691182:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.691188:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.691192:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756198 00000800:00000200:30.0:1616513070.691195:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.691198:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.707643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513070.707698:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.707713:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513070.707723:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.120@tcp1 : PUT - for me 00000400:00000200:20.0:1616513070.707733:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.120@tcp1 of length 224 into portal 26 MB=0x602ad7217c840 00000400:00000200:20.0:1616513070.707740:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.120@tcp1 of length 224/224 into md 0xf97c3f39 [1] + 0 00000400:00000200:20.0:1616513070.707745:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3b18 00000400:00000200:20.0:1616513070.707748:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513070.707751:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513070.707755:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:20.0:1616513070.707759:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.707762:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513070.707763:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.707766:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.707823:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691793827022912 00010000:00000200:23.0:1616513070.707834:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1691793827022912/t0(0) o400->08027aae-22a8-3480-e713-f3a7fc005999@10.31.141.120@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.707848:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691793827022912, offset 192 00000400:00000200:23.0:1616513070.707855:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.120@tcp1 00000400:00000200:23.0:1616513070.707863:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.120@tcp1 routed destination 00000400:00000200:23.0:1616513070.707879:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.120@tcp1(10.31.141.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.707885:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.707891:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.707894:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.707896:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.707901:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.707906:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.707908:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513070.707959:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.707966:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.707970:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.707976:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.707982:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1a90 00000800:00000200:12.0:1616513070.707986:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.707989:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.707999:0:48464:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513070.708052:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.708061:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513070.708070:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.120@tcp1 : PUT - for me 00000400:00000200:10.0:1616513070.708078:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.120@tcp1 of length 224 into portal 12 MB=0x602ad7217c8c0 00000400:00000200:10.0:1616513070.708085:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.120@tcp1 of length 224/224 into md 0xf96450b9 [1] + 14472 00000400:00000200:10.0:1616513070.708090:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.708092:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513070.708098:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.708101:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513070.708103:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.708106:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.708164:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691793827023040 00010000:00000200:14.0:1616513070.708176:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdba80 x1691793827023040/t0(0) o400->8c2a9a1f-8bbe-51e2-47d6-ed6fac19b664@10.31.141.120@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.708191:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691793827023040, offset 192 00000400:00000200:14.0:1616513070.708197:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.120@tcp1 00000400:00000200:14.0:1616513070.708205:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.120@tcp1 routed destination 00000400:00000200:14.0:1616513070.708220:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.120@tcp1(10.31.141.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.708227:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513070.708233:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.708236:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.708238:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513070.708243:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.708248:0:48464:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.708258:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.708264:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.708266:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.708272:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.708276:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756220 00000800:00000200:30.0:1616513070.708280:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.708282:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.709192:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513070.709246:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.709255:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513070.709271:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.90@tcp1 : PUT - for me 00000400:00000200:20.0:1616513070.709281:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.90@tcp1 of length 224 into portal 26 MB=0x5ffde8a631f40 00000400:00000200:20.0:1616513070.709288:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.90@tcp1 of length 224/224 into md 0xf97c3f49 [1] + 0 00000400:00000200:20.0:1616513070.709293:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2220 00000400:00000200:20.0:1616513070.709296:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513070.709299:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513070.709303:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:20.0:1616513070.709306:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.709308:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.709309:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513070.709314:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513070.709317:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.709318:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513070.709327:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.90@tcp1 : PUT - for me 00000400:00000200:16.0:1616513070.709336:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.90@tcp1 of length 224 into portal 12 MB=0x5ffde8a631fc0 00000400:00000200:16.0:1616513070.709342:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.90@tcp1 of length 224/224 into md 0x865b9f31 [1] + 45336 00000400:00000200:16.0:1616513070.709347:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513070.709350:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513070.709357:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.709361:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.709363:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.709369:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706153127744 00010000:00000200:23.0:1616513070.709380:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688706153127744/t0(0) o400->8a86e990-792e-484c-8b9f-db20162c3bf6@10.31.142.90@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.709394:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706153127744, offset 192 00000400:00000200:23.0:1616513070.709400:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.90@tcp1 00000400:00000200:23.0:1616513070.709409:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.90@tcp1 routed destination 00000400:00000200:23.0:1616513070.709425:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.90@tcp1(10.31.142.90@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.709431:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.709451:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.709454:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.709457:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513070.709459:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706153127872 00000800:00000200:23.0:1616513070.709461:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.709466:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.709469:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:24.0:1616513070.709470:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e4380 x1688706153127872/t0(0) o400->346227fa-b28b-1238-4d67-e10621214188@10.31.142.90@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513070.709484:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706153127872, offset 192 00000400:00000200:24.0:1616513070.709490:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.90@tcp1 00000400:00000200:24.0:1616513070.709497:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.90@tcp1 routed destination 00000400:00000200:24.0:1616513070.709511:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.90@tcp1(10.31.142.90@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.709518:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513070.709519:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513070.709523:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.709526:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513070.709527:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513070.709528:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513070.709530:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513070.709530:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513070.709532:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513070.709537:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513070.709538:0:48464:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513070.709542:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0198 00000800:00000200:28.0:1616513070.709546:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513070.709593:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.709600:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513070.709603:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513070.709609:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513070.709614:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:2.0:1616513070.709624:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.709627:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.712913:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.712923:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.712929:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.712937:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.136@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.712946:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.136@tcp1 of length 224 into portal 26 MB=0x604a54522b7c0 00000400:00000200:30.0:1616513070.712953:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.136@tcp1 of length 224/224 into md 0xf97c3f69 [1] + 0 00000400:00000200:30.0:1616513070.712957:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3430 00000400:00000200:30.0:1616513070.712960:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.712962:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513070.712967:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3430 00000800:00000200:30.0:1616513070.712969:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.712971:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513070.712973:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513070.712980:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.712982:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.712984:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513070.712992:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.136@tcp1 : PUT - for me 00000400:00000200:26.0:1616513070.713001:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.136@tcp1 of length 224 into portal 12 MB=0x604a54522b8c0 00000400:00000200:26.0:1616513070.713008:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.136@tcp1 of length 224/224 into md 0xf96450b9 [1] + 14696 00000400:00000200:26.0:1616513070.713015:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513070.713018:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513070.713024:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.713027:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.713029:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.713033:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693957736282048 00010000:00000200:23.0:1616513070.713044:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1693957736282048/t0(0) o400->77743738-d5a5-bf57-37cb-9e7bff522ca1@10.31.140.136@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.713058:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693957736282048, offset 192 00000400:00000200:23.0:1616513070.713072:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.136@tcp1 00000400:00000200:23.0:1616513070.713081:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.136@tcp1 routed destination 00000100:00000200:14.0:1616513070.713088:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693957736282304 00000400:00000200:23.0:1616513070.713096:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.136@tcp1(10.31.140.136@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:14.0:1616513070.713100:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695a00 x1693957736282304/t0(0) o400->445b2b62-97e4-74c9-24d1-2abc4acaf0c8@10.31.140.136@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513070.713103:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.713109:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.713112:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.713114:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:14.0:1616513070.713115:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693957736282304, offset 192 00000800:00000200:23.0:1616513070.713119:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513070.713122:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.136@tcp1 00000800:00000200:11.2:1616513070.713125:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.713126:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513070.713129:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.136@tcp1 routed destination 00000400:00000200:14.0:1616513070.713146:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.136@tcp1(10.31.140.136@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.713153:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513070.713159:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.713162:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.713164:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513070.713168:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.713173:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513070.713178:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.713186:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.713189:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.713195:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.713201:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1980 00000800:00000200:12.0:1616513070.713205:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.713208:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513070.713227:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513070.713234:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.713237:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513070.713238:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513070.713245:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513070.713249:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756770 00000800:00000200:10.0:1616513070.713253:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.737009:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513070.737066:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.737075:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513070.737085:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.30@tcp1 : PUT - for me 00000400:00000200:2.0:1616513070.737094:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.30@tcp1 of length 224 into portal 12 MB=0x5ffeb993ee4c0 00000400:00000200:2.0:1616513070.737100:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.30@tcp1 of length 224/224 into md 0xf96450b9 [1] + 14920 00000400:00000200:2.0:1616513070.737106:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513070.737108:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513070.737117:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.737119:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.737120:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513070.737124:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.737169:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:14.0:1616513070.737182:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688762236986560 00010000:00000200:14.0:1616513070.737194:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1388d80 x1688762236986560/t0(0) o400->08644e19-f39f-09ea-650b-1ce5ad6fb05c@10.31.140.30@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.737209:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688762236986560, offset 192 00000400:00000200:14.0:1616513070.737215:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.30@tcp1 00000400:00000200:14.0:1616513070.737222:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.30@tcp1 routed destination 00000800:00000200:20.0:1616513070.737224:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.737233:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:14.0:1616513070.737238:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.30@tcp1(10.31.140.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.737244:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513070.737247:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.30@tcp1 : PUT - for me 00000800:00000200:14.0:1616513070.737251:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.737254:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513070.737256:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.30@tcp1 of length 224 into portal 26 MB=0x5ffeb993ee400 00000800:00000200:14.0:1616513070.737256:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:14.0:1616513070.737261:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:20.0:1616513070.737262:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.30@tcp1 of length 224/224 into md 0xf97c3f79 [1] + 0 00000400:00000200:20.0:1616513070.737267:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3e48 00000400:00000200:20.0:1616513070.737270:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513070.737273:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513070.737278:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:20.0:1616513070.737278:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000800:00000200:20.0:1616513070.737281:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513070.737284:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:28.0:1616513070.737286:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513070.737286:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513070.737288:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513070.737289:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513070.737296:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513070.737300:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756ee0 00000800:00000200:28.0:1616513070.737303:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.737343:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688762236986368 00010000:00000200:23.0:1616513070.737355:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688762236986368/t0(0) o400->5b39c104-0cf6-304c-2304-7d883efb3bd3@10.31.140.30@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.737369:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688762236986368, offset 192 00000400:00000200:23.0:1616513070.737376:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.30@tcp1 00000400:00000200:23.0:1616513070.737384:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.30@tcp1 routed destination 00000400:00000200:23.0:1616513070.737400:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.30@tcp1(10.31.140.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.737406:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.737413:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.737416:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.737418:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.737423:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513070.737429:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513070.737429:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.737444:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.737449:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.737452:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.737458:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.737463:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d06e8 00000800:00000200:30.0:1616513070.737467:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.737469:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.746318:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513070.746374:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.746384:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513070.746393:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.10@tcp1 : PUT - for me 00000400:00000200:26.0:1616513070.746402:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.10@tcp1 of length 224 into portal 26 MB=0x603943b8dc100 00000400:00000200:26.0:1616513070.746409:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.10@tcp1 of length 224/224 into md 0xf97c3f99 [1] + 0 00000400:00000200:26.0:1616513070.746414:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2cc0 00000400:00000200:26.0:1616513070.746417:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513070.746419:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513070.746424:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2cc0 00000800:00000200:26.0:1616513070.746427:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.746430:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513070.746431:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513070.746436:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.746457:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:23.0:1616513070.746490:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692785049452800 00010000:00000200:23.0:1616513070.746501:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1692785049452800/t0(0) o400->6fd45521-bf6c-8818-b855-89119c43bf79@10.31.140.10@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513070.746513:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.746516:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692785049452800, offset 192 00000400:00000200:23.0:1616513070.746522:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.10@tcp1 00000800:00000200:16.0:1616513070.746523:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:23.0:1616513070.746531:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.10@tcp1 routed destination 00000400:00000200:16.0:1616513070.746533:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.10@tcp1 : PUT - for me 00000400:00000200:23.0:1616513070.746546:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.10@tcp1(10.31.140.10@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.746552:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513070.746556:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.10@tcp1 of length 224 into portal 12 MB=0x603943b8dc180 00000800:00000200:23.0:1616513070.746561:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.746563:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513070.746563:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.10@tcp1 of length 224/224 into md 0x865b9f31 [1] + 45560 00000800:00000200:23.0:1616513070.746566:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513070.746566:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.746568:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513070.746570:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:23.0:1616513070.746571:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:11.2:1616513070.746576:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.746577:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:23.0:1616513070.746578:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513070.746581:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513070.746583:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.746585:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.746599:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.746600:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.746601:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.746607:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.746612:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0f68 00000800:00000200:16.0:1616513070.746615:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.746644:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692785049452928 00010000:00000200:24.0:1616513070.746656:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e5580 x1692785049452928/t0(0) o400->28791973-4d95-f33a-a0ad-065c0efa0990@10.31.140.10@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513070.746671:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692785049452928, offset 192 00000400:00000200:24.0:1616513070.746677:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.10@tcp1 00000400:00000200:24.0:1616513070.746685:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.10@tcp1 routed destination 00000400:00000200:24.0:1616513070.746700:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.10@tcp1(10.31.140.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.746706:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513070.746712:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.746715:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513070.746717:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513070.746722:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.746727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.746736:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.746740:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.746742:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513070.746746:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513070.746748:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.746751:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:30.0:1616513070.746754:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.813049:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513070.813103:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.813113:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513070.813122:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.94@tcp1 : PUT - for me 00000400:00000200:2.0:1616513070.813130:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.94@tcp1 of length 224 into portal 26 MB=0x6001d4b1eccc0 00000400:00000200:2.0:1616513070.813136:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.94@tcp1 of length 224/224 into md 0xf97c3fa9 [1] + 0 00000400:00000200:2.0:1616513070.813141:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf34b8 00000400:00000200:2.0:1616513070.813143:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513070.813146:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513070.813150:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf34b8 00000800:00000200:2.0:1616513070.813154:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.813156:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513070.813158:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513070.813162:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.813217:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975674625216 00010000:00000200:23.0:1616513070.813228:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688975674625216/t0(0) o400->8cef55a2-de03-c4a4-4718-e4402b6270c3@10.31.142.94@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.813243:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688975674625216, offset 192 00000400:00000200:23.0:1616513070.813249:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.94@tcp1 00000400:00000200:23.0:1616513070.813257:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.94@tcp1 routed destination 00000400:00000200:23.0:1616513070.813272:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.94@tcp1(10.31.142.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.813279:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.813285:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.813288:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.813290:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.813295:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:23.0:1616513070.813302:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513070.813303:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513070.813359:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.813367:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513070.813370:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513070.813377:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513070.813384:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1d38 00000800:00000200:20.0:1616513070.813387:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.813390:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513070.813614:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513070.813626:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.813635:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513070.813637:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513070.813643:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.94@tcp1 : PUT - for me 00000400:00000200:10.0:1616513070.813651:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.94@tcp1 of length 224 into portal 12 MB=0x6001d4b1ed2c0 00000400:00000200:10.0:1616513070.813657:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.94@tcp1 of length 224/224 into md 0xf96450b9 [1] + 15144 00000400:00000200:10.0:1616513070.813662:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.813665:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513070.813672:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.813676:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.813677:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.813740:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975674626752 00010000:00000200:14.0:1616513070.813754:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdbf00 x1688975674626752/t0(0) o400->ced8b216-6aca-6082-2072-f13fc9ec4b03@10.31.142.94@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.813769:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688975674626752, offset 192 00000400:00000200:14.0:1616513070.813776:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.94@tcp1 00000400:00000200:14.0:1616513070.813783:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.94@tcp1 routed destination 00000400:00000200:14.0:1616513070.813799:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.94@tcp1(10.31.142.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.813806:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513070.813812:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.813815:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.813819:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513070.813824:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.813829:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.813839:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.813845:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513070.813847:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513070.813849:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513070.813854:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513070.813858:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757540 00000800:00000200:26.0:1616513070.813862:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.831149:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.831162:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.831170:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.831178:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.178@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.831187:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.178@tcp1 of length 224 into portal 26 MB=0x5ffdc89a6dcc0 00000400:00000200:30.0:1616513070.831193:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.178@tcp1 of length 224/224 into md 0xf97c3fb9 [1] + 0 00000400:00000200:30.0:1616513070.831197:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3cb0 00000400:00000200:30.0:1616513070.831200:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.831203:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513070.831207:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3cb0 00000800:00000200:30.0:1616513070.831210:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.831212:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513070.831214:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.831218:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.831273:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697550855360 00010000:00000200:23.0:1616513070.831284:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688697550855360/t0(0) o400->3f426e46-d924-8947-884a-ba9d6cfe333b@10.31.131.178@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.831299:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697550855360, offset 192 00000400:00000200:23.0:1616513070.831305:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.178@tcp1 00000400:00000200:23.0:1616513070.831313:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.178@tcp1 routed destination 00000400:00000200:23.0:1616513070.831329:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.178@tcp1(10.31.131.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.831335:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.831341:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.831344:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.831346:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.831351:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.831359:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.831360:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513070.831415:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.831423:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513070.831428:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513070.831456:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513070.831462:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0e58 00000800:00000200:16.0:1616513070.831466:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.831469:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.831475:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513070.831483:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.178@tcp1 : PUT - for me 00000400:00000200:16.0:1616513070.831491:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.178@tcp1 of length 224 into portal 12 MB=0x5ffdc89a6dec0 00000400:00000200:16.0:1616513070.831497:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.178@tcp1 of length 224/224 into md 0x865b9f31 [1] + 45784 00000400:00000200:16.0:1616513070.831503:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513070.831505:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513070.831511:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.831514:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.831516:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513070.831518:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.831579:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697550855872 00010000:00000200:24.0:1616513070.831591:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e6300 x1688697550855872/t0(0) o400->ea8b8e04-2558-5ea5-ca91-b2940fce99c6@10.31.131.178@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513070.831616:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697550855872, offset 192 00000400:00000200:24.0:1616513070.831623:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.178@tcp1 00000400:00000200:24.0:1616513070.831630:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.178@tcp1 routed destination 00000400:00000200:24.0:1616513070.831646:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.178@tcp1(10.31.131.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.831653:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513070.831659:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.831662:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513070.831664:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513070.831669:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.831674:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.831729:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.831736:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513070.831739:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513070.831745:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513070.831749:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:2.0:1616513070.831753:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513070.831756:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.832186:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513070.832197:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.832205:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.832205:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513070.832213:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.92@tcp1 : PUT - for me 00000400:00000200:10.0:1616513070.832222:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.92@tcp1 of length 224 into portal 12 MB=0x601ab57277180 00000400:00000200:10.0:1616513070.832228:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.92@tcp1 of length 224/224 into md 0xf96450b9 [1] + 15368 00000400:00000200:10.0:1616513070.832233:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513070.832236:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513070.832242:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513070.832244:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513070.832246:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513070.832307:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690685273502080 00010000:00000200:14.0:1616513070.832319:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bd8d80 x1690685273502080/t0(0) o400->9d458c94-07d8-8efa-a81f-2e4ac17d883e@10.31.140.92@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513070.832334:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690685273502080, offset 192 00000400:00000200:14.0:1616513070.832341:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.92@tcp1 00000400:00000200:14.0:1616513070.832349:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.92@tcp1 routed destination 00000400:00000200:14.0:1616513070.832364:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.92@tcp1(10.31.140.92@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513070.832371:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513070.832377:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513070.832380:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513070.832382:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513070.832387:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.832392:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513070.832449:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.832456:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513070.832459:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513070.832462:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513070.832466:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513070.832470:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756f68 00000800:00000200:12.0:1616513070.832472:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513070.832474:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.832476:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513070.832480:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.92@tcp1 : PUT - for me 00000400:00000200:12.0:1616513070.832489:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.92@tcp1 of length 224 into portal 26 MB=0x601ab57277100 00000400:00000200:12.0:1616513070.832496:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.92@tcp1 of length 224/224 into md 0xf97c3fd9 [1] + 0 00000400:00000200:12.0:1616513070.832501:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2f68 00000400:00000200:12.0:1616513070.832504:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513070.832506:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513070.832511:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2f68 00000800:00000200:12.0:1616513070.832514:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.832518:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.832520:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.832583:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690685273501952 00010000:00000200:23.0:1616513070.832594:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1690685273501952/t0(0) o400->41d5a961-71b9-a8dd-1f83-0c78d27211c6@10.31.140.92@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.832608:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690685273501952, offset 192 00000400:00000200:23.0:1616513070.832615:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.92@tcp1 00000400:00000200:23.0:1616513070.832623:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.92@tcp1 routed destination 00000400:00000200:23.0:1616513070.832639:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.92@tcp1(10.31.140.92@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.832645:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.832652:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.832659:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.832661:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.832666:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.832671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.832672:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513070.832680:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.832685:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513070.832687:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513070.832689:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513070.832692:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513070.832697:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0330 00000800:00000200:2.0:1616513070.832699:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.861211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513070.861224:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.861234:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513070.861241:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.171@tcp1 : PUT - for me 00000400:00000200:30.0:1616513070.861250:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.171@tcp1 of length 224 into portal 26 MB=0x5ffdb9099e100 00000400:00000200:30.0:1616513070.861256:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.171@tcp1 of length 224/224 into md 0xf97c3fe9 [1] + 0 00000400:00000200:30.0:1616513070.861260:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2ee0 00000400:00000200:30.0:1616513070.861263:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.861265:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513070.861269:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ee0 00000800:00000200:30.0:1616513070.861272:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.861274:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.861276:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513070.861278:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513070.861289:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513070.861298:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.171@tcp1 : PUT - for me 00000400:00000200:14.0:1616513070.861307:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.171@tcp1 of length 224 into portal 12 MB=0x5ffdb9099e3c0 00000400:00000200:14.0:1616513070.861313:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.171@tcp1 of length 224/224 into md 0xf96450b9 [1] + 15592 00000400:00000200:14.0:1616513070.861318:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513070.861321:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513070.861329:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.861332:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513070.861332:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:23.0:1616513070.861336:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693372477696 00000800:00000200:14.0:1616513070.861336:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:23.0:1616513070.861347:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688693372477696/t0(0) o400->89ba2de4-4fa9-bfd6-462d-d0adba5313c1@10.31.140.171@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.861361:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693372477696, offset 192 00000400:00000200:23.0:1616513070.861369:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.171@tcp1 00000400:00000200:23.0:1616513070.861377:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.171@tcp1 routed destination 00000400:00000200:23.0:1616513070.861393:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.171@tcp1(10.31.140.171@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513070.861397:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693372478400 00000800:00000200:23.0:1616513070.861399:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.861406:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.861408:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.861410:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:18.0:1616513070.861410:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5e80 x1688693372478400/t0(0) o400->a4226746-b337-7d89-908f-efabe9332e8a@10.31.140.171@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513070.861416:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:23.0:1616513070.861423:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513070.861425:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693372478400, offset 192 00000400:00000200:18.0:1616513070.861432:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.171@tcp1 00000400:00000200:18.0:1616513070.861454:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.171@tcp1 routed destination 00000800:00000200:11.2:1616513070.861463:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513070.861470:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.171@tcp1(10.31.140.171@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513070.861477:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513070.861483:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513070.861486:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513070.861488:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513070.861493:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.861498:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513070.861519:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.861527:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513070.861530:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513070.861537:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513070.861543:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1430 00000800:00000200:20.0:1616513070.861547:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513070.861549:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513070.861552:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.861559:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513070.861561:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513070.861562:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513070.861568:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513070.861574:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd6d8 00000800:00000200:2.0:1616513070.861578:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.862597:0:48481:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513070.862651:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.862661:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513070.862670:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.226@tcp1 : PUT - for me 00000400:00000200:26.0:1616513070.862680:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.226@tcp1 of length 224 into portal 26 MB=0x60392fb7b8040 00000400:00000200:26.0:1616513070.862686:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.226@tcp1 of length 224/224 into md 0xf97c3ff9 [1] + 0 00000400:00000200:26.0:1616513070.862691:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf25d8 00000400:00000200:26.0:1616513070.862696:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513070.862699:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513070.862704:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000800:00000200:26.0:1616513070.862707:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.862709:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513070.862709:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513070.862713:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513070.862768:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692779679547456 00010000:00000200:23.0:1616513070.862779:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1692779679547456/t0(0) o400->69cd725a-0eb6-9212-a530-effa5ed1978b@10.31.140.226@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.862794:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692779679547456, offset 192 00000400:00000200:23.0:1616513070.862803:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.226@tcp1 00000400:00000200:23.0:1616513070.862811:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.226@tcp1 routed destination 00000400:00000200:23.0:1616513070.862827:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.226@tcp1(10.31.140.226@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.862833:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.862839:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.862842:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:11.2:1616513070.862843:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.862844:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.862849:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:23.0:1616513070.862856:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513070.862899:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513070.862909:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513070.862918:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.226@tcp1 : PUT - for me 00000400:00000200:28.0:1616513070.862928:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.226@tcp1 of length 224 into portal 12 MB=0x60392fb7b82c0 00000400:00000200:28.0:1616513070.862934:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.226@tcp1 of length 224/224 into md 0x865b9f31 [1] + 46008 00000400:00000200:28.0:1616513070.862939:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.862942:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513070.862948:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513070.862951:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513070.862953:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513070.862960:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513070.862962:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.862963:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.862965:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.862972:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.862978:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0b28 00000800:00000200:12.0:1616513070.862981:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.863011:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692779679548096 00010000:00000200:24.0:1616513070.863022:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89f80 x1692779679548096/t0(0) o400->c54275a3-0809-c099-3b7c-210d79721722@10.31.140.226@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513070.863036:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692779679548096, offset 192 00000400:00000200:24.0:1616513070.863042:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.226@tcp1 00000400:00000200:24.0:1616513070.863050:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.226@tcp1 routed destination 00000400:00000200:24.0:1616513070.863064:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.226@tcp1(10.31.140.226@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.863070:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513070.863076:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.863078:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513070.863080:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513070.863085:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.863091:0:48481:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.863099:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.863102:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.863104:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.863108:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.863111:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000800:00000200:30.0:1616513070.863113:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.863115:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:19.0:1616513070.870606:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:19.0:1616513070.870614:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000100:00000200:26.0:1616513070.874522:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923904, portal 4 00000100:00000200:26.0:1616513070.874527:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 24, xid 1693067725923904, offset 0 00000400:00000200:26.0:1616513070.874534:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:26.0:1616513070.874540:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:26.0:1616513070.874553:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513070.874559:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:26.0:1616513070.874565:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513070.874568:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:26.0:1616513070.874570:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:26.0:1616513070.874575:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513070.874623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:14.0:1616513070.874638:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:14.0:1616513070.874644:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:14.0:1616513070.874647:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513070.874649:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:14.0:1616513070.874654:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000100:00000200:14.0:1616513070.874660:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b38e300 x1693067725923904/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513070.874672:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afc28 00000800:00000200:14.0:1616513070.874678:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:31.2:1616513070.874752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:30.0:1616513070.874761:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:30.0:1616513070.874768:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.234@o2ib1 00000400:00000200:30.0:1616513070.874774:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513070.874781:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 336 into portal 4 MB=0x603d60c626640 00000400:00000200:30.0:1616513070.874787:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.234@o2ib1 of length 336/336 into md 0xf97c48e9 [1] + 192 00000400:00000200:30.0:1616513070.874791:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.874795:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b38e300 x1693067725923904/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513070.874807:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:30.0:1616513070.874809:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:30.0:1616513070.874811:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:30.0:1616513070.874813:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:26.0:1616513070.874814:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b38e300 x1693067725923904/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513070.874827:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b38e300 x1693067725923904/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513070.874837:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:11.2:1616513070.914424:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513070.914482:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.914492:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513070.914501:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.153@tcp1 : PUT - for me 00000400:00000200:16.0:1616513070.914512:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.153@tcp1 of length 224 into portal 26 MB=0x5ffdbf4c45040 00000400:00000200:16.0:1616513070.914519:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.153@tcp1 of length 224/224 into md 0xf97c4049 [1] + 0 00000400:00000200:16.0:1616513070.914525:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3980 00000400:00000200:16.0:1616513070.914528:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513070.914531:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513070.914536:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3980 00000800:00000200:28.0:1616513070.914537:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.914539:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.914542:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513070.914544:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.914547:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.914548:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513070.914557:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.153@tcp1 : PUT - for me 00000400:00000200:28.0:1616513070.914566:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.153@tcp1 of length 224 into portal 12 MB=0x5ffdbf4c45100 00000400:00000200:28.0:1616513070.914573:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.153@tcp1 of length 224/224 into md 0x865b9f31 [1] + 46232 00000400:00000200:28.0:1616513070.914579:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.914581:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513070.914589:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513070.914592:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.914594:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.914603:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695052980288 00010000:00000200:23.0:1616513070.914614:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688695052980288/t0(0) o400->059e17b0-006a-d7a9-bc1d-970e823ff678@10.31.140.153@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.914629:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695052980288, offset 192 00000400:00000200:23.0:1616513070.914635:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.153@tcp1 00000400:00000200:23.0:1616513070.914644:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.153@tcp1 routed destination 00000100:00000200:24.0:1616513070.914658:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695052980480 00000400:00000200:23.0:1616513070.914659:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.153@tcp1(10.31.140.153@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.914666:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:24.0:1616513070.914672:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8de80 x1688695052980480/t0(0) o400->948f6728-2420-a2cc-267f-ef8f9009aad7@10.31.140.153@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513070.914672:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.914675:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.914678:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.914683:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513070.914686:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695052980480, offset 192 00000800:00000200:11.2:1616513070.914688:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.914690:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513070.914693:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.153@tcp1 00000400:00000200:24.0:1616513070.914701:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.153@tcp1 routed destination 00000400:00000200:24.0:1616513070.914717:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.153@tcp1(10.31.140.153@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513070.914726:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513070.914733:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.914736:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513070.914738:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513070.914743:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513070.914745:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513070.914751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513070.914753:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513070.914756:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513070.914756:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513070.914763:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513070.914769:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1f58 00000800:00000200:20.0:1616513070.914773:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513070.914810:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.914816:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513070.914819:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513070.914825:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513070.914829:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ee0 00000800:00000200:26.0:1616513070.914833:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.914835:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.942205:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513070.942276:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.942285:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513070.942294:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.34@tcp1 : PUT - for me 00000400:00000200:12.0:1616513070.942304:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.34@tcp1 of length 224 into portal 12 MB=0x5ffdb955fd280 00000400:00000200:12.0:1616513070.942310:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.34@tcp1 of length 224/224 into md 0x865b9f31 [1] + 46456 00000400:00000200:12.0:1616513070.942316:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513070.942318:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513070.942325:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.942328:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.942331:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.942335:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513070.942391:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693452558976 00010000:00000200:24.0:1616513070.942403:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa40000 x1688693452558976/t0(0) o400->a3cb54fa-a71f-d315-fbea-9684485f539a@10.31.140.34@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513070.942412:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513070.942418:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693452558976, offset 192 00000400:00000200:24.0:1616513070.942424:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.34@tcp1 00000400:00000200:24.0:1616513070.942432:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.34@tcp1 routed destination 00000800:00000200:16.0:1616513070.942456:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513070.942460:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.34@tcp1(10.31.140.34@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513070.942466:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513070.942467:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513070.942473:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513070.942475:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.34@tcp1 : PUT - for me 00000800:00000200:24.0:1616513070.942476:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513070.942478:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513070.942483:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513070.942485:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.34@tcp1 of length 224 into portal 26 MB=0x5ffdb955fd200 00000400:00000200:16.0:1616513070.942492:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.34@tcp1 of length 224/224 into md 0xf97c4079 [1] + 0 00000400:00000200:16.0:1616513070.942497:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf35c8 00000400:00000200:16.0:1616513070.942500:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513070.942503:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513070.942507:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf35c8 00000800:00000200:16.0:1616513070.942511:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513070.942512:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513070.942514:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513070.942519:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513070.942520:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513070.942522:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513070.942523:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513070.942530:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513070.942534:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2f68 00000800:00000200:20.0:1616513070.942537:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.942574:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693452558848 00010000:00000200:23.0:1616513070.942585:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688693452558848/t0(0) o400->a53203dd-a62b-3017-fb93-4aba0514f6b9@10.31.140.34@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.942600:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693452558848, offset 192 00000400:00000200:23.0:1616513070.942606:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.34@tcp1 00000400:00000200:23.0:1616513070.942614:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.34@tcp1 routed destination 00000400:00000200:23.0:1616513070.942629:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.34@tcp1(10.31.140.34@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.942641:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.942647:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.942650:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.942653:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.942658:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.942664:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.942665:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513070.942677:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.942684:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.942688:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.942695:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.942701:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d14b8 00000800:00000200:30.0:1616513070.942705:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.942707:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.963474:0:48492:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513070.963531:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.963540:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513070.963550:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.117@tcp1 : PUT - for me 00000400:00000200:26.0:1616513070.963559:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.117@tcp1 of length 224 into portal 12 MB=0x5f9153f628700 00000400:00000200:26.0:1616513070.963568:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.117@tcp1 of length 224/224 into md 0xf96450b9 [1] + 15816 00000400:00000200:26.0:1616513070.963574:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513070.963577:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513070.963584:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513070.963584:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.963587:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513070.963589:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513070.963651:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681244536604416 00010000:00000200:18.0:1616513070.963665:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c694380 x1681244536604416/t0(0) o400->922f3e09-2c84-ca46-0162-ce7e015a2d15@10.31.143.117@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513070.963680:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681244536604416, offset 192 00000400:00000200:18.0:1616513070.963686:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.117@tcp1 00000400:00000200:18.0:1616513070.963694:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.117@tcp1 routed destination 00000400:00000200:18.0:1616513070.963709:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.117@tcp1(10.31.143.117@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513070.963718:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513070.963723:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513070.963726:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513070.963728:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513070.963733:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.963741:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513070.963803:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513070.963810:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513070.963813:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513070.963820:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513070.963824:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cda08 00000800:00000200:12.0:1616513070.963828:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513070.963831:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513070.965031:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513070.965086:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513070.965095:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513070.965104:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.117@tcp1 : PUT - for me 00000400:00000200:28.0:1616513070.965113:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.117@tcp1 of length 224 into portal 26 MB=0x5f9153f628680 00000400:00000200:28.0:1616513070.965122:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.117@tcp1 of length 224/224 into md 0xf97c40a9 [1] + 0 00000400:00000200:28.0:1616513070.965127:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2330 00000400:00000200:28.0:1616513070.965130:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513070.965132:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513070.965137:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2330 00000800:00000200:28.0:1616513070.965140:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513070.965140:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513070.965143:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513070.965144:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513070.965203:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681244536604288 00010000:00000200:23.0:1616513070.965214:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1681244536604288/t0(0) o400->4c1afb07-d0b6-9889-8c1f-ce80b499925b@10.31.143.117@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513070.965229:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681244536604288, offset 192 00000400:00000200:23.0:1616513070.965235:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.117@tcp1 00000400:00000200:23.0:1616513070.965243:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.117@tcp1 routed destination 00000400:00000200:23.0:1616513070.965259:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.117@tcp1(10.31.143.117@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.965265:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513070.965272:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.965275:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513070.965277:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513070.965282:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.965288:0:48492:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513070.965289:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513070.965299:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.965304:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.965306:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.965312:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.965316:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1e48 00000800:00000200:30.0:1616513070.965319:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.965321:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513070.967276:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513070.967330:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513070.967339:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513070.967348:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.155@tcp1 : PUT - for me 00000400:00000200:14.0:1616513070.967357:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.155@tcp1 of length 224 into portal 26 MB=0x603e13d6e7840 00000400:00000200:14.0:1616513070.967363:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.155@tcp1 of length 224/224 into md 0xf97c40b9 [1] + 0 00000400:00000200:14.0:1616513070.967368:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2440 00000400:00000200:14.0:1616513070.967371:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513070.967374:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513070.967379:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2440 00000800:00000200:14.0:1616513070.967382:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513070.967384:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513070.967385:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513070.967388:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513070.967449:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:23.0:1616513070.967459:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115793438784 00000800:00000200:12.0:1616513070.967461:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:23.0:1616513070.967470:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1693115793438784/t0(0) o400->290345ad-4f3d-852e-939a-5ea5613bea39@10.31.136.155@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513070.967470:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513070.967479:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.155@tcp1 : PUT - for me 00000100:00000200:23.0:1616513070.967485:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693115793438784, offset 192 00000400:00000200:12.0:1616513070.967488:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.155@tcp1 of length 224 into portal 12 MB=0x603e13d6e7900 00000400:00000200:23.0:1616513070.967493:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.155@tcp1 00000400:00000200:12.0:1616513070.967495:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.155@tcp1 of length 224/224 into md 0x865b9f31 [1] + 46680 00000400:00000200:12.0:1616513070.967500:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:23.0:1616513070.967501:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.155@tcp1 routed destination 00000100:00000200:12.0:1616513070.967502:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:23.0:1616513070.967517:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.155@tcp1(10.31.136.155@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513070.967522:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:23.0:1616513070.967523:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513070.967523:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:23.0:1616513070.967529:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513070.967531:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:23.0:1616513070.967532:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513070.967534:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115793438976 00000800:00000200:12.0:1616513070.967534:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:23.0:1616513070.967536:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.967541:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:24.0:1616513070.967546:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa41200 x1693115793438976/t0(0) o400->69b84329-3ff4-eaaf-8b93-15559330dff7@10.31.136.155@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513070.967546:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.967548:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513070.967561:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693115793438976, offset 192 00000400:00000200:24.0:1616513070.967567:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.155@tcp1 00000400:00000200:24.0:1616513070.967575:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.155@tcp1 routed destination 00000800:00000200:20.0:1616513070.967584:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513070.967590:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.155@tcp1(10.31.136.155@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513070.967591:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513070.967595:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513070.967597:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513070.967597:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513070.967601:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513070.967604:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.967607:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513070.967607:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0880 00000800:00000200:24.0:1616513070.967609:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513070.967611:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513070.967614:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.967619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513070.967628:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513070.967631:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513070.967633:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513070.967638:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513070.967641:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:30.0:1616513070.967643:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513070.967645:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:13.0:1616513070.971657:0:6493:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923920, portal 4 00000100:00000200:13.0:1616513070.971662:0:6493:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923920, offset 0 00000400:00000200:13.0:1616513070.971668:0:6493:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:13.0:1616513070.971674:0:6493:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:13.0:1616513070.971687:0:6493:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:13.0:1616513070.971693:0:6493:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:13.0:1616513070.971699:0:6493:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513070.971702:0:6493:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:13.0:1616513070.971704:0:6493:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:13.0:1616513070.971709:0:6493:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513070.971757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513070.971813:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513070.971820:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513070.971823:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513070.971830:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513070.971837:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4558ad7980 x1693067725923920/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513070.971849:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345ed0 00000800:00000200:26.0:1616513070.971853:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513070.971856:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513070.971936:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513070.971947:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513070.971954:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:30.0:1616513070.971974:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513070.971981:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626650 00000400:00000200:30.0:1616513070.971987:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c4959 [1] + 192 00000400:00000200:30.0:1616513070.971991:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513070.971996:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4558ad7980 x1693067725923920/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513070.972010:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513070.972011:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:30.0:1616513070.972012:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:30.0:1616513070.972016:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:13.0:1616513070.972020:0:6493:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4558ad7980 x1693067725923920/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:13.0:1616513070.972033:0:6493:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4558ad7980 x1693067725923920/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513077 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:13.0:1616513070.972044:0:6493:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178344908 00000800:00000200:7.2:1616513070.981343:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513070.981398:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.981407:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513070.981415:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.32@tcp1 : PUT - for me 00000400:00000200:26.0:1616513070.981423:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.32@tcp1 of length 224 into portal 26 MB=0x5faa0911fe6c0 00000400:00000200:26.0:1616513070.981430:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.32@tcp1 of length 224/224 into md 0xf97c40c9 [1] + 0 00000400:00000200:26.0:1616513070.981448:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3210 00000400:00000200:26.0:1616513070.981450:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513070.981451:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513070.981452:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513070.981453:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513070.981458:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3210 00000800:00000200:26.0:1616513070.981461:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513070.981464:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513070.981465:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513070.981511:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513070.981520:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:23.0:1616513070.981526:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682942420051648 00000400:00000200:16.0:1616513070.981529:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.32@tcp1 : PUT - for me 00010000:00000200:23.0:1616513070.981537:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1682942420051648/t0(0) o400->2ad44288-9f89-1246-cc51-f3aab955947a@10.31.132.32@tcp1:27/0 lens 224/224 e 0 to 0 dl 1616513162 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513070.981539:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.32@tcp1 of length 224 into portal 12 MB=0x5faa0911fe740 00000400:00000200:16.0:1616513070.981545:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.32@tcp1 of length 224/224 into md 0x865b9f31 [1] + 46904 00000400:00000200:16.0:1616513070.981550:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:23.0:1616513070.981552:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1682942420051648, offset 192 00000100:00000200:16.0:1616513070.981553:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:23.0:1616513070.981559:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.32@tcp1 00000800:00000200:16.0:1616513070.981559:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513070.981562:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513070.981562:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513070.981566:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:23.0:1616513070.981567:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.32@tcp1 routed destination 00000400:00000200:23.0:1616513070.981583:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.32@tcp1(10.31.132.32@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513070.981589:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513070.981595:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513070.981598:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513070.981600:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513070.981605:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513070.981610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513070.981611:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513070.981625:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682942420051776 00010000:00000200:24.0:1616513070.981639:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e6c00 x1682942420051776/t0(0) o400->b903c161-7a2e-553a-e1ce-b1de11261aac@10.31.132.32@tcp1:696/0 lens 224/224 e 0 to 0 dl 1616513076 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513070.981653:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1682942420051776, offset 192 00000400:00000200:24.0:1616513070.981660:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.32@tcp1 00000800:00000200:28.0:1616513070.981664:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513070.981667:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.32@tcp1 routed destination 00000800:00000200:28.0:1616513070.981673:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513070.981677:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0F:1616513070.981679:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513070.981683:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513070.981683:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.32@tcp1(10.31.132.32@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513070.981689:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0880 00000800:00000200:24.0:1616513070.981689:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513070.981693:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513070.981697:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513070.981699:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513070.981702:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513070.981707:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513070.981714:0:48494:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513070.981769:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513070.981776:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513070.981779:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513070.981780:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513070.981785:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513070.981789:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3cb0 00000800:00000200:2.0:1616513070.981793:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:25.0:1616513071.034703:0:6494:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923936, portal 4 00000100:00000200:25.0:1616513071.034712:0:6494:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923936, offset 0 00000400:00000200:25.0:1616513071.034719:0:6494:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:25.0:1616513071.034725:0:6494:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:25.0:1616513071.034738:0:6494:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:25.0:1616513071.034745:0:6494:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:25.0:1616513071.034752:0:6494:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:25.0:1616513071.034755:0:6494:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:25.0:1616513071.034757:0:6494:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:25.0:1616513071.034762:0:6494:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513071.034769:0:6494:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513071.034783:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513071.034790:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513071.034793:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513071.034800:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513071.034806:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843f7b9f80 x1693067725923936/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513071.034818:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84926734b8 00000800:00000200:14.0:1616513071.034822:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513071.034825:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513071.034980:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513071.034991:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513071.035008:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:30.0:1616513071.035016:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513071.035023:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626660 00000400:00000200:30.0:1616513071.035030:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c4979 [1] + 192 00000400:00000200:30.0:1616513071.035034:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.035039:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843f7b9f80 x1693067725923936/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513071.035052:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:30.0:1616513071.035054:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:30.0:1616513071.035056:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:30.0:1616513071.035058:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:25.0:1616513071.035062:0:6494:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843f7b9f80 x1693067725923936/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:25.0:1616513071.035075:0:6494:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843f7b9f80 x1693067725923936/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:25.0:1616513071.035086:0:6494:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84926736d8 00000800:00000200:7.2:1616513071.041340:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513071.041397:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.041406:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513071.041414:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.77@tcp1 : PUT - for me 00000400:00000200:14.0:1616513071.041422:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.77@tcp1 of length 224 into portal 26 MB=0x5f90935c4a740 00000400:00000200:14.0:1616513071.041428:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.77@tcp1 of length 224/224 into md 0xf97c40f9 [1] + 0 00000400:00000200:14.0:1616513071.041441:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3f58 00000400:00000200:14.0:1616513071.041444:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513071.041447:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513071.041450:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513071.041452:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3f58 00000800:00000200:14.0:1616513071.041457:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513071.041460:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513071.041461:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513071.041461:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.041464:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513071.041470:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.77@tcp1 : PUT - for me 00000400:00000200:26.0:1616513071.041479:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.77@tcp1 of length 224 into portal 12 MB=0x5f90935c4a7c0 00000400:00000200:26.0:1616513071.041485:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.77@tcp1 of length 224/224 into md 0xf96450b9 [1] + 16040 00000400:00000200:26.0:1616513071.041492:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513071.041495:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513071.041503:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.041507:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513071.041508:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.041521:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681192835655488 00010000:00000200:23.0:1616513071.041533:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1681192835655488/t0(0) o400->b401a576-56c2-732b-8c42-d29989cfe911@10.31.131.77@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.041548:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681192835655488, offset 192 00000400:00000200:23.0:1616513071.041554:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.77@tcp1 00000400:00000200:23.0:1616513071.041562:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.77@tcp1 routed destination 00000100:00000200:18.0:1616513071.041570:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681192835655616 00000400:00000200:23.0:1616513071.041577:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.77@tcp1(10.31.131.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:18.0:1616513071.041582:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691b00 x1681192835655616/t0(0) o400->7f8693ee-07df-8e3a-39f9-226c6baf5ae4@10.31.131.77@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513071.041584:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513071.041590:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.041593:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513071.041595:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:18.0:1616513071.041596:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681192835655616, offset 192 00000800:00000200:23.0:1616513071.041600:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513071.041602:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.77@tcp1 00000100:00000200:23.0:1616513071.041607:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513071.041609:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.77@tcp1 routed destination 00000400:00000200:18.0:1616513071.041624:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.77@tcp1(10.31.131.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.041630:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513071.041636:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.041638:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513071.041640:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513071.041645:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513071.041649:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513071.041651:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.041705:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513071.041706:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513071.041712:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513071.041714:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513071.041715:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513071.041715:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513071.041717:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.041721:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513071.041724:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.041725:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd8f8 00000800:00000200:2.0:1616513071.041728:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513071.041730:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d14b8 00000800:00000200:12.0:1616513071.041733:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.041736:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:29.0:1616513071.082685:0:6495:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923952, portal 4 00000100:00000200:1.0:1616513071.082688:0:6488:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923968, portal 4 00000100:00000200:29.0:1616513071.082691:0:6495:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923952, offset 0 00000100:00000200:1.0:1616513071.082694:0:6488:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923968, offset 0 00000400:00000200:29.0:1616513071.082699:0:6495:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:29.0:1616513071.082704:0:6495:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:1.0:1616513071.082704:0:6488:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:29.0:1616513071.082717:0:6495:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513071.082724:0:6495:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:1.0:1616513071.082724:0:6488:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:29.0:1616513071.082733:0:6495:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513071.082735:0:6495:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:29.0:1616513071.082738:0:6495:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:1.0:1616513071.082738:0:6488:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513071.082742:0:6495:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513071.082750:0:6488:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513071.082760:0:6488:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:5.0:1616513071.082763:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513071.082767:0:6488:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513071.082770:0:6488:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:5.0:1616513071.082772:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513071.082772:0:6488:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:5.0:1616513071.082775:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513071.082778:0:6488:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513071.082785:0:6465:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:1.0:1616513071.082789:0:6488:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725923984, portal 4 00000100:00000200:1.0:1616513071.082792:0:6488:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725923984, offset 0 00000400:00000200:5.0:1616513071.082795:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:1.0:1616513071.082797:0:6488:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:1.0:1616513071.082801:0:6488:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000100:00000200:5.0:1616513071.082802:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843a2ec380 x1693067725923952/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513071.082813:0:6488:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:5.0:1616513071.082817:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2880 00000800:00000200:1.0:1616513071.082820:0:6488:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:5.0:1616513071.082822:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:17.0:1616513071.082825:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.0:1616513071.082825:0:6488:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513071.082828:0:6488:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:1.0:1616513071.082829:0:6488:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:1.0:1616513071.082833:0:6488:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:0.0:1616513071.082841:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:0.0:1616513071.082849:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:0.0:1616513071.082852:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513071.082859:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:0.0:1616513071.082866:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8302321680 x1693067725923968/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513071.082878:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85a90 00000800:00000200:27.2:1616513071.082883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:0.0:1616513071.082883:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513071.082886:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.0:1616513071.082904:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513071.082909:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513071.082914:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513071.082920:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513071.082927:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1390d80 x1693067725923984/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513071.082939:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85650 00000800:00000200:27.0:1616513071.082943:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513071.082946:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:1.2:1616513071.082964:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513071.082973:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513071.082979:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513071.082981:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513071.082987:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513071.082995:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626670 00000800:00000200:5.2:1616513071.083000:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000400:00000200:1.0:1616513071.083001:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff8e5 [1] + 192 00000400:00000200:1.0:1616513071.083007:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513071.083012:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843a2ec380 x1693067725923952/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513071.083027:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513071.083030:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513071.083032:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:29.0:1616513071.083034:0:6495:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843a2ec380 x1693067725923952/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:29.0:1616513071.083047:0:6495:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843a2ec380 x1693067725923952/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513071.083055:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:29.0:1616513071.083059:0:6495:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2990 00000800:00000200:12.0:1616513071.083065:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:27.2:1616513071.083069:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000400:00000200:12.0:1616513071.083074:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513071.083083:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626680 00000800:00000200:27.0:1616513071.083089:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:12.0:1616513071.083090:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d2c21 [1] + 192 00000800:00000200:27.0:1616513071.083094:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:12.0:1616513071.083095:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:31.0:1616513071.083098:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:12.0:1616513071.083101:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8302321680 x1693067725923968/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513071.083102:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513071.083110:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626690 00000800:00000200:16.0:1616513071.083114:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:27.0:1616513071.083116:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716502d [1] + 192 00000800:00000200:12.0:1616513071.083117:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:27.0:1616513071.083120:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513071.083120:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513071.083122:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:27.0:1616513071.083125:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1390d80 x1693067725923984/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513071.083125:0:6488:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8302321680 x1693067725923968/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513071.083136:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:1.0:1616513071.083136:0:6488:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8302321680 x1693067725923968/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513071.083139:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513071.083140:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:1.0:1616513071.083146:0:6488:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85cb0 00000100:00000200:1.0:1616513071.083163:0:6488:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1390d80 x1693067725923984/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513071.083173:0:6488:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1390d80 x1693067725923984/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513071.083181:0:6488:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85100 00000100:00000200:17.0:1616513071.083474:0:6490:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924000, portal 4 00000100:00000200:17.0:1616513071.083478:0:6490:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924000, offset 0 00000400:00000200:17.0:1616513071.083485:0:6490:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:17.0:1616513071.083491:0:6490:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:17.0:1616513071.083504:0:6490:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:17.0:1616513071.083510:0:6490:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:17.0:1616513071.083517:0:6490:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:17.0:1616513071.083520:0:6490:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:17.0:1616513071.083522:0:6490:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:17.0:1616513071.083527:0:6490:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513071.083535:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513071.083547:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:30.0:1616513071.083553:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:30.0:1616513071.083556:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513071.083562:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:30.0:1616513071.083567:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d853e8c9f80 x1693067725924000/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513071.083578:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3eb28 00000800:00000200:30.0:1616513071.083582:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:30.0:1616513071.083585:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513071.083710:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513071.083764:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513071.083773:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513071.083781:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513071.083789:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6266a0 00000400:00000200:14.0:1616513071.083796:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c49a9 [1] + 192 00000400:00000200:14.0:1616513071.083801:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513071.083806:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d853e8c9f80 x1693067725924000/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513071.083818:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513071.083820:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513071.083823:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513071.083825:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:17.0:1616513071.083830:0:6490:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d853e8c9f80 x1693067725924000/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:17.0:1616513071.083843:0:6490:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d853e8c9f80 x1693067725924000/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:17.0:1616513071.083853:0:6490:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3fc28 00000800:00000200:7.2:1616513071.117918:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513071.117932:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.117945:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513071.117953:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.55@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.117962:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.55@tcp1 of length 224 into portal 26 MB=0x603e2aad703c0 00000400:00000200:30.0:1616513071.117969:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.55@tcp1 of length 224/224 into md 0xf97c4109 [1] + 0 00000400:00000200:30.0:1616513071.117974:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf26e8 00000400:00000200:30.0:1616513071.117976:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.117979:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513071.117984:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf26e8 00000800:00000200:30.0:1616513071.117987:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513071.117989:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.117990:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.117994:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.118052:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693121923974080 00010000:00000200:23.0:1616513071.118064:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1693121923974080/t0(0) o400->68ca986a-2294-f0cb-6fd9-5d2c15b2f673@10.31.136.55@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.118081:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693121923974080, offset 192 00000400:00000200:23.0:1616513071.118087:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.55@tcp1 00000400:00000200:23.0:1616513071.118095:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.55@tcp1 routed destination 00000400:00000200:23.0:1616513071.118111:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.55@tcp1(10.31.136.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.118118:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:11.2:1616513071.118121:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513071.118124:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.118127:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513071.118129:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:23.0:1616513071.118134:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.118136:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:23.0:1616513071.118141:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513071.118146:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.118156:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.55@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.118166:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.55@tcp1 of length 224 into portal 12 MB=0x603e2aad70500 00000400:00000200:0.0:1616513071.118172:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.55@tcp1 of length 224/224 into md 0x865b9f31 [1] + 47128 00000400:00000200:0.0:1616513071.118178:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.118180:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.118189:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513071.118191:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513071.118192:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513071.118198:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513071.118200:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.118201:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.118204:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.118210:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.118217:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0b28 00000800:00000200:28.0:1616513071.118220:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.118258:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693121923974400 00010000:00000200:24.0:1616513071.118272:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d1680 x1693121923974400/t0(0) o400->e39b0cba-f814-015c-7e3d-76bcb0cb22b3@10.31.136.55@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.118288:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693121923974400, offset 192 00000400:00000200:24.0:1616513071.118294:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.55@tcp1 00000400:00000200:24.0:1616513071.118302:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.55@tcp1 routed destination 00000400:00000200:24.0:1616513071.118318:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.55@tcp1(10.31.136.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.118325:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513071.118331:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.118334:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513071.118336:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513071.118342:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.118347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.118402:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.118408:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.118411:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.118417:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.118421:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf34b8 00000800:00000200:2.0:1616513071.118425:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.118428:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513071.120823:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513071.120838:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.120847:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.120856:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.173@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.120865:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.173@tcp1 of length 224 into portal 26 MB=0x60198a41b6100 00000400:00000200:12.0:1616513071.120872:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.173@tcp1 of length 224/224 into md 0xf97c4119 [1] + 0 00000400:00000200:12.0:1616513071.120877:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3760 00000400:00000200:12.0:1616513071.120880:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.120883:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513071.120888:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3760 00000800:00000200:12.0:1616513071.120891:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.120891:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.120894:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.120898:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.120902:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513071.120903:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513071.120910:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.173@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.120919:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.173@tcp1 of length 224 into portal 12 MB=0x60198a41b6240 00000400:00000200:0.0:1616513071.120925:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.173@tcp1 of length 224/224 into md 0x865b9f31 [1] + 47352 00000400:00000200:0.0:1616513071.120931:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.120933:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.120940:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.120943:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.120945:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.120955:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690604960178432 00000100:00000200:24.0:1616513071.120962:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690604960178752 00010000:00000200:23.0:1616513071.120966:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1690604960178432/t0(0) o400->cfbbb5b6-7d69-7b6b-4834-c68789b3c395@10.31.141.173@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:24.0:1616513071.120975:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5580 x1690604960178752/t0(0) o400->725147e0-6daf-f1f7-77e6-e8a417a1f595@10.31.141.173@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:23.0:1616513071.120980:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690604960178432, offset 192 00000400:00000200:23.0:1616513071.120986:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.173@tcp1 00000100:00000200:24.0:1616513071.120990:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690604960178752, offset 192 00000400:00000200:23.0:1616513071.120994:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.173@tcp1 routed destination 00000400:00000200:24.0:1616513071.120996:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.173@tcp1 00000400:00000200:24.0:1616513071.121004:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.173@tcp1 routed destination 00000400:00000200:24.0:1616513071.121024:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.173@tcp1(10.31.141.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:23.0:1616513071.121024:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.173@tcp1(10.31.141.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.121031:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.121032:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.121037:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.121039:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.121041:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.121042:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.121043:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.121044:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.121048:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513071.121049:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513071.121053:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513071.121055:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513071.121094:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513071.121108:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.121116:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.121119:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513071.121120:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513071.121126:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513071.121132:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0330 00000800:00000200:16.0:1616513071.121136:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513071.121151:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513071.121158:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.121161:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513071.121161:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513071.121167:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513071.121171:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:26.0:1616513071.121175:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513071.143065:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.143125:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.143135:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513071.143144:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.105@tcp1 : PUT - for me 00000400:00000200:2.0:1616513071.143153:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.105@tcp1 of length 224 into portal 26 MB=0x5ffdbffed94c0 00000400:00000200:2.0:1616513071.143159:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.105@tcp1 of length 224/224 into md 0xf97c4129 [1] + 0 00000400:00000200:2.0:1616513071.143164:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3650 00000400:00000200:2.0:1616513071.143167:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513071.143170:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513071.143175:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3650 00000800:00000200:2.0:1616513071.143178:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.143179:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.143181:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513071.143186:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513071.143189:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513071.143189:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513071.143198:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.105@tcp1 : PUT - for me 00000400:00000200:14.0:1616513071.143207:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.105@tcp1 of length 224 into portal 12 MB=0x5ffdbffed97c0 00000400:00000200:14.0:1616513071.143213:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.105@tcp1 of length 224/224 into md 0xf96450b9 [1] + 16264 00000400:00000200:14.0:1616513071.143219:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513071.143221:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513071.143229:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.143232:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.143234:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.143242:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695240234176 00010000:00000200:23.0:1616513071.143253:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688695240234176/t0(0) o400->0bac2642-f8bf-405f-4785-66c46b1bc422@10.31.141.105@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.143267:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695240234176, offset 192 00000400:00000200:23.0:1616513071.143275:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.105@tcp1 00000400:00000200:23.0:1616513071.143283:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.105@tcp1 routed destination 00000100:00000200:18.0:1616513071.143296:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695240234944 00000400:00000200:23.0:1616513071.143301:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.105@tcp1(10.31.141.105@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.143307:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:18.0:1616513071.143309:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa1f80 x1688695240234944/t0(0) o400->1f5924ef-6390-c244-eef9-f03b9f8c771e@10.31.141.105@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513071.143314:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.143316:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513071.143319:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513071.143324:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:18.0:1616513071.143325:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695240234944, offset 192 00000100:00000200:23.0:1616513071.143331:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513071.143331:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.105@tcp1 00000400:00000200:18.0:1616513071.143339:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.105@tcp1 routed destination 00000400:00000200:18.0:1616513071.143354:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.105@tcp1(10.31.141.105@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.143360:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513071.143366:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.143369:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513071.143371:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513071.143371:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.143376:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.143381:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513071.143429:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513071.143450:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513071.143455:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513071.143457:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513071.143458:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513071.143460:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513071.143461:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513071.143465:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513071.143466:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.143471:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1d38 00000400:00000200:26.0:1616513071.143471:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd870 00000800:00000200:28.0:1616513071.143475:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513071.143475:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513071.143478:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.156356:0:48511:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.156412:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.156431:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513071.156452:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.120@tcp1 : PUT - for me 00000400:00000200:2.0:1616513071.156462:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.120@tcp1 of length 224 into portal 26 MB=0x6001dc4b21dc0 00000400:00000200:2.0:1616513071.156468:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.120@tcp1 of length 224/224 into md 0xf97c4139 [1] + 0 00000800:00000200:14.0:1616513071.156469:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513071.156473:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2ff0 00000400:00000200:2.0:1616513071.156476:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513071.156478:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513071.156483:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:2.0:1616513071.156486:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.156488:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.156490:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.156547:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688977714322880 00010000:00000200:23.0:1616513071.156557:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688977714322880/t0(0) o400->2d429ab3-9275-f1c9-6273-12b2168f8c5a@10.31.142.120@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.156571:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688977714322880, offset 192 00000400:00000200:23.0:1616513071.156576:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.120@tcp1 00000400:00000200:23.0:1616513071.156584:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.120@tcp1 routed destination 00000400:00000200:23.0:1616513071.156599:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.120@tcp1(10.31.142.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.156605:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513071.156611:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.156613:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513071.156615:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513071.156620:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:23.0:1616513071.156626:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513071.156665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.156719:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.156727:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513071.156730:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513071.156736:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513071.156742:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d06e8 00000800:00000200:12.0:1616513071.156746:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.156749:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.157174:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513071.157186:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.157194:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513071.157203:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.120@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.157213:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.120@tcp1 of length 224 into portal 12 MB=0x6001dc4b223c0 00000400:00000200:30.0:1616513071.157220:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.120@tcp1 of length 224/224 into md 0xf96450b9 [1] + 16488 00000400:00000200:30.0:1616513071.157225:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.157228:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513071.157235:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.157238:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513071.157239:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.157243:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513071.157299:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688977714324416 00010000:00000200:18.0:1616513071.157311:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38e300 x1688977714324416/t0(0) o400->0fafc198-91c3-a745-9311-7221f9c04cb2@10.31.142.120@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513071.157326:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688977714324416, offset 192 00000400:00000200:18.0:1616513071.157333:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.120@tcp1 00000400:00000200:18.0:1616513071.157340:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.120@tcp1 routed destination 00000400:00000200:18.0:1616513071.157355:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.120@tcp1(10.31.142.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.157362:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513071.157368:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.157371:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513071.157373:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513071.157378:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.157383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.157394:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.157400:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513071.157402:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513071.157404:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513071.157409:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513071.157413:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd7e8 00000800:00000200:14.0:1616513071.157417:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513071.188863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513071.188921:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.188930:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513071.188939:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.104@tcp1 : PUT - for me 00000400:00000200:28.0:1616513071.188950:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.104@tcp1 of length 224 into portal 26 MB=0x6001d2f824e00 00000400:00000200:28.0:1616513071.188957:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.104@tcp1 of length 224/224 into md 0xf97c4169 [1] + 0 00000400:00000200:28.0:1616513071.188962:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf22a8 00000400:00000200:28.0:1616513071.188966:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513071.188969:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513071.188974:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf22a8 00000800:00000200:0.0:1616513071.188975:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.188977:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.188980:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513071.188982:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.188985:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.188985:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.188993:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.104@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.189004:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.104@tcp1 of length 224 into portal 12 MB=0x6001d2f824e80 00000400:00000200:0.0:1616513071.189010:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.104@tcp1 of length 224/224 into md 0x865b9f31 [1] + 47576 00000400:00000200:0.0:1616513071.189014:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.189017:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.189023:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.189026:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.189028:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.189042:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975211384320 00010000:00000200:23.0:1616513071.189054:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688975211384320/t0(0) o400->ad487c92-73cb-631d-30c2-8c04eaf0c89f@10.31.142.104@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.189068:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688975211384320, offset 192 00000400:00000200:23.0:1616513071.189074:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.104@tcp1 00000400:00000200:23.0:1616513071.189084:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.104@tcp1 routed destination 00000100:00000200:24.0:1616513071.189090:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975211384448 00000400:00000200:23.0:1616513071.189099:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.104@tcp1(10.31.142.104@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:24.0:1616513071.189103:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d4c80 x1688975211384448/t0(0) o400->38e4c04f-51d2-5473-c668-b9d794022cc1@10.31.142.104@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513071.189106:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513071.189112:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:24.0:1616513071.189117:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688975211384448, offset 192 00000800:00000200:23.0:1616513071.189117:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513071.189119:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513071.189124:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.104@tcp1 00000800:00000200:23.0:1616513071.189130:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513071.189131:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.104@tcp1 routed destination 00000800:00000200:11.2:1616513071.189135:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513071.189137:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513071.189147:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.104@tcp1(10.31.142.104@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.189153:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513071.189159:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.189162:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513071.189164:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513071.189169:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.189177:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513071.189188:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.189197:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.189200:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513071.189200:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513071.189207:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513071.189213:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0198 00000800:00000200:16.0:1616513071.189218:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513071.189233:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513071.189240:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513071.189243:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513071.189244:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513071.189250:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513071.189254:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2cc0 00000800:00000200:26.0:1616513071.189258:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513071.196453:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513071.196509:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.196518:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.196528:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.16@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.196537:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.16@tcp1 of length 224 into portal 26 MB=0x60393e04ff040 00000400:00000200:12.0:1616513071.196544:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.16@tcp1 of length 224/224 into md 0xf97c4189 [1] + 0 00000400:00000200:12.0:1616513071.196549:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2110 00000400:00000200:12.0:1616513071.196552:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.196554:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513071.196559:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2110 00000800:00000200:12.0:1616513071.196562:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.196563:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.196565:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.196570:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.196573:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.196574:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.196582:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.16@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.196592:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.16@tcp1 of length 224 into portal 12 MB=0x60393e04ff0c0 00000400:00000200:0.0:1616513071.196598:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.16@tcp1 of length 224/224 into md 0x865b9f31 [1] + 47800 00000400:00000200:0.0:1616513071.196603:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.196606:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.196612:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.196616:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.196617:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.196624:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783518675008 00010000:00000200:23.0:1616513071.196635:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1692783518675008/t0(0) o400->0496c335-08df-09df-7c45-3456b406c5cf@10.31.140.16@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.196650:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783518675008, offset 192 00000400:00000200:23.0:1616513071.196657:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.16@tcp1 00000400:00000200:23.0:1616513071.196665:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.16@tcp1 routed destination 00000400:00000200:23.0:1616513071.196680:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.16@tcp1(10.31.140.16@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.196687:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513071.196693:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.196696:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513071.196698:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783518675136 00000800:00000200:23.0:1616513071.196698:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513071.196703:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.196708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513071.196710:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d6c00 x1692783518675136/t0(0) o400->cac226bd-6922-7933-5c39-d83a53f5f2f5@10.31.140.16@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:23.0:1616513071.196711:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513071.196725:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783518675136, offset 192 00000400:00000200:24.0:1616513071.196732:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.16@tcp1 00000400:00000200:24.0:1616513071.196740:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.16@tcp1 routed destination 00000400:00000200:24.0:1616513071.196755:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.16@tcp1(10.31.140.16@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.196762:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513071.196764:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513071.196768:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.196771:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513071.196772:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513071.196774:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513071.196775:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513071.196775:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513071.196779:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513071.196782:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513071.196786:0:48515:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513071.196788:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0198 00000800:00000200:16.0:1616513071.196791:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513071.196843:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.196851:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.196853:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.196860:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.196864:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000800:00000200:2.0:1616513071.196868:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.196871:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513071.257611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513071.257625:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.257633:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513071.257640:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513071.257641:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.11@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.257650:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.11@tcp1 of length 224 into portal 26 MB=0x603d0eaeda940 00000400:00000200:30.0:1616513071.257656:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.21.11@tcp1 of length 224/224 into md 0xf97c41d9 [1] + 0 00000400:00000200:30.0:1616513071.257660:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2880 00000400:00000200:30.0:1616513071.257663:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.257665:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513071.257669:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2880 00000800:00000200:30.0:1616513071.257672:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.257674:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.257675:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.257736:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693045689788736 00010000:00000200:23.0:1616513071.257748:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1693045689788736/t0(0) o400->ec40b684-b04b-5b2b-6eb1-78e53f711d90@10.31.21.11@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.257763:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693045689788736, offset 192 00000400:00000200:23.0:1616513071.257770:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.11@tcp1 00000400:00000200:23.0:1616513071.257779:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.11@tcp1 routed destination 00000400:00000200:23.0:1616513071.257795:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.11@tcp1(10.31.21.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.257802:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513071.257808:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.257811:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513071.257813:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513071.257819:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:23.0:1616513071.257825:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513071.257827:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.257883:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.257891:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.257894:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.257901:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.257907:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1d38 00000800:00000200:28.0:1616513071.257911:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.257914:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.259190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513071.259245:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.259255:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.259264:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.246@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.259274:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.246@tcp1 of length 224 into portal 26 MB=0x605288a2f3f40 00000400:00000200:12.0:1616513071.259280:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.246@tcp1 of length 224/224 into md 0xf97c41f9 [1] + 0 00000400:00000200:12.0:1616513071.259285:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3c28 00000400:00000200:12.0:1616513071.259288:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.259291:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513071.259295:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3c28 00000800:00000200:12.0:1616513071.259298:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.259299:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.259303:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.259305:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.259342:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.259353:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.259361:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694521535446848 00000800:00000200:2.0:1616513071.259362:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513071.259371:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.246@tcp1 : PUT - for me 00010000:00000200:23.0:1616513071.259372:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1694521535446848/t0(0) o400->dc8f1cc4-099b-ac64-3a5d-b02e7f8b08c7@10.31.140.246@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513071.259381:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.246@tcp1 of length 224 into portal 12 MB=0x605288a2f4000 00000100:00000200:23.0:1616513071.259386:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694521535446848, offset 192 00000400:00000200:2.0:1616513071.259387:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.246@tcp1 of length 224/224 into md 0xf96450b9 [1] + 16712 00000400:00000200:2.0:1616513071.259393:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:23.0:1616513071.259394:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.246@tcp1 00000100:00000200:2.0:1616513071.259395:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:23.0:1616513071.259402:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.246@tcp1 routed destination 00000800:00000200:2.0:1616513071.259402:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.259405:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513071.259407:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.259410:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:23.0:1616513071.259418:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.246@tcp1(10.31.140.246@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.259425:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.259431:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.259434:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.259452:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.259457:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.259462:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.259465:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513071.259471:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694521535447040 00010000:00000200:18.0:1616513071.259484:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1389200 x1694521535447040/t0(0) o400->34d8b736-f9cf-2485-95e3-a5c469b0b736@10.31.140.246@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513071.259499:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694521535447040, offset 192 00000400:00000200:18.0:1616513071.259506:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.246@tcp1 00000800:00000200:26.0:1616513071.259514:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513071.259514:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.246@tcp1 routed destination 00000800:00000200:26.0:1616513071.259522:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.259524:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513071.259525:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513071.259530:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.246@tcp1(10.31.140.246@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513071.259532:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513071.259537:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513071.259538:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0b28 00000800:00000200:26.0:1616513071.259542:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513071.259544:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.259546:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.259549:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.259553:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.259559:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.259614:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.259621:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.259623:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.259630:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.259634:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccf68 00000800:00000200:28.0:1616513071.259638:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.259641:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.260748:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513071.260787:0:48522:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.260797:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.260802:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:16.0:1616513071.260804:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513071.260810:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.56@tcp1 : PUT - for me 00000800:00000200:16.0:1616513071.260813:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513071.260816:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.56@tcp1 of length 224 into portal 12 MB=0x5f90c85b4b940 00000400:00000200:2.0:1616513071.260821:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.56@tcp1 of length 224/224 into md 0xf96450b9 [1] + 16936 00000400:00000200:16.0:1616513071.260823:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.56@tcp1 : PUT - for me 00000400:00000200:2.0:1616513071.260825:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513071.260828:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513071.260831:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513071.260832:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.56@tcp1 of length 224 into portal 26 MB=0x5f90c85b4b8c0 00000800:00000200:2.0:1616513071.260833:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513071.260835:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.260837:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513071.260839:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.56@tcp1 of length 224/224 into md 0xf97c4219 [1] + 0 00000400:00000200:16.0:1616513071.260844:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2990 00000400:00000200:16.0:1616513071.260846:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513071.260849:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513071.260855:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2990 00000800:00000200:16.0:1616513071.260858:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.260860:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513071.260862:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513071.260865:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513071.260900:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681207061690688 00010000:00000200:18.0:1616513071.260912:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1389680 x1681207061690688/t0(0) o400->725553bd-5325-7b93-62c8-c1c3d6180426@10.31.131.56@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:23.0:1616513071.260919:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681207061690560 00000100:00000200:18.0:1616513071.260926:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681207061690688, offset 192 00010000:00000200:23.0:1616513071.260930:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1681207061690560/t0(0) o400->06decc27-24fd-7d8b-17c1-049473294dce@10.31.131.56@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513071.260933:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.56@tcp1 00000400:00000200:18.0:1616513071.260941:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.56@tcp1 routed destination 00000100:00000200:23.0:1616513071.260944:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681207061690560, offset 192 00000400:00000200:23.0:1616513071.260951:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.56@tcp1 00000400:00000200:18.0:1616513071.260959:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.56@tcp1(10.31.131.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:23.0:1616513071.260960:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.56@tcp1 routed destination 00000800:00000200:18.0:1616513071.260966:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513071.260973:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:23.0:1616513071.260976:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.56@tcp1(10.31.131.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.260976:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513071.260978:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.260982:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.260983:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.260987:0:48522:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.260989:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.260992:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513071.260994:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513071.260995:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.260998:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:23.0:1616513071.260998:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513071.261000:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513071.261004:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513071.261005:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:23.0:1616513071.261005:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:30.0:1616513071.261008:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd100 00000800:00000200:30.0:1616513071.261010:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.261012:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513071.261065:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.261073:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.261076:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.261083:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.261089:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0880 00000800:00000200:28.0:1616513071.261092:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.261095:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.271575:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513071.271588:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.271597:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.271605:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.210@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.271614:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.210@tcp1 of length 224 into portal 26 MB=0x5ffe12d4c0ec0 00000400:00000200:12.0:1616513071.271620:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.210@tcp1 of length 224/224 into md 0xf97c4239 [1] + 0 00000400:00000200:12.0:1616513071.271624:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3078 00000400:00000200:12.0:1616513071.271627:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.271630:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513071.271634:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:12.0:1616513071.271637:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.271639:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.271641:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513071.271643:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.271654:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513071.271663:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.210@tcp1 : PUT - for me 00000400:00000200:16.0:1616513071.271671:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.210@tcp1 of length 224 into portal 12 MB=0x5ffe12d4c1240 00000400:00000200:16.0:1616513071.271678:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.210@tcp1 of length 224/224 into md 0x865b9f31 [1] + 48024 00000400:00000200:16.0:1616513071.271683:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513071.271686:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513071.271692:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.271696:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.271697:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:23.0:1616513071.271699:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717476236992 00000800:00000200:16.0:1616513071.271701:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:23.0:1616513071.271710:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688717476236992/t0(0) o400->ac37609c-27a7-e22d-84f1-51628cc4c640@10.31.142.210@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.271724:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688717476236992, offset 192 00000400:00000200:23.0:1616513071.271732:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.210@tcp1 00000400:00000200:23.0:1616513071.271740:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.210@tcp1 routed destination 00000400:00000200:23.0:1616513071.271755:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.210@tcp1(10.31.142.210@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513071.271757:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717476237888 00000800:00000200:23.0:1616513071.271762:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.271768:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:24.0:1616513071.271770:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e0d80 x1688717476237888/t0(0) o400->a01b37fc-8be2-bd78-a327-0e8d54fdde93@10.31.142.210@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513071.271771:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.271773:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.271778:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513071.271784:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688717476237888, offset 192 00000100:00000200:23.0:1616513071.271784:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513071.271784:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513071.271790:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.210@tcp1 00000800:00000200:2.0:1616513071.271795:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513071.271798:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.210@tcp1 routed destination 00000800:00000200:2.0:1616513071.271799:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.271802:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.271807:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.271812:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1d38 00000400:00000200:24.0:1616513071.271813:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.210@tcp1(10.31.142.210@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513071.271815:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.271817:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513071.271820:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.271826:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.271828:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.271831:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.271836:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.271840:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.271908:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.271915:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513071.271918:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513071.271919:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513071.271925:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513071.271929:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3430 00000800:00000200:0.0:1616513071.271933:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.274307:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513071.274316:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.274323:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513071.274330:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.222@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.274337:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.222@tcp1 of length 224 into portal 26 MB=0x5ffe5f00a80c0 00000400:00000200:30.0:1616513071.274343:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.222@tcp1 of length 224/224 into md 0xf97c4249 [1] + 0 00000400:00000200:30.0:1616513071.274346:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2c38 00000400:00000200:30.0:1616513071.274348:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.274351:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513071.274355:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:30.0:1616513071.274357:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.274359:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.274360:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.274362:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.274418:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737923367104 00010000:00000200:23.0:1616513071.274428:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688737923367104/t0(0) o400->9d6a268d-70f0-9ec2-2861-fe50fdf38fd3@10.31.141.222@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.274455:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688737923367104, offset 192 00000400:00000200:23.0:1616513071.274462:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.222@tcp1 00000400:00000200:23.0:1616513071.274469:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.222@tcp1 routed destination 00000400:00000200:23.0:1616513071.274484:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.222@tcp1(10.31.141.222@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.274490:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.274496:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.274499:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.274501:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.274506:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.274510:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.274512:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513071.274519:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.274523:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.274525:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.274530:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.274534:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1d38 00000800:00000200:2.0:1616513071.274536:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.274538:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513071.276580:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513071.276593:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.276601:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513071.276621:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.222@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.276627:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.222@tcp1 of length 224 into portal 12 MB=0x5ffe5f00a8280 00000400:00000200:30.0:1616513071.276632:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.222@tcp1 of length 224/224 into md 0xf96450b9 [1] + 17160 00000400:00000200:30.0:1616513071.276636:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.276639:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513071.276644:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.276646:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.276647:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.276649:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513071.276709:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737923367552 00010000:00000200:18.0:1616513071.276722:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7ad00 x1688737923367552/t0(0) o400->3fd0be8b-9b7d-face-051c-22095d31e46b@10.31.141.222@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513071.276736:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688737923367552, offset 192 00000400:00000200:18.0:1616513071.276743:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.222@tcp1 00000400:00000200:18.0:1616513071.276750:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.222@tcp1 routed destination 00000400:00000200:18.0:1616513071.276765:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.222@tcp1(10.31.141.222@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.276772:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.276778:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.276781:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.276783:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.276787:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.276833:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.276888:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.276898:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.276900:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.276906:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.276910:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd320 00000800:00000200:28.0:1616513071.276916:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.276918:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.280420:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513071.280488:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.280497:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.280505:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.129@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.280514:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.129@tcp1 of length 224 into portal 26 MB=0x603fc0bf28d00 00000400:00000200:12.0:1616513071.280521:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.129@tcp1 of length 224/224 into md 0xf97c4269 [1] + 0 00000400:00000200:12.0:1616513071.280525:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3a08 00000400:00000200:12.0:1616513071.280528:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.280530:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513071.280535:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a08 00000800:00000200:12.0:1616513071.280537:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.280540:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.280540:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513071.280547:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.280549:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.280550:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.280558:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.129@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.280567:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.129@tcp1 of length 224 into portal 12 MB=0x603fc0bf28d80 00000400:00000200:0.0:1616513071.280572:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.129@tcp1 of length 224/224 into md 0x865b9f31 [1] + 48248 00000400:00000200:0.0:1616513071.280577:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.280579:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.280586:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.280590:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.280592:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.280598:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230927351040 00010000:00000200:23.0:1616513071.280609:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1693230927351040/t0(0) o400->bf7ee353-6557-b743-23fa-219e8e8a2b62@10.31.138.129@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.280623:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230927351040, offset 192 00000400:00000200:23.0:1616513071.280629:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.129@tcp1 00000400:00000200:23.0:1616513071.280637:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.129@tcp1 routed destination 00000400:00000200:23.0:1616513071.280652:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.129@tcp1(10.31.138.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.280662:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.280668:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.280671:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.280673:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513071.280677:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230927351168 00000800:00000200:23.0:1616513071.280677:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513071.280684:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513071.280684:0:48524:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:24.0:1616513071.280690:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d6780 x1693230927351168/t0(0) o400->9ef2b2b6-1937-131f-c978-edecb69a4fdc@10.31.138.129@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.280704:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230927351168, offset 192 00000400:00000200:24.0:1616513071.280711:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.129@tcp1 00000400:00000200:24.0:1616513071.280719:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.129@tcp1 routed destination 00000400:00000200:24.0:1616513071.280734:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.129@tcp1(10.31.138.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.280741:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.280746:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513071.280746:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513071.280749:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.280752:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513071.280754:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513071.280756:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513071.280757:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513071.280761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513071.280764:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.280770:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1d38 00000800:00000200:2.0:1616513071.280774:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.280776:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513071.280821:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.280828:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513071.280832:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513071.280832:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513071.280839:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513071.280843:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:16.0:1616513071.280846:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:11.0:1616513071.306604:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924016, portal 4 00000100:00000200:11.0:1616513071.306609:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924016, offset 0 00000400:00000200:11.0:1616513071.306615:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:11.0:1616513071.306624:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:11.0:1616513071.306636:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513071.306642:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:11.0:1616513071.306649:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513071.306651:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:11.0:1616513071.306653:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:11.0:1616513071.306658:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513071.306705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513071.306724:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513071.306730:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513071.306733:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513071.306740:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513071.306746:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded7500 x1693067725924016/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513071.306761:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3add0 00000800:00000200:1.0:1616513071.306766:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513071.306768:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513071.306888:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513071.306898:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513071.306904:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513071.306913:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513071.306921:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6266b0 00000400:00000200:1.0:1616513071.306928:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff935 [1] + 192 00000400:00000200:1.0:1616513071.306934:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513071.306938:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded7500 x1693067725924016/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513071.306949:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513071.306952:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513071.306955:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513071.306957:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:11.0:1616513071.306959:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded7500 x1693067725924016/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513071.306971:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded7500 x1693067725924016/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513071.306980:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3aaa0 00000800:00000200:7.2:1616513071.326147:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513071.326159:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.326166:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513071.326173:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.91@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.326181:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.91@tcp1 of length 224 into portal 26 MB=0x602d30b447c40 00000400:00000200:30.0:1616513071.326186:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.91@tcp1 of length 224/224 into md 0xf97c4389 [1] + 0 00000400:00000200:30.0:1616513071.326190:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2e58 00000400:00000200:30.0:1616513071.326192:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.326195:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513071.326198:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2e58 00000800:00000200:30.0:1616513071.326200:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.326202:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.326204:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.326205:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.326209:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513071.326211:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513071.326214:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.91@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.326218:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.91@tcp1 of length 224 into portal 12 MB=0x602d30b447cc0 00000400:00000200:30.0:1616513071.326222:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.91@tcp1 of length 224/224 into md 0xf96450b9 [1] + 17384 00000400:00000200:30.0:1616513071.326225:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.326227:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513071.326232:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.326233:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.326234:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.326267:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691955310656576 00010000:00000200:23.0:1616513071.326279:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1691955310656576/t0(0) o400->2c4321b6-f028-873d-c5c5-6cc493f0f475@10.31.140.91@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.326293:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691955310656576, offset 192 00000100:00000200:18.0:1616513071.326295:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691955310656704 00000400:00000200:23.0:1616513071.326301:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.91@tcp1 00010000:00000200:18.0:1616513071.326308:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7b180 x1691955310656704/t0(0) o400->35de64a1-b486-0b4f-4367-e2502302382c@10.31.140.91@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:23.0:1616513071.326309:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.91@tcp1 routed destination 00000100:00000200:18.0:1616513071.326322:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691955310656704, offset 192 00000400:00000200:23.0:1616513071.326325:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.91@tcp1(10.31.140.91@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513071.326328:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.91@tcp1 00000800:00000200:23.0:1616513071.326342:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513071.326344:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.91@tcp1 routed destination 00000800:00000200:23.0:1616513071.326348:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.326351:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.326353:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.326358:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513071.326358:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.91@tcp1(10.31.140.91@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513071.326363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.326365:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513071.326365:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.326370:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.326373:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.326377:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.326381:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.326390:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513071.326416:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.326425:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513071.326429:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513071.326429:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513071.326446:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.326452:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513071.326453:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.326455:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513071.326459:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f3a8 00000400:00000200:28.0:1616513071.326461:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513071.326463:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513071.326465:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc440 00000800:00000200:28.0:1616513071.326469:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.326472:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.328419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.328473:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.328483:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513071.328484:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513071.328494:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.188@tcp1 : PUT - for me 00000400:00000200:2.0:1616513071.328503:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.188@tcp1 of length 224 into portal 26 MB=0x6058f2cbf8f80 00000400:00000200:2.0:1616513071.328510:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.188@tcp1 of length 224/224 into md 0xf97c4399 [1] + 0 00000400:00000200:2.0:1616513071.328515:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2908 00000400:00000200:2.0:1616513071.328517:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513071.328520:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513071.328525:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2908 00000800:00000200:2.0:1616513071.328528:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.328531:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.328533:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513071.328539:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513071.328555:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.328566:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513071.328574:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.188@tcp1 : PUT - for me 00000400:00000200:26.0:1616513071.328583:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.188@tcp1 of length 224 into portal 12 MB=0x6058f2cbf9000 00000400:00000200:26.0:1616513071.328590:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.188@tcp1 of length 224/224 into md 0xf96450b9 [1] + 17608 00000100:00000200:23.0:1616513071.328592:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694962349477760 00000400:00000200:26.0:1616513071.328595:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513071.328597:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:23.0:1616513071.328603:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1694962349477760/t0(0) o400->4cd7a0c0-535b-5dbe-9fba-28790020b0b5@10.31.129.188@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513071.328604:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513071.328607:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513071.328609:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.328612:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.328617:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694962349477760, offset 192 00000400:00000200:23.0:1616513071.328623:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.188@tcp1 00000400:00000200:23.0:1616513071.328632:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.188@tcp1 routed destination 00000400:00000200:23.0:1616513071.328647:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.188@tcp1(10.31.129.188@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.328654:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.328660:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.328663:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:18.0:1616513071.328667:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694962349477888 00000800:00000200:23.0:1616513071.328668:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.328673:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.328678:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.328680:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:18.0:1616513071.328680:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c696c00 x1694962349477888/t0(0) o400->e7446ee0-1789-6315-b20c-cca96b84d484@10.31.129.188@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:30.0:1616513071.328687:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.328692:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513071.328694:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:18.0:1616513071.328695:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694962349477888, offset 192 00000800:00000200:2.0:1616513071.328696:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513071.328702:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.188@tcp1 00000400:00000200:30.0:1616513071.328713:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513071.328714:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.188@tcp1 routed destination 00000400:00000200:30.0:1616513071.328716:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3edd0 00000800:00000200:30.0:1616513071.328720:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:18.0:1616513071.328730:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.188@tcp1(10.31.129.188@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.328736:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.328742:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.328745:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.328747:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.328752:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.328799:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.328856:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.328863:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513071.328866:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513071.328872:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513071.328878:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cda90 00000800:00000200:12.0:1616513071.328882:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.328885:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.346300:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0c00] (20)++ 00000800:00000200:28.0:1616513071.346357:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (21)++ 00000800:00000200:28.0:1616513071.346367:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.225@o2ib1 00000400:00000200:28.0:1616513071.346375:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.225@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513071.346383:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.225@o2ib1 of length 440 into portal 12 MB=0x5ff4f8848ad80 00000400:00000200:28.0:1616513071.346389:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.225@o2ib1 of length 440/440 into md 0x865b9f31 [1] + 48472 00000400:00000200:28.0:1616513071.346395:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513071.346397:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513071.346405:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0c00] (22)++ 00000800:00000200:28.0:1616513071.346408:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0c00] (23)-- 00000800:00000200:28.0:1616513071.346410:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (22)-- 00000800:00000200:0.0:1616513071.346413:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (21)-- 00000100:00000200:24.0:1616513071.346470:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688091937516928 00010000:00000200:24.0:1616513071.346562:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ec800 x1688091937516928/t0(0) o41->ea9001c2-9b9d-06ca-5a97-b0b385792512@10.242.46.225@o2ib1:697/0 lens 440/368 e 0 to 0 dl 1616513077 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513071.346577:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1688091937516928, offset 192 00000400:00000200:24.0:1616513071.346585:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.225@o2ib1 00000400:00000200:24.0:1616513071.346592:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.225@o2ib1 local destination 00000400:00000200:24.0:1616513071.346605:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.225@o2ib1(10.242.46.225@o2ib1:10.242.46.225@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.346612:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.225@o2ib1 00000800:00000200:24.0:1616513071.346618:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f00] -> 10.242.46.225@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.346621:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0c00] (20)++ 00000800:00000200:24.0:1616513071.346623:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0c00] (21)++ 00000800:00000200:24.0:1616513071.346628:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0c00] (22)-- 00000800:00000200:11.2:1616513071.346633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0c00] (21)++ 00000800:00000200:12.0:1616513071.346687:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (22)++ 00000800:00000200:12.0:1616513071.346694:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0c00] (23)-- 00000400:00000200:12.0:1616513071.346697:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513071.346704:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.225@o2ib1: PUT: OK 00000400:00000200:12.0:1616513071.346707:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:12.0:1616513071.346711:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (22)-- 00000800:00000200:12.0:1616513071.346714:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (21)-- 00000800:00000200:7.2:1616513071.404436:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513071.404494:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.404503:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513071.404513:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.147@tcp1 : PUT - for me 00000400:00000200:26.0:1616513071.404523:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.147@tcp1 of length 224 into portal 26 MB=0x603fcdb8754c0 00000400:00000200:26.0:1616513071.404530:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.147@tcp1 of length 224/224 into md 0xf97c43b9 [1] + 0 00000400:00000200:26.0:1616513071.404534:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf38f8 00000400:00000200:26.0:1616513071.404537:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513071.404540:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513071.404545:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000800:00000200:26.0:1616513071.404548:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.404548:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513071.404551:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513071.404555:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513071.404558:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.404559:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513071.404567:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.147@tcp1 : PUT - for me 00000400:00000200:14.0:1616513071.404577:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.147@tcp1 of length 224 into portal 12 MB=0x603fcdb875580 00000400:00000200:14.0:1616513071.404583:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.147@tcp1 of length 224/224 into md 0xf96450b9 [1] + 17832 00000400:00000200:14.0:1616513071.404588:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513071.404591:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513071.404598:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.404602:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.404603:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.404614:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234409985216 00010000:00000200:23.0:1616513071.404633:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1693234409985216/t0(0) o400->bb39e619-fcc6-5ad3-9d65-6ec1faa6f1c2@10.31.138.147@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.404648:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234409985216, offset 192 00000400:00000200:23.0:1616513071.404654:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.147@tcp1 00000400:00000200:23.0:1616513071.404663:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.147@tcp1 routed destination 00000100:00000200:18.0:1616513071.404664:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234409985408 00010000:00000200:18.0:1616513071.404677:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa6300 x1693234409985408/t0(0) o400->ed94a6eb-07bc-d767-1fd7-7653ba4c0fca@10.31.138.147@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:23.0:1616513071.404679:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.147@tcp1(10.31.138.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.404686:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:18.0:1616513071.404692:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234409985408, offset 192 00000800:00000200:23.0:1616513071.404693:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.404696:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.404698:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513071.404699:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.147@tcp1 00000800:00000200:23.0:1616513071.404703:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513071.404707:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.147@tcp1 routed destination 00000800:00000200:7.2:1616513071.404708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.404710:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513071.404717:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513071.404722:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.147@tcp1(10.31.138.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513071.404723:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513071.404726:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513071.404727:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513071.404729:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:30.0:1616513071.404731:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513071.404736:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f4b8 00000800:00000200:18.0:1616513071.404736:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.404738:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513071.404740:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513071.404741:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.404745:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.404793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.404848:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.404856:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.404858:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.404865:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.404869:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdcb0 00000800:00000200:28.0:1616513071.404873:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.404876:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.404882:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513071.404885:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.404887:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.404888:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.404890:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.428311:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513071.428367:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.428377:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513071.428386:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.199@tcp1 : PUT - for me 00000400:00000200:16.0:1616513071.428395:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.199@tcp1 of length 224 into portal 26 MB=0x60182d9bbdbc0 00000400:00000200:16.0:1616513071.428402:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.199@tcp1 of length 224/224 into md 0xf97c43d9 [1] + 0 00000400:00000200:16.0:1616513071.428407:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2aa0 00000400:00000200:16.0:1616513071.428409:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513071.428412:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513071.428417:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2aa0 00000800:00000200:16.0:1616513071.428420:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.428421:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513071.428423:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513071.428425:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.428484:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511370607552 00010000:00000200:23.0:1616513071.428496:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1690511370607552/t0(0) o400->03d9fb55-c574-8f08-be7a-df83b0425227@10.31.140.199@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.428511:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511370607552, offset 192 00000400:00000200:23.0:1616513071.428517:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.199@tcp1 00000400:00000200:23.0:1616513071.428525:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.199@tcp1 routed destination 00000400:00000200:23.0:1616513071.428541:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.199@tcp1(10.31.140.199@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.428547:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.428554:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.428557:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.428559:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.428564:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513071.428571:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513071.428571:0:48541:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513071.428606:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513071.428626:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.428634:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.428637:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.428644:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.428650:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fba0 00000800:00000200:2.0:1616513071.428654:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.428657:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513071.428660:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.428669:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.428678:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.199@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.428687:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.199@tcp1 of length 224 into portal 12 MB=0x60182d9bbde00 00000400:00000200:0.0:1616513071.428693:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.199@tcp1 of length 224/224 into md 0x865b9f31 [1] + 48912 00000400:00000200:0.0:1616513071.428698:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.428701:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.428708:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.428711:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.428713:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.428717:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.428773:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511370608128 00010000:00000200:24.0:1616513071.428785:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3600 x1690511370608128/t0(0) o400->98c87c69-437a-d7b0-37e5-006ff1d03d16@10.31.140.199@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.428800:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511370608128, offset 192 00000400:00000200:24.0:1616513071.428806:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.199@tcp1 00000400:00000200:24.0:1616513071.428814:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.199@tcp1 routed destination 00000400:00000200:24.0:1616513071.428829:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.199@tcp1(10.31.140.199@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.428835:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.428841:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.428844:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.428846:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.428851:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.428857:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.428867:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.428873:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513071.428877:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.428881:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.428888:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.428892:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:28.0:1616513071.428896:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.430870:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513071.430925:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.430935:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513071.430937:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513071.430944:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.184@tcp1 : PUT - for me 00000800:00000200:30.0:1616513071.430947:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513071.430955:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.184@tcp1 of length 224 into portal 26 MB=0x60194c7434300 00000400:00000200:30.0:1616513071.430956:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.184@tcp1 : PUT - for me 00000400:00000200:26.0:1616513071.430962:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.184@tcp1 of length 224/224 into md 0xf97c43e9 [1] + 0 00000400:00000200:30.0:1616513071.430966:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.184@tcp1 of length 224 into portal 12 MB=0x60194c7434440 00000400:00000200:26.0:1616513071.430966:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf24c8 00000400:00000200:26.0:1616513071.430969:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513071.430972:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.184@tcp1 of length 224/224 into md 0xf96450b9 [1] + 18056 00000400:00000200:30.0:1616513071.430977:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513071.430977:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513071.430982:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf24c8 00000100:00000200:30.0:1616513071.430986:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513071.430987:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513071.430990:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513071.430991:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513071.430993:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513071.430995:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.430996:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.430998:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.431055:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588370125568 00000100:00000200:18.0:1616513071.431059:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588370125888 00010000:00000200:23.0:1616513071.431066:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1690588370125568/t0(0) o400->eca2550f-268d-62d5-3523-62fcd6e6a292@10.31.132.184@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:18.0:1616513071.431071:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7ba80 x1690588370125888/t0(0) o400->053d684e-eb3a-5478-208b-a5ec58780d17@10.31.132.184@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:23.0:1616513071.431083:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588370125568, offset 192 00000100:00000200:18.0:1616513071.431086:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588370125888, offset 192 00000400:00000200:23.0:1616513071.431089:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.184@tcp1 00000400:00000200:18.0:1616513071.431095:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.184@tcp1 00000400:00000200:23.0:1616513071.431098:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.184@tcp1 routed destination 00000400:00000200:18.0:1616513071.431103:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.184@tcp1 routed destination 00000400:00000200:18.0:1616513071.431120:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.184@tcp1(10.31.132.184@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:23.0:1616513071.431121:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.184@tcp1(10.31.132.184@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.431128:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513071.431128:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.431134:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.431135:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.431137:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513071.431138:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513071.431140:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513071.431141:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513071.431144:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:23.0:1616513071.431146:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.431151:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.431153:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513071.431153:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513071.431204:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.431211:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513071.431214:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513071.431215:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513071.431221:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513071.431227:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eee0 00000800:00000200:14.0:1616513071.431232:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513071.431236:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.431242:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513071.431245:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513071.431251:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513071.431255:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccdd0 00000800:00000200:12.0:1616513071.431258:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.431261:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.435661:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513071.435715:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.435725:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513071.435726:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513071.435734:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.212@tcp1 : PUT - for me 00000400:00000200:26.0:1616513071.435742:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.212@tcp1 of length 224 into portal 26 MB=0x6000e7c35f080 00000400:00000200:26.0:1616513071.435749:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.212@tcp1 of length 224/224 into md 0xf97c43f9 [1] + 0 00000400:00000200:26.0:1616513071.435758:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3188 00000400:00000200:26.0:1616513071.435761:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513071.435764:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513071.435769:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000800:00000200:26.0:1616513071.435771:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.435774:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513071.435776:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.435834:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688912073715840 00010000:00000200:23.0:1616513071.435845:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688912073715840/t0(0) o400->5c997e49-4f83-bfcc-c22e-8a6e42e5416e@10.31.136.212@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.435859:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688912073715840, offset 192 00000400:00000200:23.0:1616513071.435867:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.212@tcp1 00000400:00000200:23.0:1616513071.435875:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.212@tcp1 routed destination 00000400:00000200:23.0:1616513071.435890:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.212@tcp1(10.31.136.212@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.435897:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.435903:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.435906:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.435909:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.435914:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.435918:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.435920:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513071.435972:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.435979:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.435983:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.435989:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.435995:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e5d8 00000800:00000200:2.0:1616513071.435999:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.436002:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513071.436021:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513071.436078:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.436087:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513071.436096:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.212@tcp1 : PUT - for me 00000400:00000200:16.0:1616513071.436105:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.212@tcp1 of length 224 into portal 12 MB=0x6000e7c35f100 00000400:00000200:16.0:1616513071.436111:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.212@tcp1 of length 224/224 into md 0x865b9f31 [1] + 49136 00000400:00000200:16.0:1616513071.436116:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513071.436118:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513071.436125:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.436128:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513071.436130:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.436132:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.436189:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688912073715968 00010000:00000200:24.0:1616513071.436201:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e5100 x1688912073715968/t0(0) o400->d04eb00b-774f-61ec-949e-e75a93d78b67@10.31.136.212@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.436215:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688912073715968, offset 192 00000400:00000200:24.0:1616513071.436222:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.212@tcp1 00000400:00000200:24.0:1616513071.436229:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.212@tcp1 routed destination 00000400:00000200:24.0:1616513071.436244:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.212@tcp1(10.31.136.212@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.436251:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.436256:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.436261:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.436263:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.436268:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.436273:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.436326:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.436333:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513071.436335:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513071.436342:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513071.436346:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2b28 00000800:00000200:12.0:1616513071.436350:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.436353:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.442496:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513071.442553:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.442562:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513071.442571:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.160@tcp1 : PUT - for me 00000400:00000200:16.0:1616513071.442580:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.160@tcp1 of length 224 into portal 26 MB=0x5ffd51cc023c0 00000400:00000200:16.0:1616513071.442587:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.160@tcp1 of length 224/224 into md 0xf97c4419 [1] + 0 00000400:00000200:16.0:1616513071.442594:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2198 00000400:00000200:16.0:1616513071.442596:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513071.442599:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513071.442604:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2198 00000800:00000200:28.0:1616513071.442607:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513071.442607:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.442610:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513071.442611:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.442669:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665659024320 00010000:00000200:23.0:1616513071.442681:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688665659024320/t0(0) o400->957105a4-c94b-06fd-6c43-b840e72adc42@10.31.132.160@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.442695:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665659024320, offset 192 00000400:00000200:23.0:1616513071.442701:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.160@tcp1 00000400:00000200:23.0:1616513071.442709:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.160@tcp1 routed destination 00000400:00000200:23.0:1616513071.442727:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.160@tcp1(10.31.132.160@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.442734:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.442740:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.442743:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.442745:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.442750:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513071.442757:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513071.442757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.442767:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.442772:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513071.442774:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513071.442780:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513071.442785:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ff58 00000800:00000200:30.0:1616513071.442789:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.442791:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513071.443227:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513071.443283:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.443292:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513071.443299:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513071.443301:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.160@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.443310:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.160@tcp1 of length 224 into portal 12 MB=0x5ffd51cc025c0 00000400:00000200:12.0:1616513071.443316:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.160@tcp1 of length 224/224 into md 0x865b9f31 [1] + 49360 00000400:00000200:12.0:1616513071.443322:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.443324:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513071.443331:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.443334:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.443338:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.443399:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665659024832 00010000:00000200:24.0:1616513071.443411:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa40d80 x1688665659024832/t0(0) o400->8009775a-f2f8-f5fb-c096-70a16ab8a8e6@10.31.132.160@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.443424:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665659024832, offset 192 00000400:00000200:24.0:1616513071.443430:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.160@tcp1 00000400:00000200:24.0:1616513071.443447:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.160@tcp1 routed destination 00000400:00000200:24.0:1616513071.443461:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.160@tcp1(10.31.132.160@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.443467:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.443473:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.443475:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.443477:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.443482:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.443487:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.443498:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.443504:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.443506:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513071.443508:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513071.443513:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.443517:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3298 00000800:00000200:28.0:1616513071.443521:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.531779:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513071.531836:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.531847:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.531856:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.116@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.531867:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.116@tcp1 of length 224 into portal 26 MB=0x60215e01e1e80 00000400:00000200:0.0:1616513071.531874:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.116@tcp1 of length 224/224 into md 0xf97c4449 [1] + 0 00000400:00000200:0.0:1616513071.531879:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3a90 00000400:00000200:0.0:1616513071.531882:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.531885:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513071.531891:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513071.531891:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:0.0:1616513071.531895:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.531897:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513071.531899:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.531902:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.531903:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.531912:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.116@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.531922:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.116@tcp1 of length 224 into portal 12 MB=0x60215e01e1f00 00000400:00000200:12.0:1616513071.531928:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.116@tcp1 of length 224/224 into md 0x865b9f31 [1] + 49584 00000400:00000200:12.0:1616513071.531934:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.531937:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513071.531944:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.531948:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.531950:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.531958:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691142837902976 00010000:00000200:23.0:1616513071.531969:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1691142837902976/t0(0) o400->baac9af8-c008-178e-e088-580f63ec7aac@10.31.141.116@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.531983:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691142837902976, offset 192 00000400:00000200:23.0:1616513071.531989:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.116@tcp1 00000400:00000200:23.0:1616513071.531996:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.116@tcp1 routed destination 00000400:00000200:23.0:1616513071.532011:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.116@tcp1(10.31.141.116@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513071.532014:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691142837903104 00000800:00000200:23.0:1616513071.532017:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.532024:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.532026:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.532028:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:24.0:1616513071.532029:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d846aa42400 x1691142837903104/t0(0) o400->4fbca977-13bd-5b78-8284-76cf7a7ac775@10.31.141.116@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513071.532034:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513071.532041:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513071.532041:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513071.532045:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691142837903104, offset 192 00000400:00000200:24.0:1616513071.532051:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.116@tcp1 00000400:00000200:24.0:1616513071.532059:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.116@tcp1 routed destination 00000400:00000200:24.0:1616513071.532075:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.116@tcp1(10.31.141.116@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.532082:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.532089:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.532092:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.532094:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513071.532096:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513071.532099:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.532103:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513071.532105:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.532108:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.532115:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.532121:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e550 00000800:00000200:2.0:1616513071.532131:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.532134:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513071.532173:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.532181:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.532184:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513071.532185:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513071.532191:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.532195:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000800:00000200:28.0:1616513071.532198:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513071.562570:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924032, portal 4 00000100:00000200:10.0:1616513071.562576:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924032, offset 0 00000400:00000200:10.0:1616513071.562581:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:10.0:1616513071.562587:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:10.0:1616513071.562597:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513071.562603:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:10.0:1616513071.562611:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513071.562613:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:10.0:1616513071.562615:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:10.0:1616513071.562620:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513071.562667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513071.562687:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513071.562693:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513071.562696:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513071.562703:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513071.562709:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d1680 x1693067725924032/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513071.562721:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82880 00000800:00000200:27.0:1616513071.562726:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513071.562729:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513071.562797:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513071.562807:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513071.562813:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513071.562817:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513071.562822:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513071.562830:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6266c0 00000400:00000200:27.0:1616513071.562836:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716518d [1] + 192 00000400:00000200:27.0:1616513071.562843:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513071.562848:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d1680 x1693067725924032/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513071.562862:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513071.562865:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513071.562867:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:10.0:1616513071.562872:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d1680 x1693067725924032/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513071.562883:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d1680 x1693067725924032/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513078 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513071.562891:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82550 00000800:00000200:11.2:1616513071.598365:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513071.598423:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.598451:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513071.598461:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.31@tcp1 : PUT - for me 00000400:00000200:16.0:1616513071.598471:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.31@tcp1 of length 224 into portal 26 MB=0x5ffea2e369580 00000400:00000200:16.0:1616513071.598478:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.31@tcp1 of length 224/224 into md 0xf97c4459 [1] + 0 00000800:00000200:12.0:1616513071.598478:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513071.598483:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0bb0 00000400:00000200:16.0:1616513071.598486:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513071.598489:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513071.598494:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0bb0 00000800:00000200:16.0:1616513071.598498:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.598501:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513071.598503:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.598562:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688756146312576 00010000:00000200:23.0:1616513071.598574:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688756146312576/t0(0) o400->8d94cfda-efb8-4622-bc9f-cb499f0a0bb0@10.31.142.31@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.598589:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688756146312576, offset 192 00000400:00000200:23.0:1616513071.598596:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.31@tcp1 00000400:00000200:23.0:1616513071.598604:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.31@tcp1 routed destination 00000400:00000200:23.0:1616513071.598620:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.31@tcp1(10.31.142.31@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.598627:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.598634:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.598639:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.598641:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.598647:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.598653:0:48558:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.598654:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513071.598667:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.598675:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513071.598679:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513071.598685:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513071.598691:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e2a8 00000800:00000200:30.0:1616513071.598695:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.598698:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513071.598711:0:48558:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513071.598723:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.598735:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513071.598738:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513071.598741:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513071.598742:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513071.598745:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513071.599056:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513071.599112:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.599122:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.599130:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.31@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.599140:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.31@tcp1 of length 224 into portal 12 MB=0x5ffea2e36a140 00000400:00000200:0.0:1616513071.599146:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.31@tcp1 of length 224/224 into md 0x865b9f31 [1] + 49808 00000400:00000200:0.0:1616513071.599153:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.599156:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.599163:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.599166:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.599167:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.599171:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.599249:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688756146315584 00010000:00000200:24.0:1616513071.599264:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d4380 x1688756146315584/t0(0) o400->a7c56294-829d-329a-d2f3-8233c7acaff2@10.31.142.31@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.599290:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688756146315584, offset 192 00000400:00000200:24.0:1616513071.599297:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.31@tcp1 00000400:00000200:24.0:1616513071.599304:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.31@tcp1 routed destination 00000400:00000200:24.0:1616513071.599320:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.31@tcp1(10.31.142.31@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.599327:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.599333:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.599336:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.599338:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.599343:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.599349:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.599359:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.599366:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513071.599368:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513071.599370:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513071.599375:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513071.599379:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2000 00000800:00000200:12.0:1616513071.599383:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.602623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513071.602683:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.602692:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513071.602701:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.53@tcp1 : PUT - for me 00000400:00000200:28.0:1616513071.602710:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.53@tcp1 of length 224 into portal 26 MB=0x6052127b4f500 00000400:00000200:28.0:1616513071.602717:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.53@tcp1 of length 224/224 into md 0xf97c44a9 [1] + 0 00000400:00000200:28.0:1616513071.602722:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1dc0 00000400:00000200:28.0:1616513071.602724:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513071.602727:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513071.602732:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1dc0 00000800:00000200:28.0:1616513071.602735:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.602738:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.602739:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513071.602746:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.602749:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.602750:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.602758:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.53@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.602767:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.53@tcp1 of length 224 into portal 12 MB=0x6052127b4f580 00000400:00000200:0.0:1616513071.602773:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.53@tcp1 of length 224/224 into md 0x865b9f31 [1] + 50032 00000400:00000200:0.0:1616513071.602778:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.602781:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.602788:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.602791:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.602793:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.602798:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694489818494208 00010000:00000200:23.0:1616513071.602809:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1694489818494208/t0(0) o400->5ce708f6-d1e2-2639-07a5-23c884be4152@10.31.141.53@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.602824:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694489818494208, offset 192 00000400:00000200:23.0:1616513071.602830:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.53@tcp1 00000400:00000200:23.0:1616513071.602838:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.53@tcp1 routed destination 00000100:00000200:24.0:1616513071.602851:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694489818494336 00000400:00000200:23.0:1616513071.602853:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.53@tcp1(10.31.141.53@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.602860:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:24.0:1616513071.602864:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5100 x1694489818494336/t0(0) o400->81bcc19b-a047-67b1-e23d-90d37a9a289c@10.31.141.53@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513071.602866:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.602869:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.602872:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.602876:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513071.602878:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694489818494336, offset 192 00000100:00000200:23.0:1616513071.602883:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513071.602884:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.53@tcp1 00000800:00000200:7.2:1616513071.602884:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513071.602892:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.53@tcp1 routed destination 00000800:00000200:30.0:1616513071.602897:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.602902:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513071.602906:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513071.602907:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.53@tcp1(10.31.141.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513071.602911:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:26.0:1616513071.602911:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513071.602913:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:30.0:1616513071.602915:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fdc0 00000800:00000200:30.0:1616513071.602918:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513071.602919:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.602922:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.602924:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.602929:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.602934:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.602987:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.602993:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513071.602996:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513071.602997:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513071.603002:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513071.603005:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf27f8 00000800:00000200:12.0:1616513071.603009:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.614075:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.614131:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.614140:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513071.614144:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513071.614149:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.57@tcp1 : PUT - for me 00000400:00000200:2.0:1616513071.614160:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.57@tcp1 of length 224 into portal 26 MB=0x6018358a702c0 00000800:00000200:11.2:1616513071.614162:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513071.614167:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.57@tcp1 of length 224/224 into md 0xf97c44d9 [1] + 0 00000400:00000200:2.0:1616513071.614171:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1ed0 00000400:00000200:2.0:1616513071.614174:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513071.614177:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513071.614181:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1ed0 00000800:00000200:2.0:1616513071.614185:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.614187:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.614189:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513071.614217:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.614227:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513071.614236:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.57@tcp1 : PUT - for me 00000400:00000200:16.0:1616513071.614245:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.57@tcp1 of length 224 into portal 12 MB=0x6018358a703c0 00000100:00000200:23.0:1616513071.614248:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513499947712 00000400:00000200:16.0:1616513071.614252:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.57@tcp1 of length 224/224 into md 0x865b9f31 [1] + 50256 00000400:00000200:16.0:1616513071.614257:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:23.0:1616513071.614259:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1690513499947712/t0(0) o400->a0f67f03-2543-fbb7-db4a-4dfa67d9ef68@10.31.131.57@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513071.614259:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513071.614266:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.614269:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.614271:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:23.0:1616513071.614273:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513499947712, offset 192 00000800:00000200:16.0:1616513071.614275:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:23.0:1616513071.614281:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.57@tcp1 00000400:00000200:23.0:1616513071.614290:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.57@tcp1 routed destination 00000400:00000200:23.0:1616513071.614305:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.57@tcp1(10.31.131.57@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.614312:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.614318:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.614321:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.614324:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.614329:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513071.614332:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513499947968 00000800:00000200:7.2:1616513071.614333:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.614336:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:24.0:1616513071.614345:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e3f00 x1690513499947968/t0(0) o400->d28db30c-f3a8-d209-8417-b804f04f15bc@10.31.131.57@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.614359:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513499947968, offset 192 00000400:00000200:24.0:1616513071.614366:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.57@tcp1 00000400:00000200:24.0:1616513071.614374:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.57@tcp1 routed destination 00000400:00000200:24.0:1616513071.614389:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.57@tcp1(10.31.131.57@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.614397:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513071.614399:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513071.614403:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.614406:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513071.614407:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513071.614408:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513071.614409:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513071.614411:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513071.614413:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513071.614417:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513071.614418:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513071.614423:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eaa0 00000800:00000200:26.0:1616513071.614427:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513071.614472:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.614481:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.614483:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.614490:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.614494:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:28.0:1616513071.614498:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.614501:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.712093:0:48570:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513071.712150:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.712160:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513071.712170:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.181@tcp1 : PUT - for me 00000400:00000200:14.0:1616513071.712179:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.181@tcp1 of length 224 into portal 26 MB=0x5ffde71aa2340 00000400:00000200:14.0:1616513071.712186:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.181@tcp1 of length 224/224 into md 0xf97c44f9 [1] + 0 00000400:00000200:14.0:1616513071.712191:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d17e8 00000400:00000200:14.0:1616513071.712194:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513071.712197:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513071.712202:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d17e8 00000800:00000200:2.0:1616513071.712204:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.712206:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.712211:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.712213:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.712270:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705738351424 00010000:00000200:23.0:1616513071.712281:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688705738351424/t0(0) o400->69e0a1be-289f-96f3-09a5-2904c5fb2e44@10.31.132.181@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.712296:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705738351424, offset 192 00000400:00000200:23.0:1616513071.712301:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.181@tcp1 00000400:00000200:23.0:1616513071.712309:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.181@tcp1 routed destination 00000400:00000200:23.0:1616513071.712325:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.181@tcp1(10.31.132.181@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.712331:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.712337:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.712340:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.712342:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.712348:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.712353:0:48570:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.712355:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513071.712364:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.712371:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513071.712374:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513071.712381:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513071.712387:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e4c8 00000800:00000200:30.0:1616513071.712391:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.712394:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513071.712684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513071.712741:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.712751:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.712760:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.181@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.712769:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.181@tcp1 of length 224 into portal 12 MB=0x5ffde71aa2440 00000400:00000200:0.0:1616513071.712775:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.181@tcp1 of length 224/224 into md 0x865b9f31 [1] + 50480 00000400:00000200:0.0:1616513071.712780:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.712783:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.712790:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.712793:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.712795:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513071.712798:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.712858:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705738351680 00010000:00000200:24.0:1616513071.712872:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d7080 x1688705738351680/t0(0) o400->7cb89c0f-1f69-a03e-13ee-d2da71f4f158@10.31.132.181@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.712887:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705738351680, offset 192 00000400:00000200:24.0:1616513071.712893:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.181@tcp1 00000400:00000200:24.0:1616513071.712901:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.181@tcp1 routed destination 00000400:00000200:24.0:1616513071.712916:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.181@tcp1(10.31.132.181@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.712923:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.712929:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.712932:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.712934:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.712939:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.712944:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.712998:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.713006:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.713008:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.713015:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.713019:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2d48 00000800:00000200:28.0:1616513071.713023:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.713026:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.747980:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.748037:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.748047:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513071.748051:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513071.748056:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.69@tcp1 : PUT - for me 00000800:00000200:14.0:1616513071.748061:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513071.748063:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.748066:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.69@tcp1 of length 224 into portal 26 MB=0x603ff8ee4ef00 00000400:00000200:14.0:1616513071.748070:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.69@tcp1 : PUT - for me 00000400:00000200:2.0:1616513071.748073:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.69@tcp1 of length 224/224 into md 0xf97c4519 [1] + 0 00000400:00000200:2.0:1616513071.748077:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1540 00000400:00000200:14.0:1616513071.748079:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.69@tcp1 of length 224 into portal 12 MB=0x603ff8ee4efc0 00000400:00000200:2.0:1616513071.748080:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513071.748083:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513071.748088:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1540 00000800:00000200:2.0:1616513071.748091:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513071.748093:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.69@tcp1 of length 224/224 into md 0xf96450b9 [1] + 18280 00000800:00000200:2.0:1616513071.748094:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513071.748096:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513071.748098:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513071.748101:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513071.748108:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.748111:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.748113:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.748156:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693246009175808 00010000:00000200:23.0:1616513071.748167:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1693246009175808/t0(0) o400->08738ba8-00d3-350d-16db-1f3f3badb4a6@10.31.132.69@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513071.748176:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693246009176000 00000100:00000200:23.0:1616513071.748182:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693246009175808, offset 192 00000400:00000200:23.0:1616513071.748189:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.69@tcp1 00010000:00000200:18.0:1616513071.748190:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa2880 x1693246009176000/t0(0) o400->dd9d825e-1cd0-85db-b258-396400dceb6c@10.31.132.69@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:23.0:1616513071.748197:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.69@tcp1 routed destination 00000100:00000200:18.0:1616513071.748205:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693246009176000, offset 192 00000400:00000200:18.0:1616513071.748212:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.69@tcp1 00000400:00000200:23.0:1616513071.748214:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.69@tcp1(10.31.132.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.748221:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513071.748223:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.69@tcp1 routed destination 00000800:00000200:23.0:1616513071.748227:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.748230:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.748233:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.748238:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513071.748239:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.69@tcp1(10.31.132.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513071.748244:0:48574:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.748245:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513071.748246:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.748253:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.748256:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.748258:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.748263:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513071.748297:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513071.748305:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.748306:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513071.748308:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513071.748310:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513071.748315:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513071.748321:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eb28 00000800:00000200:26.0:1616513071.748324:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513071.748365:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.748373:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513071.748376:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513071.748382:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513071.748386:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccbb0 00000800:00000200:0.0:1616513071.748391:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.748395:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.752270:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.752284:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.752294:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513071.752297:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513071.752303:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.97@tcp1 : PUT - for me 00000400:00000200:2.0:1616513071.752312:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.97@tcp1 of length 224 into portal 26 MB=0x60467b95d77c0 00000400:00000200:2.0:1616513071.752319:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.97@tcp1 of length 224/224 into md 0xf97c4549 [1] + 0 00000400:00000200:2.0:1616513071.752324:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1b18 00000400:00000200:2.0:1616513071.752327:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513071.752332:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513071.752337:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1b18 00000800:00000200:2.0:1616513071.752340:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.752342:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.752344:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.752402:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693693398316992 00010000:00000200:23.0:1616513071.752413:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1693693398316992/t0(0) o400->734d68a0-8a91-6824-76ce-168803b87303@10.31.128.97@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.752428:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693693398316992, offset 192 00000400:00000200:23.0:1616513071.752469:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.97@tcp1 00000400:00000200:23.0:1616513071.752478:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.97@tcp1 routed destination 00000400:00000200:23.0:1616513071.752494:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.97@tcp1(10.31.128.97@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.752500:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.752507:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.752510:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.752512:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.752517:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.752522:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.752524:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513071.752531:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.752536:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513071.752539:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513071.752546:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513071.752550:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f430 00000800:00000200:30.0:1616513071.752554:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.752556:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513071.752610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513071.752621:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.752628:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513071.752630:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513071.752637:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.97@tcp1 : PUT - for me 00000400:00000200:14.0:1616513071.752647:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.97@tcp1 of length 224 into portal 12 MB=0x60467b95d7c40 00000400:00000200:14.0:1616513071.752653:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.97@tcp1 of length 224/224 into md 0xf96450b9 [1] + 18504 00000400:00000200:14.0:1616513071.752659:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513071.752662:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513071.752668:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.752671:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.752673:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513071.752733:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693693398318144 00010000:00000200:18.0:1616513071.752745:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa4c80 x1693693398318144/t0(0) o400->a338b4d5-df5e-cc81-e859-ab382a1dd185@10.31.128.97@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513071.752759:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693693398318144, offset 192 00000400:00000200:18.0:1616513071.752765:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.97@tcp1 00000400:00000200:18.0:1616513071.752773:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.97@tcp1 routed destination 00000400:00000200:18.0:1616513071.752788:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.97@tcp1(10.31.128.97@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.752795:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.752801:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.752806:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.752808:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.752813:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.752858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.752916:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.752923:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.752926:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.752932:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.752936:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc5d8 00000800:00000200:28.0:1616513071.752940:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.752943:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.810857:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513071.810915:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.810925:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.810934:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.81@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.810943:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.81@tcp1 of length 224 into portal 26 MB=0x5f90c0df91240 00000400:00000200:0.0:1616513071.810950:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.81@tcp1 of length 224/224 into md 0xf97c4559 [1] + 0 00000400:00000200:0.0:1616513071.810955:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1a08 00000400:00000200:0.0:1616513071.810958:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.810961:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513071.810966:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1a08 00000800:00000200:0.0:1616513071.810969:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.810970:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.810972:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.810980:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.811032:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681205052904000 00010000:00000200:23.0:1616513071.811044:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1681205052904000/t0(0) o400->68bfc844-9c81-d401-2247-290519a043d3@10.31.131.81@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.811058:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681205052904000, offset 192 00000400:00000200:23.0:1616513071.811064:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.81@tcp1 00000400:00000200:23.0:1616513071.811073:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.81@tcp1 routed destination 00000800:00000200:11.2:1616513071.811076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:23.0:1616513071.811088:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.81@tcp1(10.31.131.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.811095:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.811101:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.811104:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.811106:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.811112:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513071.811119:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513071.811120:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.811133:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.811140:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513071.811140:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513071.811143:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513071.811149:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513071.811150:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:30.0:1616513071.811155:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e000 00000400:00000200:28.0:1616513071.811158:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.81@tcp1 : PUT - for me 00000800:00000200:30.0:1616513071.811159:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.811162:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513071.811167:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.81@tcp1 of length 224 into portal 12 MB=0x5f90c0df912c0 00000400:00000200:28.0:1616513071.811173:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.81@tcp1 of length 224/224 into md 0x865b9f31 [1] + 50704 00000400:00000200:28.0:1616513071.811179:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513071.811181:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513071.811188:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.811191:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.811193:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.811196:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.811259:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681205052904128 00010000:00000200:24.0:1616513071.811272:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0e8480 x1681205052904128/t0(0) o400->98d41b88-0bd4-35a4-f097-c35b19675f59@10.31.131.81@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.811286:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681205052904128, offset 192 00000400:00000200:24.0:1616513071.811293:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.81@tcp1 00000400:00000200:24.0:1616513071.811300:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.81@tcp1 routed destination 00000400:00000200:24.0:1616513071.811316:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.81@tcp1(10.31.131.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.811322:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.811329:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.811331:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.811334:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.811339:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.811344:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.811355:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.811360:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513071.811363:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513071.811365:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513071.811370:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513071.811374:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:16.0:1616513071.811377:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.822789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.822844:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.822853:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513071.822862:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.74@tcp1 : PUT - for me 00000400:00000200:2.0:1616513071.822872:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.74@tcp1 of length 224 into portal 26 MB=0x5f90a4473fe00 00000400:00000200:2.0:1616513071.822878:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.74@tcp1 of length 224/224 into md 0xf97c4579 [1] + 0 00000400:00000200:2.0:1616513071.822883:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0dd0 00000400:00000200:2.0:1616513071.822886:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513071.822889:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513071.822893:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0dd0 00000800:00000200:2.0:1616513071.822896:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.822898:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.822899:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513071.822902:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513071.822953:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:23.0:1616513071.822959:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681197376994816 00000800:00000200:30.0:1616513071.822964:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:23.0:1616513071.822970:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1681197376994816/t0(0) o400->c0d3215a-467f-04d9-083b-76e8d6b134b6@10.31.131.74@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:30.0:1616513071.822972:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513071.822980:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.74@tcp1 : PUT - for me 00000100:00000200:23.0:1616513071.822985:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681197376994816, offset 192 00000400:00000200:30.0:1616513071.822990:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.74@tcp1 of length 224 into portal 12 MB=0x5f90a4473fe80 00000400:00000200:23.0:1616513071.822992:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.74@tcp1 00000400:00000200:30.0:1616513071.822996:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.74@tcp1 of length 224/224 into md 0xf96450b9 [1] + 18728 00000400:00000200:23.0:1616513071.823000:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.74@tcp1 routed destination 00000400:00000200:30.0:1616513071.823002:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.823005:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:23.0:1616513071.823016:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.74@tcp1(10.31.131.74@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513071.823017:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.823021:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:23.0:1616513071.823022:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513071.823024:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.823026:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:23.0:1616513071.823031:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.823034:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.823036:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.823041:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.823045:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.823048:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513071.823054:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.823060:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513071.823063:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513071.823064:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513071.823070:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513071.823076:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ec38 00000100:00000200:18.0:1616513071.823078:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681197376994944 00000800:00000200:14.0:1616513071.823080:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:18.0:1616513071.823091:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a79680 x1681197376994944/t0(0) o400->bbc9e879-be0d-ef64-f453-0b6d247be607@10.31.131.74@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513071.823106:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681197376994944, offset 192 00000400:00000200:18.0:1616513071.823112:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.74@tcp1 00000400:00000200:18.0:1616513071.823120:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.74@tcp1 routed destination 00000400:00000200:18.0:1616513071.823135:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.74@tcp1(10.31.131.74@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.823142:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.823148:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.823151:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.823153:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.823158:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.823166:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.823221:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.823228:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.823231:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.823240:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.823244:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cca18 00000800:00000200:28.0:1616513071.823248:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.823251:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.828379:0:48582:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513071.828449:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.828458:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513071.828459:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513071.828467:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.37@tcp1 : PUT - for me 00000400:00000200:26.0:1616513071.828477:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.37@tcp1 of length 224 into portal 26 MB=0x603cf018b22c0 00000400:00000200:26.0:1616513071.828484:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.37@tcp1 of length 224/224 into md 0xf97c4589 [1] + 0 00000400:00000200:26.0:1616513071.828488:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0908 00000400:00000200:26.0:1616513071.828491:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513071.828494:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513071.828499:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0908 00000800:00000200:26.0:1616513071.828502:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.828505:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513071.828506:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.828565:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693037479273152 00010000:00000200:23.0:1616513071.828575:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1693037479273152/t0(0) o400->8cb93be6-29a8-6abf-a359-33552ee1be3c@10.31.132.37@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.828588:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693037479273152, offset 192 00000400:00000200:23.0:1616513071.828594:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.37@tcp1 00000400:00000200:23.0:1616513071.828602:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.37@tcp1 routed destination 00000400:00000200:23.0:1616513071.828617:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.37@tcp1(10.31.132.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.828623:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.828629:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.828632:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.828634:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.828638:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.828642:0:48582:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.828645:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513071.828695:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513071.828708:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.828716:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.828719:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.828726:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.828732:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ef68 00000800:00000200:2.0:1616513071.828736:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.828739:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513071.828751:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.828761:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.828770:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.37@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.828779:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.37@tcp1 of length 224 into portal 12 MB=0x603cf018b2340 00000400:00000200:0.0:1616513071.828786:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.37@tcp1 of length 224/224 into md 0x865b9f31 [1] + 50928 00000400:00000200:0.0:1616513071.828791:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.828794:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.828801:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.828804:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513071.828806:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.828810:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.828881:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693037479273280 00010000:00000200:24.0:1616513071.828894:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d1f80 x1693037479273280/t0(0) o400->cfd407a6-d984-94ab-f12d-68042dfc55ed@10.31.132.37@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.828909:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693037479273280, offset 192 00000400:00000200:24.0:1616513071.828915:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.37@tcp1 00000400:00000200:24.0:1616513071.828923:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.37@tcp1 routed destination 00000400:00000200:24.0:1616513071.828939:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.37@tcp1(10.31.132.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.828946:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.828952:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.828955:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.828957:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.828962:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.828967:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.829020:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.829027:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.829030:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.829036:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.829040:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3870 00000800:00000200:28.0:1616513071.829045:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.829047:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.840863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513071.840918:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.840927:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513071.840936:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.218@tcp1 : PUT - for me 00000400:00000200:16.0:1616513071.840946:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.218@tcp1 of length 224 into portal 26 MB=0x5ffd4f5acb100 00000400:00000200:16.0:1616513071.840953:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.218@tcp1 of length 224/224 into md 0xf97c45a9 [1] + 0 00000400:00000200:16.0:1616513071.840958:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0550 00000400:00000200:16.0:1616513071.840960:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513071.840963:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513071.840968:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0550 00000800:00000200:16.0:1616513071.840971:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.840974:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513071.840976:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.840978:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.841031:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665003438336 00010000:00000200:23.0:1616513071.841042:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688665003438336/t0(0) o400->f8900430-2cff-33a4-3f33-7ed5ba0d941b@10.31.132.218@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.841056:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665003438336, offset 192 00000400:00000200:23.0:1616513071.841062:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.218@tcp1 00000400:00000200:23.0:1616513071.841070:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.218@tcp1 routed destination 00000400:00000200:23.0:1616513071.841085:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.218@tcp1(10.31.132.218@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.841091:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.841097:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.841100:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.841102:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.841106:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.841112:0:48583:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.841114:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513071.841125:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.841129:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513071.841130:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:30.0:1616513071.841132:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513071.841137:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513071.841141:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f298 00000800:00000200:30.0:1616513071.841145:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.841147:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513071.841199:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.841209:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513071.841217:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.218@tcp1 : PUT - for me 00000400:00000200:28.0:1616513071.841227:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.218@tcp1 of length 224 into portal 12 MB=0x5ffd4f5acb200 00000400:00000200:28.0:1616513071.841233:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.218@tcp1 of length 224/224 into md 0x865b9f31 [1] + 51152 00000400:00000200:28.0:1616513071.841238:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513071.841240:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513071.841247:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.841250:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.841252:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.841255:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.841310:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665003438592 00010000:00000200:24.0:1616513071.841322:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ea880 x1688665003438592/t0(0) o400->332ada9b-a836-e098-bfa4-0347399b57ca@10.31.132.218@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.841335:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665003438592, offset 192 00000400:00000200:24.0:1616513071.841341:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.218@tcp1 00000400:00000200:24.0:1616513071.841349:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.218@tcp1 routed destination 00000400:00000200:24.0:1616513071.841363:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.218@tcp1(10.31.132.218@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.841369:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.841374:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.841377:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.841379:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.841384:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.841389:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.841399:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.841403:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513071.841405:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513071.841407:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513071.841411:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513071.841414:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2550 00000800:00000200:16.0:1616513071.841417:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.852459:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.852520:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.852529:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513071.852532:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513071.852537:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.75@tcp1 : PUT - for me 00000400:00000200:2.0:1616513071.852546:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.75@tcp1 of length 224 into portal 26 MB=0x5ffd67b2db640 00000400:00000200:2.0:1616513071.852552:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.75@tcp1 of length 224/224 into md 0xf97c45c9 [1] + 0 00000400:00000200:2.0:1616513071.852556:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0aa0 00000400:00000200:2.0:1616513071.852558:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513071.852563:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513071.852568:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0aa0 00000800:00000200:2.0:1616513071.852571:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.852573:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.852575:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.852633:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688671538230848 00010000:00000200:23.0:1616513071.852644:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688671538230848/t0(0) o400->6bf922de-1c55-1ad2-2a2c-512c3c500f65@10.31.129.75@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.852659:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688671538230848, offset 192 00000400:00000200:23.0:1616513071.852666:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.75@tcp1 00000400:00000200:23.0:1616513071.852674:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.75@tcp1 routed destination 00000800:00000200:7.2:1616513071.852678:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:23.0:1616513071.852690:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.75@tcp1(10.31.129.75@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.852696:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513071.852700:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.852703:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.852706:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.852708:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:23.0:1616513071.852718:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.852723:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:23.0:1616513071.852724:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:30.0:1616513071.852730:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.75@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.852738:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.75@tcp1 of length 224 into portal 12 MB=0x5ffd67b2dba00 00000400:00000200:30.0:1616513071.852743:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.75@tcp1 of length 224/224 into md 0xf96450b9 [1] + 18952 00000400:00000200:30.0:1616513071.852748:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.852750:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513071.852756:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513071.852758:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513071.852760:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.852761:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.852765:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513071.852767:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513071.852769:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513071.852772:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513071.852775:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f650 00000800:00000200:30.0:1616513071.852778:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513071.852823:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688671538231808 00010000:00000200:18.0:1616513071.852836:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a78d80 x1688671538231808/t0(0) o400->bd99cbf3-cf4c-3498-7b52-195324dc6469@10.31.129.75@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513071.852850:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688671538231808, offset 192 00000400:00000200:18.0:1616513071.852857:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.75@tcp1 00000400:00000200:18.0:1616513071.852864:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.75@tcp1 routed destination 00000400:00000200:18.0:1616513071.852880:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.75@tcp1(10.31.129.75@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.852887:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.852893:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.852896:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.852898:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.852903:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.852909:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.852962:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.852968:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513071.852970:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513071.852976:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513071.852980:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdba0 00000800:00000200:12.0:1616513071.852984:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.852987:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.855224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513071.855281:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.855291:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.855300:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.83@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.855309:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.83@tcp1 of length 224 into portal 26 MB=0x604a3111efd00 00000400:00000200:0.0:1616513071.855316:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.83@tcp1 of length 224/224 into md 0xf97c45e9 [1] + 0 00000400:00000200:0.0:1616513071.855321:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0d48 00000400:00000200:0.0:1616513071.855325:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.855328:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513071.855333:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0d48 00000800:00000200:16.0:1616513071.855336:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.855336:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.855339:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.855341:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.855381:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:23.0:1616513071.855398:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693948273687808 00010000:00000200:23.0:1616513071.855409:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1693948273687808/t0(0) o400->fd25278e-3676-4d3a-0391-a7fd88301aed@10.31.143.83@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.855423:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693948273687808, offset 192 00000400:00000200:23.0:1616513071.855430:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.83@tcp1 00000800:00000200:12.0:1616513071.855433:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:23.0:1616513071.855448:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.83@tcp1 routed destination 00000800:00000200:12.0:1616513071.855455:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.855464:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.83@tcp1 : PUT - for me 00000400:00000200:23.0:1616513071.855465:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.83@tcp1(10.31.143.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.855471:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513071.855476:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.83@tcp1 of length 224 into portal 12 MB=0x604a3111efe40 00000800:00000200:23.0:1616513071.855479:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.855482:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513071.855482:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.83@tcp1 of length 224/224 into md 0x865b9f31 [1] + 51376 00000800:00000200:23.0:1616513071.855484:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513071.855486:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513071.855488:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:23.0:1616513071.855489:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513071.855490:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513071.855494:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.855496:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513071.855497:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.855501:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.855502:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.855561:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693948273688128 00000800:00000200:2.0:1616513071.855564:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.855571:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:24.0:1616513071.855574:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc579680 x1693948273688128/t0(0) o400->d0a28e26-69bb-8579-694f-2f71df3dba35@10.31.143.83@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513071.855575:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513071.855576:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513071.855581:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.855587:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eff0 00000100:00000200:24.0:1616513071.855588:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693948273688128, offset 192 00000800:00000200:2.0:1616513071.855591:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513071.855595:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.83@tcp1 00000400:00000200:24.0:1616513071.855603:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.83@tcp1 routed destination 00000400:00000200:24.0:1616513071.855619:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.83@tcp1(10.31.143.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.855626:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.855632:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.855635:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.855637:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.855642:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.855648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.855659:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.855664:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513071.855666:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513071.855669:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513071.855673:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513071.855677:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf37e8 00000800:00000200:16.0:1616513071.855681:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:23.0:1616513071.870667:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:23.0:1616513071.870675:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513071.874627:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513071.874688:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.874697:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513071.874706:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.252@tcp1 : PUT - for me 00000400:00000200:28.0:1616513071.874715:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.252@tcp1 of length 224 into portal 26 MB=0x5fffcb379d380 00000400:00000200:28.0:1616513071.874722:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.252@tcp1 of length 224/224 into md 0xf97c45f9 [1] + 0 00000400:00000200:28.0:1616513071.874727:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1078 00000400:00000200:28.0:1616513071.874730:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513071.874732:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513071.874737:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1078 00000800:00000200:28.0:1616513071.874740:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.874741:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.874745:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513071.874748:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.874751:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.874752:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.874760:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.252@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.874769:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.252@tcp1 of length 224 into portal 12 MB=0x5fffcb379d400 00000400:00000200:12.0:1616513071.874775:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.252@tcp1 of length 224/224 into md 0x865b9f31 [1] + 51600 00000400:00000200:12.0:1616513071.874780:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.874783:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513071.874789:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.874792:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513071.874794:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.874804:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688835691500416 00010000:00000200:23.0:1616513071.874817:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688835691500416/t0(0) o400->130ed0d0-26ba-006b-b1cb-a2a1cc86a230@10.31.141.252@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.874831:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688835691500416, offset 192 00000400:00000200:23.0:1616513071.874838:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.252@tcp1 00000400:00000200:23.0:1616513071.874846:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.252@tcp1 routed destination 00000400:00000200:23.0:1616513071.874861:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.252@tcp1(10.31.141.252@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513071.874866:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688835691500544 00000800:00000200:23.0:1616513071.874868:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.874874:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.874877:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:24.0:1616513071.874879:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57f980 x1688835691500544/t0(0) o400->daaa1c64-1f9e-03df-9d4c-448574f347b2@10.31.141.252@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513071.874879:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.874884:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513071.874891:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513071.874892:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513071.874893:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688835691500544, offset 192 00000400:00000200:24.0:1616513071.874899:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.252@tcp1 00000400:00000200:24.0:1616513071.874907:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.252@tcp1 routed destination 00000400:00000200:24.0:1616513071.874922:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.252@tcp1(10.31.141.252@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.874929:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.874935:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.874938:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.874940:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.874945:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513071.874947:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513071.874951:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513071.874955:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513071.874958:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513071.874959:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513071.874965:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513071.874971:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fcb0 00000800:00000200:26.0:1616513071.874975:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513071.875003:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.875010:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.875014:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513071.875015:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513071.875022:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513071.875026:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:16.0:1616513071.875030:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.920847:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513071.920904:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.920914:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513071.920923:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.80@tcp1 : PUT - for me 00000400:00000200:14.0:1616513071.920933:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.80@tcp1 of length 224 into portal 26 MB=0x601887482c600 00000400:00000200:14.0:1616513071.920940:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.80@tcp1 of length 224/224 into md 0xf97c4629 [1] + 0 00000400:00000200:14.0:1616513071.920945:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1cb0 00000400:00000200:14.0:1616513071.920948:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513071.920951:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513071.920957:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1cb0 00000800:00000200:2.0:1616513071.920959:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.920960:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.920963:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.920965:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.921020:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690535442171392 00010000:00000200:23.0:1616513071.921032:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1690535442171392/t0(0) o400->5ea6261a-f8bd-371b-9779-0513c4e57538@10.31.142.80@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.921045:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690535442171392, offset 192 00000400:00000200:23.0:1616513071.921051:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.80@tcp1 00000400:00000200:23.0:1616513071.921059:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.80@tcp1 routed destination 00000400:00000200:23.0:1616513071.921073:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.80@tcp1(10.31.142.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.921079:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.921085:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.921088:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.921090:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.921095:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.921099:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.921101:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513071.921110:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.921117:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513071.921120:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513071.921127:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513071.921133:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e770 00000800:00000200:30.0:1616513071.921138:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.921140:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513071.921306:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513071.921363:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.921372:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.921381:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.80@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.921391:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.80@tcp1 of length 224 into portal 12 MB=0x601887482d080 00000400:00000200:0.0:1616513071.921398:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.80@tcp1 of length 224/224 into md 0x865b9f31 [1] + 51824 00000400:00000200:0.0:1616513071.921403:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.921406:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.921417:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.921419:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.921420:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.921423:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513071.921447:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690535442174080 00010000:00000200:24.0:1616513071.921461:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3a80 x1690535442174080/t0(0) o400->790ed2e6-ccd9-9c5a-74fc-0c4c172bf42a@10.31.142.80@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.921476:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690535442174080, offset 192 00000400:00000200:24.0:1616513071.921483:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.80@tcp1 00000400:00000200:24.0:1616513071.921492:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.80@tcp1 routed destination 00000400:00000200:24.0:1616513071.921507:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.80@tcp1(10.31.142.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.921514:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.921520:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.921523:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.921527:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.921533:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.921538:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.921593:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.921600:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.921603:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.921609:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.921613:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ed0 00000800:00000200:28.0:1616513071.921617:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.921620:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.930226:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513071.930279:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.930289:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.930298:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.185@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.930307:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.185@tcp1 of length 224 into portal 26 MB=0x5ffd5f1451d80 00000400:00000200:12.0:1616513071.930314:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.185@tcp1 of length 224/224 into md 0xf97c4669 [1] + 0 00000400:00000200:12.0:1616513071.930319:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d07f8 00000400:00000200:12.0:1616513071.930321:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.930324:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513071.930329:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d07f8 00000800:00000200:12.0:1616513071.930332:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.930335:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.930336:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513071.930343:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.930346:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.930347:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.930355:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.185@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.930365:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.185@tcp1 of length 224 into portal 12 MB=0x5ffd5f1451e80 00000400:00000200:0.0:1616513071.930371:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.185@tcp1 of length 224/224 into md 0x865b9f31 [1] + 52048 00000400:00000200:0.0:1616513071.930376:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.930379:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513071.930386:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.930389:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.930391:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.930395:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688669224508800 00010000:00000200:23.0:1616513071.930407:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688669224508800/t0(0) o400->9ba29459-7fe8-eaf3-7c14-f16fd2af03c1@10.31.138.185@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.930421:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688669224508800, offset 192 00000400:00000200:23.0:1616513071.930428:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.185@tcp1 00000400:00000200:23.0:1616513071.930452:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.185@tcp1 routed destination 00000100:00000200:24.0:1616513071.930465:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688669224509056 00000400:00000200:23.0:1616513071.930467:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.185@tcp1(10.31.138.185@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.930474:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:24.0:1616513071.930478:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d6300 x1688669224509056/t0(0) o400->41d40723-389d-042a-3d13-6ffd3a572c49@10.31.138.185@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513071.930480:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.930483:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.930486:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.930491:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513071.930492:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688669224509056, offset 192 00000800:00000200:7.2:1616513071.930496:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.930498:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513071.930499:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.185@tcp1 00000400:00000200:24.0:1616513071.930507:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.185@tcp1 routed destination 00000400:00000200:24.0:1616513071.930522:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.185@tcp1(10.31.138.185@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.930529:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.930535:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.930538:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.930540:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.930545:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.930550:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513071.930550:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.930557:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.930561:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.930567:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.930573:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f210 00000800:00000200:2.0:1616513071.930577:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.930580:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513071.930605:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.930612:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513071.930615:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513071.930616:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513071.930622:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513071.930626:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2a18 00000800:00000200:16.0:1616513071.930630:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.933412:0:48593:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513071.933425:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.933433:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513071.933447:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.65@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.933457:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.65@tcp1 of length 224 into portal 26 MB=0x5ffd6ca929000 00000400:00000200:30.0:1616513071.933463:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.65@tcp1 of length 224/224 into md 0xf97c4679 [1] + 0 00000400:00000200:30.0:1616513071.933468:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0cc0 00000400:00000200:30.0:1616513071.933471:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.933474:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513071.933478:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0cc0 00000800:00000200:26.0:1616513071.933479:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.933491:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.933493:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513071.933495:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.933542:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672870240256 00010000:00000200:23.0:1616513071.933554:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688672870240256/t0(0) o400->e14207ca-646d-bd4c-b6fe-a6f195d53a07@10.31.129.65@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.933568:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688672870240256, offset 192 00000400:00000200:23.0:1616513071.933574:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.65@tcp1 00000400:00000200:23.0:1616513071.933582:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.65@tcp1 routed destination 00000400:00000200:23.0:1616513071.933598:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.65@tcp1(10.31.129.65@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.933605:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.933611:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.933614:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.933616:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.933621:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513071.933626:0:48593:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.933628:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513071.933681:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.933688:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.933691:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.933698:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.933704:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fb18 00000800:00000200:2.0:1616513071.933707:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.933710:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513071.933777:0:48593:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513071.933788:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.933797:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513071.933797:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513071.933806:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.65@tcp1 : PUT - for me 00000400:00000200:26.0:1616513071.933815:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.65@tcp1 of length 224 into portal 12 MB=0x5ffd6ca929340 00000400:00000200:26.0:1616513071.933822:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.65@tcp1 of length 224/224 into md 0xf96450b9 [1] + 19176 00000400:00000200:26.0:1616513071.933827:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513071.933830:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513071.933837:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.933840:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513071.933842:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513071.933903:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672870241088 00010000:00000200:18.0:1616513071.933916:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c696780 x1688672870241088/t0(0) o400->14bb0b9d-d1b1-9e74-4048-5d9aabd0b98e@10.31.129.65@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513071.933941:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688672870241088, offset 192 00000400:00000200:18.0:1616513071.933948:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.65@tcp1 00000400:00000200:18.0:1616513071.933955:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.65@tcp1 routed destination 00000400:00000200:18.0:1616513071.933971:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.65@tcp1(10.31.129.65@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.933978:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.933983:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.933986:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.933988:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.933993:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.934039:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.934094:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.934101:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.934103:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.934110:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.934114:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccaa0 00000800:00000200:28.0:1616513071.934118:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.934121:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.934476:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513071.934528:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513071.934537:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.934546:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.148@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.934556:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.148@tcp1 of length 224 into portal 26 MB=0x603fcd2a6b300 00000400:00000200:12.0:1616513071.934563:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.148@tcp1 of length 224/224 into md 0xf97c4689 [1] + 0 00000400:00000200:12.0:1616513071.934568:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0088 00000400:00000200:12.0:1616513071.934571:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.934573:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513071.934578:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0088 00000800:00000200:12.0:1616513071.934581:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.934583:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.934584:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.934590:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:23.0:1616513071.934593:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234261046016 00000800:00000200:16.0:1616513071.934593:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513071.934593:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:23.0:1616513071.934599:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1693234261046016/t0(0) o400->766aa7e3-8e20-3252-2b1c-6dc3596aac4c@10.31.138.148@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513071.934602:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.148@tcp1 : PUT - for me 00000100:00000200:23.0:1616513071.934610:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234261046016, offset 192 00000400:00000200:16.0:1616513071.934612:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.148@tcp1 of length 224 into portal 12 MB=0x603fcd2a6b380 00000400:00000200:23.0:1616513071.934615:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.148@tcp1 00000400:00000200:16.0:1616513071.934618:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.148@tcp1 of length 224/224 into md 0x865b9f31 [1] + 52272 00000400:00000200:23.0:1616513071.934622:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.148@tcp1 routed destination 00000400:00000200:16.0:1616513071.934623:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513071.934626:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513071.934633:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:23.0:1616513071.934635:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.148@tcp1(10.31.138.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513071.934636:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513071.934638:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:23.0:1616513071.934641:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.934648:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.934650:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.934652:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.934655:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513071.934660:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513071.934660:0:48593:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.934670:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.934674:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.934676:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513071.934679:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513071.934682:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.934685:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f540 00000800:00000200:2.0:1616513071.934688:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513071.934699:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234261046144 00010000:00000200:24.0:1616513071.934711:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e2400 x1693234261046144/t0(0) o400->74869c6e-502a-b4ff-3529-60c3bea231e2@10.31.138.148@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.934726:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234261046144, offset 192 00000400:00000200:24.0:1616513071.934733:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.148@tcp1 00000400:00000200:24.0:1616513071.934745:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.148@tcp1 routed destination 00000400:00000200:24.0:1616513071.934760:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.148@tcp1(10.31.138.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.934767:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.934773:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.934775:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.934778:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.934783:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.934787:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.934799:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.934806:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513071.934809:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513071.934810:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513071.934816:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513071.934820:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ba0 00000800:00000200:0.0:1616513071.934824:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.939864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513071.939887:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.939897:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513071.939906:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.45@tcp1 : PUT - for me 00000400:00000200:28.0:1616513071.939914:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.45@tcp1 of length 224 into portal 26 MB=0x60194bf3147c0 00000400:00000200:28.0:1616513071.939922:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.45@tcp1 of length 224/224 into md 0xf97c46a9 [1] + 0 00000400:00000200:28.0:1616513071.939926:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d02a8 00000400:00000200:28.0:1616513071.939929:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513071.939932:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513071.939936:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d02a8 00000800:00000200:28.0:1616513071.939940:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.939941:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.939942:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.939945:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.939970:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:23.0:1616513071.939999:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588234729408 00010000:00000200:23.0:1616513071.940010:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1690588234729408/t0(0) o400->2f1ad12d-12e3-3fec-26db-65a22323481a@10.31.141.45@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513071.940021:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513071.940023:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588234729408, offset 192 00000400:00000200:23.0:1616513071.940030:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.45@tcp1 00000800:00000200:12.0:1616513071.940030:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:23.0:1616513071.940038:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.45@tcp1 routed destination 00000400:00000200:12.0:1616513071.940039:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.45@tcp1 : PUT - for me 00000400:00000200:12.0:1616513071.940049:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.45@tcp1 of length 224 into portal 12 MB=0x60194bf3148c0 00000400:00000200:23.0:1616513071.940053:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.45@tcp1(10.31.141.45@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513071.940055:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.45@tcp1 of length 224/224 into md 0x865b9f31 [1] + 52496 00000800:00000200:23.0:1616513071.940060:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513071.940060:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513071.940062:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:23.0:1616513071.940066:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.940068:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513071.940068:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:23.0:1616513071.940070:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513071.940071:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.940072:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:23.0:1616513071.940075:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513071.940075:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.940081:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513071.940081:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.940093:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.940098:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513071.940101:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513071.940106:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:26.0:1616513071.940107:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513071.940111:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e110 00000800:00000200:30.0:1616513071.940114:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513071.940130:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588234729664 00010000:00000200:24.0:1616513071.940141:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57a880 x1690588234729664/t0(0) o400->f557d763-ff2b-ba2a-d20c-e1ee33fc71e3@10.31.141.45@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513071.940154:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588234729664, offset 192 00000400:00000200:24.0:1616513071.940162:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.45@tcp1 00000400:00000200:24.0:1616513071.940169:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.45@tcp1 routed destination 00000400:00000200:24.0:1616513071.940183:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.45@tcp1(10.31.141.45@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.940189:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.940195:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.940197:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.940200:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.940204:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.940209:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.940218:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.940223:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513071.940226:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513071.940228:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513071.940232:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513071.940235:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:16.0:1616513071.940239:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.957042:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513071.957097:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.957107:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513071.957116:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.140@tcp1 : PUT - for me 00000400:00000200:14.0:1616513071.957125:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.140@tcp1 of length 224 into portal 12 MB=0x5ffd7073cf600 00000400:00000200:14.0:1616513071.957131:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.140@tcp1 of length 224/224 into md 0xf96450b9 [1] + 19400 00000400:00000200:14.0:1616513071.957137:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513071.957139:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513071.957146:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.957149:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513071.957152:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513071.957156:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513071.957184:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513071.957208:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673888040448 00010000:00000200:18.0:1616513071.957219:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa1200 x1688673888040448/t0(0) o400->9319f1e4-8e08-e2a4-7669-0f880241f9f0@10.31.129.140@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513071.957233:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688673888040448, offset 192 00000400:00000200:18.0:1616513071.957239:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.140@tcp1 00000800:00000200:12.0:1616513071.957239:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513071.957246:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.140@tcp1 routed destination 00000800:00000200:12.0:1616513071.957249:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513071.957257:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.140@tcp1 : PUT - for me 00000400:00000200:18.0:1616513071.957259:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.140@tcp1(10.31.129.140@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.957265:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513071.957265:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.140@tcp1 of length 224 into portal 26 MB=0x5ffd7073cf240 00000800:00000200:18.0:1616513071.957271:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513071.957272:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.140@tcp1 of length 224/224 into md 0xf97c46c9 [1] + 0 00000800:00000200:18.0:1616513071.957273:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513071.957275:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513071.957276:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1320 00000400:00000200:12.0:1616513071.957279:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513071.957280:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513071.957281:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:7.2:1616513071.957285:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513071.957287:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1320 00000800:00000200:12.0:1616513071.957290:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.957293:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.957294:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513071.957298:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513071.957340:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513071.957347:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513071.957350:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513071.957351:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513071.957355:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673888039488 00000400:00000200:26.0:1616513071.957358:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513071.957362:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccc38 00000800:00000200:26.0:1616513071.957366:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:23.0:1616513071.957366:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688673888039488/t0(0) o400->d859710f-98af-5985-d3f8-914f71e91d65@10.31.129.140@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.957381:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688673888039488, offset 192 00000400:00000200:23.0:1616513071.957387:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.140@tcp1 00000400:00000200:23.0:1616513071.957396:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.140@tcp1 routed destination 00000400:00000200:23.0:1616513071.957412:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.140@tcp1(10.31.129.140@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.957419:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513071.957425:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.957428:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513071.957430:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513071.957449:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513071.957454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513071.957459:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513071.957508:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.957515:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.957519:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.957525:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.957534:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f100 00000800:00000200:28.0:1616513071.957538:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.957541:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.984692:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513071.984748:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513071.984757:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513071.984768:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.219@tcp1 : PUT - for me 00000400:00000200:2.0:1616513071.984777:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.219@tcp1 of length 224 into portal 26 MB=0x5fff14da18640 00000400:00000200:2.0:1616513071.984784:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.219@tcp1 of length 224/224 into md 0xf97c46e9 [1] + 0 00000400:00000200:2.0:1616513071.984788:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1210 00000400:00000200:2.0:1616513071.984791:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513071.984794:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513071.984799:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1210 00000800:00000200:14.0:1616513071.984802:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.984802:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513071.984805:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513071.984809:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513071.984812:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.984813:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513071.984821:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.219@tcp1 : PUT - for me 00000400:00000200:14.0:1616513071.984830:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.219@tcp1 of length 224 into portal 12 MB=0x5fff14da18900 00000400:00000200:14.0:1616513071.984836:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.219@tcp1 of length 224/224 into md 0xf96450b9 [1] + 19624 00000400:00000200:14.0:1616513071.984842:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513071.984844:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513071.984851:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513071.984854:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513071.984858:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513071.984866:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688786738185792 00010000:00000200:23.0:1616513071.984877:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688786738185792/t0(0) o400->0fed50f4-4c3a-8355-dd39-a25d8290a6d0@10.31.141.219@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.984892:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688786738185792, offset 192 00000400:00000200:23.0:1616513071.984898:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.219@tcp1 00000400:00000200:23.0:1616513071.984906:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.219@tcp1 routed destination 00000100:00000200:18.0:1616513071.984916:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688786738186496 00000400:00000200:23.0:1616513071.984922:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.219@tcp1(10.31.141.219@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:18.0:1616513071.984928:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa4380 x1688786738186496/t0(0) o400->a4fd2db0-d819-0206-796e-94892dba3857@10.31.141.219@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513071.984929:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.984935:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.984938:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.984940:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:18.0:1616513071.984942:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688786738186496, offset 192 00000800:00000200:23.0:1616513071.984945:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513071.984949:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.219@tcp1 00000800:00000200:7.2:1616513071.984950:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.984952:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513071.984956:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.219@tcp1 routed destination 00000400:00000200:18.0:1616513071.984972:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.219@tcp1(10.31.141.219@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.984980:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.984987:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.984989:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.984991:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.984996:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513071.985003:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513071.985011:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.985014:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513071.985014:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513071.985021:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513071.985027:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f870 00000800:00000200:26.0:1616513071.985030:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513071.985044:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.985100:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513071.985110:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513071.985113:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513071.985120:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513071.985124:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc990 00000800:00000200:0.0:1616513071.985128:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.985131:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.992596:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513071.992655:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513071.992664:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513071.992673:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.41@tcp1 : PUT - for me 00000400:00000200:28.0:1616513071.992682:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.41@tcp1 of length 224 into portal 26 MB=0x604d6e4281fc0 00000400:00000200:28.0:1616513071.992689:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.41@tcp1 of length 224/224 into md 0xf97c4709 [1] + 0 00000400:00000200:28.0:1616513071.992693:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0440 00000400:00000200:28.0:1616513071.992696:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513071.992699:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513071.992704:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0440 00000800:00000200:28.0:1616513071.992707:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.992708:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.992709:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.992716:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513071.992719:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513071.992719:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513071.992728:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.41@tcp1 : PUT - for me 00000400:00000200:16.0:1616513071.992737:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.41@tcp1 of length 224 into portal 12 MB=0x604d6e4282200 00000400:00000200:16.0:1616513071.992743:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.41@tcp1 of length 224/224 into md 0x865b9f31 [1] + 52720 00000400:00000200:16.0:1616513071.992748:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513071.992753:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513071.992759:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513071.992763:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513071.992765:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513071.992769:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694170857611200 00010000:00000200:23.0:1616513071.992780:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1694170857611200/t0(0) o400->12bfbb1a-74ee-7d72-33f1-7c0f60125339@10.31.142.41@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.992795:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694170857611200, offset 192 00000400:00000200:23.0:1616513071.992801:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.41@tcp1 00000400:00000200:23.0:1616513071.992809:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.41@tcp1 routed destination 00000400:00000200:23.0:1616513071.992824:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.41@tcp1(10.31.142.41@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513071.992831:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.992838:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:24.0:1616513071.992841:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694170857611776 00000800:00000200:23.0:1616513071.992841:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.992843:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.992848:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:24.0:1616513071.992853:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e5a00 x1694170857611776/t0(0) o400->9973b1b3-6571-904b-450c-99f746f742fb@10.31.142.41@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513071.992854:0:48599:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.992855:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513071.992866:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694170857611776, offset 192 00000400:00000200:24.0:1616513071.992872:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.41@tcp1 00000400:00000200:24.0:1616513071.992880:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.41@tcp1 routed destination 00000400:00000200:24.0:1616513071.992894:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.41@tcp1(10.31.142.41@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513071.992900:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513071.992906:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513071.992909:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513071.992911:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513071.992915:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513071.992915:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513071.992920:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513071.992922:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.992925:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.992932:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.992938:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e908 00000800:00000200:2.0:1616513071.992941:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.992944:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513071.992989:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513071.992996:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513071.992999:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513071.993000:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513071.993006:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513071.993010:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3d38 00000800:00000200:12.0:1616513071.993014:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513071.997954:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513071.998011:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513071.998020:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513071.998029:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.107@tcp1 : PUT - for me 00000400:00000200:0.0:1616513071.998039:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.107@tcp1 of length 224 into portal 26 MB=0x5fda82e709dc0 00000400:00000200:0.0:1616513071.998045:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.107@tcp1 of length 224/224 into md 0xf97c4749 [1] + 0 00000400:00000200:0.0:1616513071.998050:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d13a8 00000400:00000200:0.0:1616513071.998053:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513071.998056:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513071.998061:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d13a8 00000800:00000200:16.0:1616513071.998064:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513071.998064:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513071.998067:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513071.998067:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513071.998068:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513071.998078:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.998086:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513071.998094:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.107@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.998102:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.107@tcp1 of length 224 into portal 12 MB=0x5fda82e709e40 00000400:00000200:30.0:1616513071.998108:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.107@tcp1 of length 224/224 into md 0xf96450b9 [1] + 19848 00000400:00000200:30.0:1616513071.998113:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513071.998115:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513071.998121:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513071.998124:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513071.998125:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513071.998125:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1686273659018688 00000800:00000200:30.0:1616513071.998129:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:23.0:1616513071.998136:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1686273659018688/t0(0) o400->14b8eb5c-09a3-9323-3e02-6e470b301362@10.31.143.107@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513071.998151:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1686273659018688, offset 192 00000400:00000200:23.0:1616513071.998160:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.107@tcp1 00000400:00000200:23.0:1616513071.998168:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.107@tcp1 routed destination 00000400:00000200:23.0:1616513071.998184:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.107@tcp1(10.31.143.107@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513071.998186:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1686273659018816 00000800:00000200:23.0:1616513071.998190:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513071.998197:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:18.0:1616513071.998198:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7a400 x1686273659018816/t0(0) o400->b2138d9a-2d6d-25fe-7439-85ea2c7c7ae0@10.31.143.107@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513071.998200:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513071.998202:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.998207:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:18.0:1616513071.998212:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1686273659018816, offset 192 00000800:00000200:7.2:1616513071.998212:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513071.998214:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513071.998219:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.107@tcp1 00000400:00000200:18.0:1616513071.998226:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.107@tcp1 routed destination 00000400:00000200:18.0:1616513071.998241:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.107@tcp1(10.31.143.107@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.998248:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513071.998254:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513071.998257:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513071.998259:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513071.998264:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513071.998267:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513071.998269:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513071.998274:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513071.998277:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513071.998284:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513071.998290:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e908 00000800:00000200:2.0:1616513071.998294:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513071.998296:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513071.998321:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513071.998328:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513071.998331:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513071.998338:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513071.998342:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdd38 00000800:00000200:28.0:1616513071.998346:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513071.998349:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513071.999555:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513071.999609:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513071.999619:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513071.999619:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513071.999624:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513071.999628:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.195@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.999630:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.195@tcp1 : PUT - for me 00000400:00000200:30.0:1616513071.999641:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.195@tcp1 of length 224 into portal 12 MB=0x604a295c8fa40 00000400:00000200:26.0:1616513071.999641:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.195@tcp1 of length 224 into portal 26 MB=0x604a295c8f9c0 00000400:00000200:30.0:1616513071.999645:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.195@tcp1 of length 224/224 into md 0xf96450b9 [1] + 20072 00000400:00000200:30.0:1616513071.999649:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513071.999652:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.195@tcp1 of length 224/224 into md 0xf97c4759 [1] + 0 00000400:00000200:26.0:1616513071.999656:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1650 00000100:00000200:30.0:1616513071.999658:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:26.0:1616513071.999659:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513071.999662:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000100:00000200:26.0:1616513071.999663:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:30.0:1616513071.999664:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513071.999666:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513071.999667:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1650 00000800:00000200:30.0:1616513071.999668:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513071.999671:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513071.999673:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513071.999675:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513071.999723:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946204453440 00000100:00000200:23.0:1616513071.999733:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946204453312 00010000:00000200:18.0:1616513071.999734:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7e300 x1693946204453440/t0(0) o400->043c4316-8951-bb6e-76f9-70920af4213e@10.31.140.195@tcp1:697/0 lens 224/224 e 0 to 0 dl 1616513077 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:23.0:1616513071.999744:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1693946204453312/t0(0) o400->494f4090-4f61-c3a3-d5f3-b2db0b91238f@10.31.140.195@tcp1:28/0 lens 224/224 e 0 to 0 dl 1616513163 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513071.999748:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946204453440, offset 192 00000400:00000200:18.0:1616513071.999754:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.195@tcp1 00000100:00000200:23.0:1616513071.999759:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946204453312, offset 192 00000400:00000200:18.0:1616513071.999762:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.195@tcp1 routed destination 00000400:00000200:23.0:1616513071.999766:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.195@tcp1 00000400:00000200:23.0:1616513071.999775:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.195@tcp1 routed destination 00000400:00000200:18.0:1616513071.999776:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.195@tcp1(10.31.140.195@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.999782:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513071.999788:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:23.0:1616513071.999791:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.195@tcp1(10.31.140.195@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513071.999791:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513071.999793:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513071.999797:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:23.0:1616513071.999798:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513071.999802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513071.999804:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513071.999807:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513071.999809:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513071.999814:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:23.0:1616513071.999821:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513071.999855:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513071.999862:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513071.999865:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513071.999867:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513071.999871:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513071.999873:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513071.999877:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd650 00000800:00000200:14.0:1616513071.999881:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513071.999927:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513071.999935:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513071.999938:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513071.999945:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513071.999951:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f100 00000800:00000200:16.0:1616513071.999955:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513071.999958:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.014426:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.014482:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.014492:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.014501:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.51@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.014510:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.51@tcp1 of length 224 into portal 12 MB=0x5ffd4daaa7b40 00000400:00000200:28.0:1616513072.014517:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.51@tcp1 of length 224/224 into md 0x865b9f31 [1] + 52944 00000400:00000200:28.0:1616513072.014522:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.014525:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513072.014531:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.014534:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.014536:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.014541:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.014596:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664550308672 00010000:00000200:24.0:1616513072.014609:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0e9f80 x1688664550308672/t0(0) o400->872f0e94-83d9-43c8-eacb-8388ff2a7cf6@10.31.138.51@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.014624:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664550308672, offset 192 00000400:00000200:24.0:1616513072.014630:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.51@tcp1 00000400:00000200:24.0:1616513072.014637:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.51@tcp1 routed destination 00000400:00000200:24.0:1616513072.014653:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.51@tcp1(10.31.138.51@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.014659:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513072.014665:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.014670:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513072.014672:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513072.014677:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.014685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.014697:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.014702:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.014704:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513072.014709:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.014712:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3d38 00000800:00000200:30.0:1616513072.014715:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513072.014717:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.015989:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513072.016046:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.016056:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.016064:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.51@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.016073:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.51@tcp1 of length 224 into portal 26 MB=0x5ffd4daaa7940 00000400:00000200:16.0:1616513072.016080:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.51@tcp1 of length 224/224 into md 0xf97c4779 [1] + 0 00000400:00000200:16.0:1616513072.016085:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1870 00000400:00000200:16.0:1616513072.016087:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.016090:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513072.016095:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1870 00000800:00000200:16.0:1616513072.016098:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.016101:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513072.016102:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.016105:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513072.016161:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664550308160 00010000:00000200:23.0:1616513072.016173:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688664550308160/t0(0) o400->d801d4ab-c060-3e26-f604-4f700f02c396@10.31.138.51@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513072.016187:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664550308160, offset 192 00000400:00000200:23.0:1616513072.016193:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.51@tcp1 00000400:00000200:23.0:1616513072.016202:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.51@tcp1 routed destination 00000400:00000200:23.0:1616513072.016217:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.51@tcp1(10.31.138.51@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513072.016228:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:23.0:1616513072.016234:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513072.016237:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:23.0:1616513072.016239:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:23.0:1616513072.016244:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.016249:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513072.016251:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513072.016303:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.016310:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.016313:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513072.016320:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.016326:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f540 00000800:00000200:12.0:1616513072.016329:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.016332:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.017617:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513072.017674:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.017684:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.017693:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.214@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.017702:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.214@tcp1 of length 224 into portal 26 MB=0x5ffe5056c7840 00000400:00000200:16.0:1616513072.017709:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.214@tcp1 of length 224/224 into md 0xf97c4789 [1] + 0 00000400:00000200:16.0:1616513072.017714:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0990 00000400:00000200:16.0:1616513072.017716:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.017719:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513072.017726:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0990 00000800:00000200:0.0:1616513072.017728:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.017729:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.017731:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.017733:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513072.017791:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688733987141696 00010000:00000200:23.0:1616513072.017802:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688733987141696/t0(0) o400->40fa3414-bfff-8044-87a5-6f2faf2c96b3@10.31.141.214@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513072.017816:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688733987141696, offset 192 00000400:00000200:23.0:1616513072.017822:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.214@tcp1 00000400:00000200:23.0:1616513072.017830:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.214@tcp1 routed destination 00000400:00000200:23.0:1616513072.017845:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.214@tcp1(10.31.141.214@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513072.017852:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:7.2:1616513072.017853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513072.017858:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513072.017861:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513072.017863:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:23.0:1616513072.017868:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:23.0:1616513072.017874:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513072.017906:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.017916:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.017926:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.214@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.017935:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.214@tcp1 of length 224 into portal 12 MB=0x5ffe5056c7b80 00000400:00000200:2.0:1616513072.017941:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.214@tcp1 of length 224/224 into md 0xf96450b9 [1] + 20296 00000400:00000200:2.0:1616513072.017946:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.017949:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513072.017957:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513072.017960:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513072.017960:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513072.017967:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513072.017968:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513072.017970:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513072.017972:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513072.017978:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513072.017984:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f210 00000800:00000200:14.0:1616513072.017988:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.018018:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688733987142528 00010000:00000200:18.0:1616513072.018030:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1389b00 x1688733987142528/t0(0) o400->7da90d6c-0c40-96ce-c2c7-c4f64fd1fe84@10.31.141.214@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.018044:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688733987142528, offset 192 00000400:00000200:18.0:1616513072.018050:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.214@tcp1 00000400:00000200:18.0:1616513072.018057:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.214@tcp1 routed destination 00000400:00000200:18.0:1616513072.018071:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.214@tcp1(10.31.141.214@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.018077:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.018083:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.018086:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.018088:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.018092:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.018097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.018149:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.018156:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.018158:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513072.018165:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.018169:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cddc0 00000800:00000200:12.0:1616513072.018173:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.018176:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.044904:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513072.044961:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.044970:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513072.044979:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.99@tcp1 : PUT - for me 00000400:00000200:0.0:1616513072.044989:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.99@tcp1 of length 224 into portal 26 MB=0x5fff5c8dfef40 00000400:00000200:0.0:1616513072.044996:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.99@tcp1 of length 224/224 into md 0xf97c4799 [1] + 0 00000400:00000200:0.0:1616513072.045001:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d05d8 00000400:00000200:0.0:1616513072.045004:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.045007:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513072.045012:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d05d8 00000800:00000200:0.0:1616513072.045015:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.045016:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.045018:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513072.045021:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513072.045079:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688805985742656 00010000:00000200:23.0:1616513072.045090:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688805985742656/t0(0) o400->c43f3f3b-0720-d0f4-9006-c2a5d000e5c2@10.31.136.99@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513072.045105:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688805985742656, offset 192 00000400:00000200:23.0:1616513072.045111:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.99@tcp1 00000400:00000200:23.0:1616513072.045120:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.99@tcp1 routed destination 00000400:00000200:23.0:1616513072.045135:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.99@tcp1(10.31.136.99@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513072.045142:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513072.045148:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513072.045151:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513072.045155:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513072.045160:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513072.045167:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.045167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.045221:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.045229:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513072.045232:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.045232:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.045239:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.045245:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fcb0 00000800:00000200:26.0:1616513072.045248:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.045328:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513072.045392:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.045401:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513072.045409:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.99@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.045418:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.99@tcp1 of length 224 into portal 12 MB=0x5fff5c8dff040 00000400:00000200:12.0:1616513072.045424:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.99@tcp1 of length 224/224 into md 0x865b9f31 [1] + 53168 00000400:00000200:12.0:1616513072.045429:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.045431:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513072.045446:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.045451:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.045454:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.045458:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.045517:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688805985742912 00010000:00000200:24.0:1616513072.045530:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc578d80 x1688805985742912/t0(0) o400->0d27cdc7-88d1-cccf-abc8-7f4f161eabb3@10.31.136.99@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.045544:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688805985742912, offset 192 00000400:00000200:24.0:1616513072.045550:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.99@tcp1 00000400:00000200:24.0:1616513072.045558:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.99@tcp1 routed destination 00000400:00000200:24.0:1616513072.045573:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.99@tcp1(10.31.136.99@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.045580:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.045586:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.045589:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.045591:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.045596:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.045600:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.045611:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.045616:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.045619:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513072.045621:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.045625:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.045629:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:16.0:1616513072.045633:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.054150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.054205:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.054215:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.054224:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.134@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.054232:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.134@tcp1 of length 224 into portal 26 MB=0x603fc5a4022c0 00000400:00000200:2.0:1616513072.054239:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.134@tcp1 of length 224/224 into md 0xf97c47b9 [1] + 0 00000400:00000200:2.0:1616513072.054243:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d15c8 00000400:00000200:2.0:1616513072.054246:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.054249:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513072.054254:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d15c8 00000800:00000200:14.0:1616513072.054257:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.054257:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.054260:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.054264:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.054267:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513072.054267:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513072.054276:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.134@tcp1 : PUT - for me 00000400:00000200:14.0:1616513072.054284:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.134@tcp1 of length 224 into portal 12 MB=0x603fc5a402340 00000400:00000200:14.0:1616513072.054290:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.134@tcp1 of length 224/224 into md 0xf96450b9 [1] + 20520 00000400:00000200:14.0:1616513072.054294:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.054297:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513072.054303:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.054306:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.054308:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513072.054320:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693232241058496 00010000:00000200:23.0:1616513072.054331:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1693232241058496/t0(0) o400->4b7b9e8c-6ff2-9324-15fb-a70279f1ef87@10.31.138.134@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513072.054346:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693232241058496, offset 192 00000400:00000200:23.0:1616513072.054353:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.134@tcp1 00000400:00000200:23.0:1616513072.054361:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.134@tcp1 routed destination 00000100:00000200:18.0:1616513072.054363:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693232241058624 00010000:00000200:18.0:1616513072.054374:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa3600 x1693232241058624/t0(0) o400->ca354380-a953-14ef-106d-47919181477f@10.31.138.134@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:23.0:1616513072.054377:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.134@tcp1(10.31.138.134@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513072.054384:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:18.0:1616513072.054387:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693232241058624, offset 192 00000800:00000200:23.0:1616513072.054391:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513072.054393:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.134@tcp1 00000800:00000200:23.0:1616513072.054394:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513072.054396:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513072.054400:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.134@tcp1 routed destination 00000800:00000200:23.0:1616513072.054401:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513072.054409:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.054409:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513072.054413:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.134@tcp1(10.31.138.134@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.054418:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.054424:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.054426:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.054428:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.054432:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513072.054463:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.054471:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513072.054473:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.054474:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513072.054475:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513072.054481:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.054487:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f650 00000800:00000200:26.0:1616513072.054491:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513072.054529:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.054536:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.054539:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.054545:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.054549:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc880 00000800:00000200:28.0:1616513072.054553:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.054556:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.064237:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.064293:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.064302:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.064311:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.191@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.064320:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.191@tcp1 of length 224 into portal 26 MB=0x6001bde6bd680 00000400:00000200:2.0:1616513072.064327:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.191@tcp1 of length 224/224 into md 0xf97c47c9 [1] + 0 00000400:00000200:2.0:1616513072.064332:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0660 00000400:00000200:2.0:1616513072.064335:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.064337:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513072.064342:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0660 00000800:00000200:2.0:1616513072.064345:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.064347:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.064348:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.064353:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.064357:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513072.064357:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513072.064366:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.191@tcp1 : PUT - for me 00000400:00000200:14.0:1616513072.064375:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.191@tcp1 of length 224 into portal 12 MB=0x6001bde6bd780 00000400:00000200:14.0:1616513072.064381:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.191@tcp1 of length 224/224 into md 0xf96450b9 [1] + 20744 00000400:00000200:14.0:1616513072.064387:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.064389:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513072.064396:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.064399:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.064401:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513072.064407:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969555990144 00010000:00000200:23.0:1616513072.064419:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688969555990144/t0(0) o400->cbdf0d10-5244-a157-311f-89319d08b9c9@10.31.138.191@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513072.064448:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688969555990144, offset 192 00000400:00000200:23.0:1616513072.064455:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.191@tcp1 00000400:00000200:23.0:1616513072.064464:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.191@tcp1 routed destination 00000100:00000200:18.0:1616513072.064473:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969555990400 00000400:00000200:23.0:1616513072.064480:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.191@tcp1(10.31.138.191@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:18.0:1616513072.064486:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa7980 x1688969555990400/t0(0) o400->bab4dec9-d6bd-7345-23c6-464a5e107a50@10.31.138.191@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513072.064487:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:23.0:1616513072.064493:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513072.064496:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513072.064498:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:18.0:1616513072.064500:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688969555990400, offset 192 00000800:00000200:23.0:1616513072.064503:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513072.064507:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.191@tcp1 00000800:00000200:7.2:1616513072.064509:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513072.064511:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513072.064515:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.191@tcp1 routed destination 00000400:00000200:18.0:1616513072.064532:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.191@tcp1(10.31.138.191@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.064539:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.064545:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.064548:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.064550:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.064555:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513072.064562:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.064571:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513072.064574:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.064575:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.064581:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.064587:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ef68 00000800:00000200:26.0:1616513072.064591:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.064601:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.064619:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.064626:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513072.064628:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513072.064635:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513072.064639:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc198 00000800:00000200:0.0:1616513072.064643:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.064646:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.072331:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.072388:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.072397:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.072406:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.115@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.072416:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.115@tcp1 of length 224 into portal 26 MB=0x5f90cb9068600 00000400:00000200:28.0:1616513072.072423:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.115@tcp1 of length 224/224 into md 0xf97c47e9 [1] + 0 00000400:00000200:28.0:1616513072.072427:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1c28 00000400:00000200:28.0:1616513072.072430:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513072.072455:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:28.0:1616513072.072465:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513072.072465:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.072471:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1c28 00000800:00000200:28.0:1616513072.072474:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:12.0:1616513072.072474:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.115@tcp1 : PUT - for me 00000800:00000200:28.0:1616513072.072477:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513072.072479:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.072482:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513072.072483:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.115@tcp1 of length 224 into portal 12 MB=0x5f90cb9068680 00000400:00000200:12.0:1616513072.072490:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.115@tcp1 of length 224/224 into md 0x865b9f31 [1] + 53392 00000400:00000200:12.0:1616513072.072495:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.072498:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513072.072504:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.072507:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.072509:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513072.072538:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681207922689536 00010000:00000200:23.0:1616513072.072548:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1681207922689536/t0(0) o400->40fd59f3-2375-da21-55a8-23fb3f15750e@10.31.143.115@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513072.072564:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681207922689536, offset 192 00000400:00000200:23.0:1616513072.072569:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.115@tcp1 00000100:00000200:24.0:1616513072.072570:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681207922689664 00000400:00000200:23.0:1616513072.072578:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.115@tcp1 routed destination 00010000:00000200:24.0:1616513072.072583:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57e300 x1681207922689664/t0(0) o400->a1297c4d-9dd8-2167-4fcb-4112bb134af3@10.31.143.115@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:23.0:1616513072.072592:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.115@tcp1(10.31.143.115@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513072.072597:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681207922689664, offset 192 00000800:00000200:23.0:1616513072.072598:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513072.072603:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.115@tcp1 00000800:00000200:23.0:1616513072.072605:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513072.072607:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513072.072609:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513072.072611:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.115@tcp1 routed destination 00000800:00000200:23.0:1616513072.072614:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:23.0:1616513072.072621:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.072621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513072.072627:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.115@tcp1(10.31.143.115@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.072633:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.072639:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.072642:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.072644:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.072649:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.072653:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513072.072677:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.072684:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513072.072688:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.072697:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.072703:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e000 00000800:00000200:2.0:1616513072.072707:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513072.072708:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513072.072709:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513072.072715:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.072718:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513072.072719:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.072725:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.072729:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ba0 00000800:00000200:16.0:1616513072.072733:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.076009:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513072.076065:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.076074:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513072.076083:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.176@tcp1 : PUT - for me 00000400:00000200:0.0:1616513072.076092:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.176@tcp1 of length 224 into portal 12 MB=0x60184a098c300 00000400:00000200:0.0:1616513072.076098:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.176@tcp1 of length 224/224 into md 0x865b9f31 [1] + 53616 00000400:00000200:0.0:1616513072.076104:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.076106:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513072.076112:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.076115:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513072.076117:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.076120:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.076130:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513072.076139:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.176@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.076147:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.176@tcp1 of length 224 into portal 26 MB=0x60184a098be40 00000400:00000200:12.0:1616513072.076154:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.176@tcp1 of length 224/224 into md 0xf97c4809 [1] + 0 00000400:00000200:12.0:1616513072.076159:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0c38 00000400:00000200:12.0:1616513072.076162:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.076164:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513072.076169:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0c38 00000800:00000200:12.0:1616513072.076172:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.076175:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.076176:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:24.0:1616513072.076176:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519001940736 00000800:00000200:12.0:1616513072.076180:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:24.0:1616513072.076188:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d2880 x1690519001940736/t0(0) o400->022b9999-8934-5419-42e8-c3b483a9f3f3@10.31.129.176@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.076213:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690519001940736, offset 192 00000400:00000200:24.0:1616513072.076219:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.176@tcp1 00000400:00000200:24.0:1616513072.076226:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.176@tcp1 routed destination 00000100:00000200:23.0:1616513072.076234:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519001939520 00000400:00000200:24.0:1616513072.076241:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.176@tcp1(10.31.129.176@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:23.0:1616513072.076245:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1690519001939520/t0(0) o400->528a5d44-44f3-e928-5d9e-bad040e77898@10.31.129.176@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513072.076248:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513072.076254:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.076257:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:23.0:1616513072.076258:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690519001939520, offset 192 00000800:00000200:24.0:1616513072.076259:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513072.076264:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:23.0:1616513072.076264:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.176@tcp1 00000800:00000200:7.2:1616513072.076270:0:48608:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:23.0:1616513072.076272:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.176@tcp1 routed destination 00000800:00000200:30.0:1616513072.076282:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:23.0:1616513072.076286:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.176@tcp1(10.31.129.176@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513072.076287:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.076289:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:23.0:1616513072.076293:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:30.0:1616513072.076296:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.076299:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2a18 00000800:00000200:23.0:1616513072.076299:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513072.076301:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513072.076303:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:23.0:1616513072.076304:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513072.076306:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:23.0:1616513072.076308:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.076313:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:23.0:1616513072.076314:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513072.076378:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.076387:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.076390:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513072.076390:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.076397:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.076403:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eb28 00000800:00000200:16.0:1616513072.076406:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.077080:0:48608:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.077134:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.077144:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.077153:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.11@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.077162:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.11@tcp1 of length 224 into portal 26 MB=0x5f8f9e9811440 00000400:00000200:2.0:1616513072.077168:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.11@tcp1 of length 224/224 into md 0xf97c4829 [1] + 0 00000400:00000200:2.0:1616513072.077173:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0220 00000400:00000200:2.0:1616513072.077176:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.077179:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513072.077184:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0220 00000800:00000200:2.0:1616513072.077187:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.077188:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.077189:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.077195:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:23.0:1616513072.077198:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681127131649088 00000800:00000200:2.0:1616513072.077198:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.077199:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:23.0:1616513072.077204:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1681127131649088/t0(0) o400->667ab010-8498-6683-5189-57c554b31d03@10.31.132.11@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513072.077208:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.83@tcp1 : PUT - for me 00000100:00000200:23.0:1616513072.077215:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681127131649088, offset 192 00000400:00000200:14.0:1616513072.077217:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.83@tcp1 of length 224 into portal 26 MB=0x5ffd4139b6f80 00000400:00000200:23.0:1616513072.077220:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.11@tcp1 00000400:00000200:14.0:1616513072.077224:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.83@tcp1 of length 224/224 into md 0xf97c4849 [1] + 0 00000400:00000200:23.0:1616513072.077227:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.11@tcp1 routed destination 00000400:00000200:14.0:1616513072.077228:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d04c8 00000400:00000200:14.0:1616513072.077231:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.077234:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513072.077238:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d04c8 00000400:00000200:23.0:1616513072.077242:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.11@tcp1(10.31.132.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513072.077247:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513072.077247:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.077250:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.077252:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:23.0:1616513072.077253:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513072.077255:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:23.0:1616513072.077257:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:23.0:1616513072.077261:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513072.077262:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661210656640 00000800:00000200:7.2:1616513072.077265:0:48608:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:23.0:1616513072.077267:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:4.0:1616513072.077273:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688661210656640/t0(0) o400->94e52fc8-d6d8-6c1c-ed9a-c7b8cd529989@10.31.129.83@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513072.077278:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.077285:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513072.077287:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513072.077288:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661210656640, offset 192 00000400:00000200:26.0:1616513072.077289:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.077295:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:4.0:1616513072.077295:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.83@tcp1 00000400:00000200:26.0:1616513072.077302:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eaa0 00000400:00000200:4.0:1616513072.077303:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.83@tcp1 routed destination 00000800:00000200:26.0:1616513072.077305:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513072.077319:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.83@tcp1(10.31.129.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.077326:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.077332:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.077335:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.077337:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.077342:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.077347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513072.077349:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.077368:0:48608:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.077378:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.077383:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513072.077386:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513072.077391:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.83@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.077397:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.83@tcp1 of length 224 into portal 12 MB=0x5ffd4139b7380 00000400:00000200:2.0:1616513072.077402:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.83@tcp1 of length 224/224 into md 0xf96450b9 [1] + 20968 00000400:00000200:2.0:1616513072.077406:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.077409:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513072.077414:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513072.077415:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513072.077416:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.077417:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513072.077422:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.077424:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513072.077427:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513072.077431:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.077448:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbcb0 00000800:00000200:28.0:1616513072.077451:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513072.077498:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661210657664 00010000:00000200:18.0:1616513072.077509:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1389f80 x1688661210657664/t0(0) o400->00a945a1-759c-a503-e19f-5cd1ef61ead8@10.31.129.83@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.077523:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661210657664, offset 192 00000400:00000200:18.0:1616513072.077529:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.83@tcp1 00000400:00000200:18.0:1616513072.077536:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.83@tcp1 routed destination 00000400:00000200:18.0:1616513072.077549:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.83@tcp1(10.31.129.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.077555:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513072.077561:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.077564:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513072.077566:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513072.077570:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.077576:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.077584:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.077588:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.077589:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.077592:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513072.077594:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.077598:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd298 00000800:00000200:30.0:1616513072.077600:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.078984:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513072.079037:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.079046:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513072.079055:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.11@tcp1 : PUT - for me 00000800:00000200:11.2:1616513072.079057:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513072.079064:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.11@tcp1 of length 224 into portal 12 MB=0x5f8f9e9811460 00000400:00000200:14.0:1616513072.079070:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.11@tcp1 of length 224/224 into md 0xf96450b9 [1] + 21192 00000400:00000200:14.0:1616513072.079075:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.079078:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513072.079084:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.079087:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.079089:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.079091:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513072.079116:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.079125:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513072.079134:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.124@tcp1 : PUT - for me 00000400:00000200:0.0:1616513072.079144:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.124@tcp1 of length 224 into portal 26 MB=0x5ffd3a1705680 00000100:00000200:18.0:1616513072.079150:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681127131649120 00000400:00000200:0.0:1616513072.079151:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.124@tcp1 of length 224/224 into md 0xf97c4859 [1] + 0 00000400:00000200:0.0:1616513072.079156:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0000 00000400:00000200:0.0:1616513072.079159:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.079161:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00010000:00000200:18.0:1616513072.079162:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa0480 x1681127131649120/t0(0) o400->c0890a42-8dbf-9c53-ba22-978e9586d616@10.31.132.11@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513072.079166:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0000 00000800:00000200:0.0:1616513072.079169:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.079171:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.079172:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513072.079175:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513072.079177:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681127131649120, offset 192 00000400:00000200:18.0:1616513072.079184:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.11@tcp1 00000400:00000200:18.0:1616513072.079191:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.11@tcp1 routed destination 00000800:00000200:7.2:1616513072.079193:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:18.0:1616513072.079206:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.11@tcp1(10.31.132.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.079213:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.079219:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.079221:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.079225:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.079230:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.079235:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513072.079241:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659295229568 00000800:00000200:26.0:1616513072.079249:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:4.0:1616513072.079252:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688659295229568/t0(0) o400->a7684c8f-c002-320e-dcc8-6aab1f21f750@10.31.136.124@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:30.0:1616513072.079258:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.079259:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:4.0:1616513072.079266:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688659295229568, offset 192 00000400:00000200:26.0:1616513072.079267:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.124@tcp1 : PUT - for me 00000400:00000200:4.0:1616513072.079272:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.124@tcp1 00000400:00000200:26.0:1616513072.079276:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.124@tcp1 of length 224 into portal 12 MB=0x5ffd3a1705800 00000400:00000200:4.0:1616513072.079280:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.124@tcp1 routed destination 00000400:00000200:26.0:1616513072.079283:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.124@tcp1 of length 224/224 into md 0xf96450b9 [1] + 21416 00000800:00000200:12.0:1616513072.079287:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513072.079288:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.079291:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513072.079294:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513072.079295:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.124@tcp1(10.31.136.124@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513072.079296:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.079302:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513072.079303:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:4.0:1616513072.079303:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513072.079305:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.079307:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513072.079307:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd298 00000100:00000200:18.0:1616513072.079309:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659295229952 00000800:00000200:4.0:1616513072.079309:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.079311:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513072.079312:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.079314:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:4.0:1616513072.079314:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:18.0:1616513072.079315:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c692d00 x1688659295229952/t0(0) o400->a39265d0-faba-2945-f15b-d7bb8b502a1f@10.31.136.124@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:4.0:1616513072.079319:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.079324:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:18.0:1616513072.079326:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688659295229952, offset 192 00000100:00000200:4.0:1616513072.079326:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513072.079332:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.124@tcp1 00000800:00000200:14.0:1616513072.079333:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.079337:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513072.079338:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.124@tcp1 routed destination 00000400:00000200:14.0:1616513072.079339:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513072.079341:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513072.079345:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513072.079349:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbdc0 00000400:00000200:18.0:1616513072.079352:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.124@tcp1(10.31.136.124@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513072.079352:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513072.079358:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.079363:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.079365:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.079367:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.079370:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.079385:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.079395:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.079400:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.079403:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513072.079405:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.079409:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.079413:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc198 00000800:00000200:16.0:1616513072.079417:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.080803:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513072.080810:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.080816:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513072.080824:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.240@tcp1 : PUT - for me 00000400:00000200:30.0:1616513072.080844:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.240@tcp1 of length 224 into portal 26 MB=0x60182bc40abc0 00000400:00000200:30.0:1616513072.080849:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.240@tcp1 of length 224/224 into md 0xf97c4869 [1] + 0 00000400:00000200:30.0:1616513072.080853:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1188 00000400:00000200:30.0:1616513072.080856:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513072.080858:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513072.080861:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1188 00000800:00000200:30.0:1616513072.080863:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.080865:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513072.080867:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513072.080869:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513072.080925:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510875995072 00010000:00000200:4.0:1616513072.080936:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1690510875995072/t0(0) o400->b17954f8-7655-541a-9218-23fb972a9281@10.31.132.240@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513072.080951:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510875995072, offset 192 00000400:00000200:4.0:1616513072.080957:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.240@tcp1 00000400:00000200:4.0:1616513072.080965:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.240@tcp1 routed destination 00000400:00000200:4.0:1616513072.080980:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.240@tcp1(10.31.132.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.080987:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513072.080992:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.080995:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513072.080997:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513072.081002:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.081007:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513072.081009:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513072.081058:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.081064:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513072.081067:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.081073:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.081080:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baff0 00000800:00000200:2.0:1616513072.081083:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.081086:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.081892:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513072.081900:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.081906:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513072.081910:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513072.081913:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.240@tcp1 : PUT - for me 00000400:00000200:30.0:1616513072.081919:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.240@tcp1 of length 224 into portal 12 MB=0x60182bc40ac40 00000400:00000200:30.0:1616513072.081924:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.240@tcp1 of length 224/224 into md 0xf96450b9 [1] + 21640 00000400:00000200:30.0:1616513072.081928:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513072.081930:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513072.081934:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.081936:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513072.081938:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.082001:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510875995200 00010000:00000200:18.0:1616513072.082014:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a78900 x1690510875995200/t0(0) o400->9124b41b-2380-60e6-8644-a5faaf2769e7@10.31.132.240@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.082028:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510875995200, offset 192 00000400:00000200:18.0:1616513072.082034:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.240@tcp1 00000400:00000200:18.0:1616513072.082041:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.240@tcp1 routed destination 00000400:00000200:18.0:1616513072.082056:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.240@tcp1(10.31.132.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.082063:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.082068:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.082071:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.082073:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.082078:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.082121:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.082176:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.082183:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.082187:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513072.082194:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.082198:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc880 00000800:00000200:12.0:1616513072.082202:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.082205:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.127481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513072.127539:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.127550:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513072.127559:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.110@tcp1 : PUT - for me 00000400:00000200:0.0:1616513072.127570:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.110@tcp1 of length 224 into portal 26 MB=0x6059813e2c1c0 00000400:00000200:0.0:1616513072.127577:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.110@tcp1 of length 224/224 into md 0xf97c4889 [1] + 0 00000400:00000200:0.0:1616513072.127582:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0ee0 00000400:00000200:0.0:1616513072.127586:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.127589:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513072.127594:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0ee0 00000800:00000200:16.0:1616513072.127595:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.127597:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.127600:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.127601:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513072.127660:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695000587059648 00010000:00000200:4.0:1616513072.127673:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1695000587059648/t0(0) o400->621fc4b6-ebf6-80af-0a8e-e066bd337684@10.31.136.110@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513072.127688:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695000587059648, offset 192 00000400:00000200:4.0:1616513072.127695:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.110@tcp1 00000400:00000200:4.0:1616513072.127703:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.110@tcp1 routed destination 00000400:00000200:4.0:1616513072.127718:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.110@tcp1(10.31.136.110@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.127725:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513072.127732:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.127735:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513072.127739:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513072.127744:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513072.127751:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.127752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.127810:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.127817:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513072.127820:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.127827:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.127833:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbcb0 00000800:00000200:2.0:1616513072.127836:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.127839:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.128167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.128223:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.128233:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513072.128234:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.128242:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.110@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.128251:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.110@tcp1 of length 224 into portal 12 MB=0x6059813e2c300 00000400:00000200:26.0:1616513072.128258:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.110@tcp1 of length 224/224 into md 0xf96450b9 [1] + 21864 00000400:00000200:26.0:1616513072.128263:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.128266:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513072.128273:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.128276:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.128278:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.128341:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695000587059968 00010000:00000200:18.0:1616513072.128354:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c694800 x1695000587059968/t0(0) o400->082c53dc-557a-c769-a907-60c3379b2744@10.31.136.110@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.128368:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695000587059968, offset 192 00000400:00000200:18.0:1616513072.128377:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.110@tcp1 00000400:00000200:18.0:1616513072.128384:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.110@tcp1 routed destination 00000400:00000200:18.0:1616513072.128400:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.110@tcp1(10.31.136.110@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.128406:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.128412:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.128415:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.128417:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.128422:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.128427:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.128485:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.128499:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.128501:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513072.128507:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.128511:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cddc0 00000800:00000200:12.0:1616513072.128515:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.128517:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.140394:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513072.140452:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.140462:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.140471:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.140479:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x60399133bb140 00000400:00000200:16.0:1616513072.140487:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0x865b9f31 [1] + 53840 00000400:00000200:16.0:1616513072.140492:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.140495:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513072.140502:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.140505:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513072.140505:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513072.140510:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.140567:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805847822656 00010000:00000200:24.0:1616513072.140642:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e2d00 x1692805847822656/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:698/0 lens 336/336 e 0 to 0 dl 1616513078 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513072.140656:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805847822656, offset 192 00000400:00000200:24.0:1616513072.140663:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:24.0:1616513072.140671:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:24.0:1616513072.140685:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.140691:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513072.140697:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.140700:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513072.140702:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513072.140707:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.140714:0:48615:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.140771:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.140778:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513072.140781:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.140787:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.140792:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ed0 00000800:00000200:2.0:1616513072.140796:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.140798:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.168701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513072.168713:0:48618:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.168717:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.168726:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513072.168734:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.186@tcp1 : PUT - for me 00000800:00000200:30.0:1616513072.168738:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513072.168744:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.186@tcp1 of length 224 into portal 12 MB=0x60183627e6540 00000800:00000200:30.0:1616513072.168746:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513072.168750:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.186@tcp1 of length 224/224 into md 0x865b9f31 [1] + 54176 00000400:00000200:30.0:1616513072.168754:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.186@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.168756:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.168758:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:30.0:1616513072.168763:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.186@tcp1 of length 224 into portal 26 MB=0x60183627e6400 00000800:00000200:12.0:1616513072.168765:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:30.0:1616513072.168768:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.186@tcp1 of length 224/224 into md 0xf97c48a9 [1] + 0 00000800:00000200:12.0:1616513072.168768:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.168769:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.168772:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513072.168773:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0ff0 00000400:00000200:30.0:1616513072.168776:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513072.168779:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513072.168782:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0ff0 00000800:00000200:30.0:1616513072.168785:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.168787:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513072.168789:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513072.168792:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513072.168831:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513665058112 00010000:00000200:24.0:1616513072.168843:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc579f80 x1690513665058112/t0(0) o400->e36eae56-f618-2a21-d8fc-4b955624683f@10.31.132.186@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513072.168849:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513665057792 00000100:00000200:24.0:1616513072.168858:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513665058112, offset 192 00010000:00000200:4.0:1616513072.168860:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1690513665057792/t0(0) o400->68d64690-31bc-8c15-7da1-934ad02a3d7e@10.31.132.186@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:24.0:1616513072.168864:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.186@tcp1 00000400:00000200:24.0:1616513072.168871:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.186@tcp1 routed destination 00000100:00000200:4.0:1616513072.168875:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513665057792, offset 192 00000400:00000200:4.0:1616513072.168882:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.186@tcp1 00000400:00000200:24.0:1616513072.168886:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.186@tcp1(10.31.132.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.168894:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:4.0:1616513072.168897:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.186@tcp1 routed destination 00000800:00000200:24.0:1616513072.168904:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.168907:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.168910:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:4.0:1616513072.168912:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.186@tcp1(10.31.132.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.168914:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513072.168919:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513072.168925:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.168927:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513072.168930:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.168930:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513072.168934:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.168940:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513072.168941:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513072.168985:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.168992:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513072.168995:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513072.169001:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513072.169003:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513072.169006:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:0.0:1616513072.169009:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513072.169010:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513072.169012:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513072.169013:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.169020:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.169025:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb540 00000800:00000200:2.0:1616513072.169029:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.169032:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.174982:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.175035:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.175045:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513072.175046:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.175054:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.89@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.175064:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.89@tcp1 of length 224 into portal 26 MB=0x601aa4ec200c0 00000400:00000200:26.0:1616513072.175070:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.89@tcp1 of length 224/224 into md 0xf97c48c9 [1] + 0 00000400:00000200:26.0:1616513072.175075:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1ba0 00000400:00000200:26.0:1616513072.175077:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.175080:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513072.175084:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1ba0 00000800:00000200:26.0:1616513072.175087:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.175090:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.175092:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513072.175148:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690680837669056 00010000:00000200:4.0:1616513072.175158:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1690680837669056/t0(0) o400->c6997ea3-6336-a326-3fff-c676f65a49cf@10.31.140.89@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513072.175173:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690680837669056, offset 192 00000400:00000200:4.0:1616513072.175179:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.89@tcp1 00000400:00000200:4.0:1616513072.175187:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.89@tcp1 routed destination 00000400:00000200:4.0:1616513072.175202:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.89@tcp1(10.31.140.89@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.175208:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.175214:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.175217:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.175219:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.175223:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:4.0:1616513072.175230:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513072.175269:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.175324:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.175331:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.175334:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513072.175340:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.175346:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba198 00000800:00000200:12.0:1616513072.175349:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.175352:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.175634:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.175690:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.175700:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.175708:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.89@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.175717:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.89@tcp1 of length 224 into portal 12 MB=0x601aa4ec20440 00000400:00000200:2.0:1616513072.175723:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.89@tcp1 of length 224/224 into md 0xf96450b9 [1] + 22088 00000400:00000200:2.0:1616513072.175728:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.175731:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513072.175738:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.175741:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.175743:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.175748:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.175802:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690680837669952 00010000:00000200:18.0:1616513072.175814:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138a400 x1690680837669952/t0(0) o400->ac01f08c-6db5-f6a7-2d6d-4add6f3eb909@10.31.140.89@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.175829:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690680837669952, offset 192 00000400:00000200:18.0:1616513072.175835:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.89@tcp1 00000400:00000200:18.0:1616513072.175843:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.89@tcp1 routed destination 00000400:00000200:18.0:1616513072.175857:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.89@tcp1(10.31.140.89@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.175864:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513072.175869:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.175872:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513072.175874:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513072.175879:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.175884:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.175892:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.175896:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.175898:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.175901:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513072.175904:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.175907:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd650 00000800:00000200:30.0:1616513072.175911:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.180231:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513072.180285:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.180294:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513072.180304:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.118@tcp1 : PUT - for me 00000400:00000200:14.0:1616513072.180314:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.118@tcp1 of length 224 into portal 12 MB=0x603da55080540 00000400:00000200:14.0:1616513072.180323:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.118@tcp1 of length 224/224 into md 0xf96450b9 [1] + 22312 00000400:00000200:14.0:1616513072.180328:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.180331:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513072.180338:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.180339:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.180341:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.180344:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.180399:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513072.180403:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693086124606784 00010000:00000200:18.0:1616513072.180416:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa6780 x1693086124606784/t0(0) o400->373f65c7-7906-6ac9-207a-7d37957c485c@10.31.141.118@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.180430:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693086124606784, offset 192 00000400:00000200:18.0:1616513072.180449:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.118@tcp1 00000400:00000200:18.0:1616513072.180457:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.118@tcp1 routed destination 00000800:00000200:0.0:1616513072.180468:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513072.180472:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.118@tcp1(10.31.141.118@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513072.180477:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.180479:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.180485:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:0.0:1616513072.180487:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.118@tcp1 : PUT - for me 00000800:00000200:18.0:1616513072.180492:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513072.180494:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:0.0:1616513072.180496:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.118@tcp1 of length 224 into portal 26 MB=0x603da550801c0 00000800:00000200:18.0:1616513072.180499:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:0.0:1616513072.180503:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.118@tcp1 of length 224/224 into md 0xf97c48d9 [1] + 0 00000400:00000200:0.0:1616513072.180507:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0770 00000400:00000200:0.0:1616513072.180510:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.180513:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513072.180517:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0770 00000800:00000200:0.0:1616513072.180520:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513072.180523:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513072.180523:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513072.180530:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513072.180531:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513072.180532:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.180534:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513072.180540:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.180544:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdd38 00000800:00000200:16.0:1616513072.180548:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513072.180580:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693086124605888 00010000:00000200:4.0:1616513072.180591:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1693086124605888/t0(0) o400->f259b593-17ee-1349-17cf-722422c520c2@10.31.141.118@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513072.180607:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693086124605888, offset 192 00000400:00000200:4.0:1616513072.180613:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.118@tcp1 00000400:00000200:4.0:1616513072.180620:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.118@tcp1 routed destination 00000400:00000200:4.0:1616513072.180636:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.118@tcp1(10.31.141.118@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.180642:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513072.180648:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.180651:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513072.180653:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513072.180658:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.180663:0:48619:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513072.180665:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513072.180717:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.180725:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513072.180727:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.180728:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.180734:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.180740:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baee0 00000800:00000200:26.0:1616513072.180743:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.219818:0:48623:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.219873:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.219883:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.219892:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.186@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.219901:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.186@tcp1 of length 224 into portal 26 MB=0x60182956bbd80 00000400:00000200:2.0:1616513072.219908:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.186@tcp1 of length 224/224 into md 0xf97c4909 [1] + 0 00000400:00000200:2.0:1616513072.219913:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1100 00000400:00000200:2.0:1616513072.219916:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.219919:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513072.219924:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1100 00000800:00000200:14.0:1616513072.219927:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.219927:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.219930:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.219932:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513072.219992:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510224506240 00010000:00000200:4.0:1616513072.220003:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1690510224506240/t0(0) o400->3d5f0cac-774f-e2e2-18b7-2ecdcf69aef6@10.31.140.186@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513072.220018:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510224506240, offset 192 00000400:00000200:4.0:1616513072.220024:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.186@tcp1 00000400:00000200:4.0:1616513072.220032:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.186@tcp1 routed destination 00000400:00000200:4.0:1616513072.220048:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.186@tcp1(10.31.140.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.220054:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.220060:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.220063:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.220065:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.220070:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.220077:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513072.220077:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513072.220134:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.220142:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.220145:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.220151:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.220159:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bad48 00000800:00000200:28.0:1616513072.220162:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.220165:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.220426:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513072.220482:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.220492:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.220501:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.186@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.220509:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.186@tcp1 of length 224 into portal 12 MB=0x60182956bc200 00000400:00000200:16.0:1616513072.220516:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.186@tcp1 of length 224/224 into md 0x865b9f31 [1] + 54400 00000400:00000200:16.0:1616513072.220521:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.220524:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513072.220530:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.220533:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513072.220535:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.220538:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.220610:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510224507392 00010000:00000200:24.0:1616513072.220623:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e0900 x1690510224507392/t0(0) o400->17b2e9ef-fbc9-83bd-95fb-d02193658d78@10.31.140.186@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.220637:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510224507392, offset 192 00000400:00000200:24.0:1616513072.220643:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.186@tcp1 00000400:00000200:24.0:1616513072.220651:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.186@tcp1 routed destination 00000400:00000200:24.0:1616513072.220666:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.186@tcp1(10.31.140.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.220672:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513072.220679:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.220681:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513072.220683:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513072.220688:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.220694:0:48623:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.220705:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.220710:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.220712:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513072.220717:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.220721:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf37e8 00000800:00000200:30.0:1616513072.220724:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513072.220726:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.229318:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.229373:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.229382:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.229391:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.247@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.229401:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.247@tcp1 of length 224 into portal 26 MB=0x604192fcd4080 00000400:00000200:28.0:1616513072.229408:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.247@tcp1 of length 224/224 into md 0xf97c4919 [1] + 0 00000400:00000200:28.0:1616513072.229412:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d03b8 00000400:00000200:28.0:1616513072.229415:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.229418:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513072.229422:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d03b8 00000800:00000200:28.0:1616513072.229425:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.229426:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.229428:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.229430:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513072.229483:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356082937984 00010000:00000200:4.0:1616513072.229494:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1693356082937984/t0(0) o400->2285178c-787d-4979-2bc3-199ab9742dcc@10.31.128.247@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513072.229509:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693356082937984, offset 192 00000400:00000200:4.0:1616513072.229515:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.247@tcp1 00000400:00000200:4.0:1616513072.229523:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.247@tcp1 routed destination 00000400:00000200:4.0:1616513072.229540:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.247@tcp1(10.31.128.247@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.229546:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.229552:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.229554:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.229557:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.229561:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.229567:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513072.229568:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513072.229620:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.229627:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.229630:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513072.229632:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.229637:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:11.2:1616513072.229637:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513072.229642:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00000800:00000200:16.0:1616513072.229646:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.229649:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.229655:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513072.229658:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.229662:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.247@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.229671:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.247@tcp1 of length 224 into portal 12 MB=0x604192fcd4440 00000400:00000200:16.0:1616513072.229677:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.247@tcp1 of length 224/224 into md 0x865b9f31 [1] + 54624 00000400:00000200:16.0:1616513072.229684:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.229686:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513072.229692:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.229695:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.229696:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.229755:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356082938944 00010000:00000200:24.0:1616513072.229767:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e6780 x1693356082938944/t0(0) o400->23d3c885-84e9-c914-1e9b-db50f844a0a4@10.31.128.247@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.229782:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693356082938944, offset 192 00000400:00000200:24.0:1616513072.229788:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.247@tcp1 00000400:00000200:24.0:1616513072.229796:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.247@tcp1 routed destination 00000400:00000200:24.0:1616513072.229811:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.247@tcp1(10.31.128.247@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.229817:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513072.229823:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.229826:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513072.229828:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513072.229833:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.229839:0:48624:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.229894:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.229901:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513072.229904:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513072.229907:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513072.229911:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513072.229915:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2550 00000800:00000200:14.0:1616513072.229919:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.254762:0:48627:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513072.254775:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.254783:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513072.254791:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.13@tcp1 : PUT - for me 00000400:00000200:30.0:1616513072.254800:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.13@tcp1 of length 224 into portal 26 MB=0x603d1734dab80 00000400:00000200:30.0:1616513072.254805:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.21.13@tcp1 of length 224/224 into md 0xf97c4939 [1] + 0 00000800:00000200:11.2:1616513072.254808:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:30.0:1616513072.254810:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1760 00000400:00000200:30.0:1616513072.254812:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513072.254814:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513072.254818:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1760 00000800:00000200:30.0:1616513072.254821:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.254823:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513072.254827:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513072.254831:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513072.254860:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.254869:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513072.254878:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.13@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.254887:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.13@tcp1 of length 224 into portal 12 MB=0x603d1734daf40 00000400:00000200:12.0:1616513072.254893:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.21.13@tcp1 of length 224/224 into md 0x865b9f31 [1] + 54848 00000100:00000200:4.0:1616513072.254895:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693047977782144 00000400:00000200:12.0:1616513072.254897:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.254899:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513072.254905:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:4.0:1616513072.254906:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1693047977782144/t0(0) o400->7bc5c3f4-4bae-ebcf-b7a3-7c476c7827b7@10.31.21.13@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513072.254908:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.254909:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.254912:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513072.254921:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693047977782144, offset 192 00000400:00000200:4.0:1616513072.254927:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.13@tcp1 00000400:00000200:4.0:1616513072.254935:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.13@tcp1 routed destination 00000400:00000200:4.0:1616513072.254950:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.13@tcp1(10.31.21.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.254957:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.254963:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.254965:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.254968:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513072.254970:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693047977783104 00000800:00000200:4.0:1616513072.254973:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.254977:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513072.254979:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:24.0:1616513072.254982:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57b600 x1693047977783104/t0(0) o400->6666dc47-ae5c-9379-ac2e-22b416a942d2@10.31.21.13@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.254997:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693047977783104, offset 192 00000400:00000200:24.0:1616513072.255003:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.13@tcp1 00000400:00000200:24.0:1616513072.255010:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.13@tcp1 routed destination 00000400:00000200:24.0:1616513072.255026:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.13@tcp1(10.31.21.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.255032:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513072.255035:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513072.255038:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513072.255041:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513072.255041:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513072.255043:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513072.255044:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513072.255048:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513072.255051:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513072.255052:0:48627:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513072.255056:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba2a8 00000800:00000200:28.0:1616513072.255060:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.255063:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513072.255063:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.255067:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513072.255069:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.255075:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.255079:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3870 00000800:00000200:2.0:1616513072.255082:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.255084:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.256028:0:48627:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.256084:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.256093:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513072.256095:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.256102:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.151@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.256112:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.151@tcp1 of length 224 into portal 26 MB=0x602169647c380 00000400:00000200:26.0:1616513072.256119:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.151@tcp1 of length 224/224 into md 0xf97c4949 [1] + 0 00000400:00000200:26.0:1616513072.256124:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1298 00000400:00000200:26.0:1616513072.256126:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.256129:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513072.256133:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1298 00000800:00000200:26.0:1616513072.256136:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.256139:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.256141:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513072.256201:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145894085504 00010000:00000200:4.0:1616513072.256212:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1691145894085504/t0(0) o400->69467242-871c-763a-34d3-a314f482e108@10.31.140.151@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513072.256226:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691145894085504, offset 192 00000400:00000200:4.0:1616513072.256233:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.151@tcp1 00000400:00000200:4.0:1616513072.256240:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.151@tcp1 routed destination 00000400:00000200:4.0:1616513072.256255:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.151@tcp1(10.31.140.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.256262:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.256268:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.256272:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.256274:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.256279:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.256284:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513072.256286:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513072.256337:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.256344:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513072.256346:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513072.256353:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513072.256359:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb7e8 00000800:00000200:0.0:1616513072.256362:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.256365:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.256570:0:48627:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.256624:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.256633:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.256642:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.151@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.256650:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.151@tcp1 of length 224 into portal 12 MB=0x602169647c800 00000400:00000200:2.0:1616513072.256657:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.151@tcp1 of length 224/224 into md 0xf96450b9 [1] + 22536 00000400:00000200:2.0:1616513072.256663:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.256665:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513072.256673:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.256676:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.256677:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.256682:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.256738:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145894086656 00010000:00000200:18.0:1616513072.256750:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138a880 x1691145894086656/t0(0) o400->efb42e0c-79c4-7d89-33a3-25e7c2d6b41f@10.31.140.151@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.256764:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691145894086656, offset 192 00000400:00000200:18.0:1616513072.256771:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.151@tcp1 00000400:00000200:18.0:1616513072.256778:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.151@tcp1 routed destination 00000400:00000200:18.0:1616513072.256793:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.151@tcp1(10.31.140.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.256800:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513072.256805:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.256808:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513072.256810:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513072.256815:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.256819:0:48627:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.256829:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.256834:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.256836:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.256838:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513072.256842:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.256845:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc990 00000800:00000200:30.0:1616513072.256848:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.273678:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.273733:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.273743:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.273752:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.20@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.273761:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.20@tcp1 of length 224 into portal 26 MB=0x5fff3420a4c80 00000400:00000200:28.0:1616513072.273768:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.20@tcp1 of length 224/224 into md 0xf97c4969 [1] + 0 00000400:00000200:28.0:1616513072.273773:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1e48 00000400:00000200:28.0:1616513072.273775:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.273778:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513072.273782:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1e48 00000800:00000200:28.0:1616513072.273785:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.273787:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.273790:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.273792:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513072.273845:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795133660288 00010000:00000200:4.0:1616513072.273856:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688795133660288/t0(0) o400->d3a705de-084b-d7da-01ec-50cf8735f666@10.31.136.20@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513072.273870:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688795133660288, offset 192 00000400:00000200:4.0:1616513072.273876:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.20@tcp1 00000400:00000200:4.0:1616513072.273884:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.20@tcp1 routed destination 00000400:00000200:4.0:1616513072.273899:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.20@tcp1(10.31.136.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.273905:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.273911:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.273913:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.273915:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.273920:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.273925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513072.273927:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513072.273979:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.273986:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513072.273989:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513072.273996:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513072.274001:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbb18 00000800:00000200:0.0:1616513072.274005:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.274008:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.274014:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513072.274016:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513072.274022:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.20@tcp1 : PUT - for me 00000400:00000200:0.0:1616513072.274030:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.20@tcp1 of length 224 into portal 12 MB=0x5fff3420a4dc0 00000400:00000200:0.0:1616513072.274036:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.20@tcp1 of length 224/224 into md 0x865b9f31 [1] + 55072 00000400:00000200:0.0:1616513072.274041:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.274044:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513072.274050:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.274052:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.274054:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.274114:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795133660608 00010000:00000200:24.0:1616513072.274126:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3f00 x1688795133660608/t0(0) o400->7b1378de-43a3-7cf3-bdb2-b3d902a7bacb@10.31.136.20@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.274140:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688795133660608, offset 192 00000400:00000200:24.0:1616513072.274147:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.20@tcp1 00000400:00000200:24.0:1616513072.274154:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.20@tcp1 routed destination 00000400:00000200:24.0:1616513072.274169:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.20@tcp1(10.31.136.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.274176:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513072.274182:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.274185:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513072.274187:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513072.274192:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.274200:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.274256:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.274263:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513072.274266:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513072.274272:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513072.274276:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:14.0:1616513072.274280:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.274283:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.285960:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.286016:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.286026:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513072.286027:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.286035:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.33@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.286045:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.33@tcp1 of length 224 into portal 26 MB=0x5ffe74acd0f80 00000400:00000200:26.0:1616513072.286051:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.33@tcp1 of length 224/224 into md 0xf97c4999 [1] + 0 00000400:00000200:26.0:1616513072.286056:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1f58 00000400:00000200:26.0:1616513072.286059:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.286062:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513072.286066:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1f58 00000800:00000200:26.0:1616513072.286069:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.286072:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.286074:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513072.286129:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688743741034368 00010000:00000200:4.0:1616513072.286139:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688743741034368/t0(0) o400->a59fa261-dea7-3adb-35b7-5b37c41eb2a4@10.31.142.33@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513072.286152:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688743741034368, offset 192 00000400:00000200:4.0:1616513072.286158:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.33@tcp1 00000400:00000200:4.0:1616513072.286165:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.33@tcp1 routed destination 00000400:00000200:4.0:1616513072.286179:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.33@tcp1(10.31.142.33@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.286184:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.286190:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.286192:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.286194:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.286199:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.286204:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513072.286205:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513072.286257:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.286264:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.286267:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.286274:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.286279:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb298 00000800:00000200:28.0:1616513072.286283:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.286286:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.286545:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513072.286598:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.286608:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513072.286616:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.33@tcp1 : PUT - for me 00000400:00000200:14.0:1616513072.286625:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.33@tcp1 of length 224 into portal 12 MB=0x5ffe74acd1940 00000400:00000200:14.0:1616513072.286631:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.33@tcp1 of length 224/224 into md 0xf96450b9 [1] + 22760 00000400:00000200:14.0:1616513072.286637:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.286639:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513072.286646:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.286649:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.286651:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.286654:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.286713:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688743741036864 00010000:00000200:18.0:1616513072.286725:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa7500 x1688743741036864/t0(0) o400->2147a5cd-7f49-a583-1572-a9893497543d@10.31.142.33@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.286739:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688743741036864, offset 192 00000400:00000200:18.0:1616513072.286746:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.33@tcp1 00000400:00000200:18.0:1616513072.286754:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.33@tcp1 routed destination 00000400:00000200:18.0:1616513072.286768:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.33@tcp1(10.31.142.33@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.286774:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513072.286781:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.286783:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513072.286785:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513072.286790:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.286796:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.286804:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.286807:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.286809:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.286813:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513072.286814:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.286818:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccc38 00000800:00000200:30.0:1616513072.286821:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.305125:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513072.305182:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.305192:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513072.305204:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.223@tcp1 : PUT - for me 00000400:00000200:14.0:1616513072.305213:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.223@tcp1 of length 224 into portal 26 MB=0x5ffe3951fd8c0 00000400:00000200:14.0:1616513072.305220:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.223@tcp1 of length 224/224 into md 0xf97c49b9 [1] + 0 00000400:00000200:14.0:1616513072.305225:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1430 00000400:00000200:14.0:1616513072.305228:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.305230:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513072.305235:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1430 00000800:00000200:14.0:1616513072.305238:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.305238:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.305241:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.305245:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.305248:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.305249:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:4.0:1616513072.305255:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727808104640 00000400:00000200:2.0:1616513072.305257:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.223@tcp1 : PUT - for me 00010000:00000200:4.0:1616513072.305265:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688727808104640/t0(0) o400->61cbcd09-817f-375f-70c3-9f4f9179769c@10.31.141.223@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513072.305266:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.223@tcp1 of length 224 into portal 12 MB=0x5ffe3951fd940 00000400:00000200:2.0:1616513072.305273:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.223@tcp1 of length 224/224 into md 0xf96450b9 [1] + 22984 00000100:00000200:4.0:1616513072.305278:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688727808104640, offset 192 00000400:00000200:2.0:1616513072.305278:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.305281:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:4.0:1616513072.305285:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.223@tcp1 00000800:00000200:2.0:1616513072.305288:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.305291:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513072.305292:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.223@tcp1 routed destination 00000800:00000200:2.0:1616513072.305293:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513072.305306:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.223@tcp1(10.31.141.223@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.305312:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.305318:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.305320:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.305322:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.305327:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:4.0:1616513072.305333:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513072.305335:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:18.0:1616513072.305353:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727808104768 00010000:00000200:18.0:1616513072.305365:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138ad00 x1688727808104768/t0(0) o400->16cb6aad-d5d4-907f-8bf8-0c3c88f569a6@10.31.141.223@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.305390:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688727808104768, offset 192 00000800:00000200:12.0:1616513072.305390:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513072.305397:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.223@tcp1 00000800:00000200:12.0:1616513072.305397:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.305400:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513072.305404:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.223@tcp1 routed destination 00000400:00000200:12.0:1616513072.305407:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.305412:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bacc0 00000800:00000200:12.0:1616513072.305416:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513072.305419:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.223@tcp1(10.31.141.223@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.305419:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513072.305425:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513072.305431:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.305434:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513072.305451:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513072.305456:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.305460:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.305470:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.305474:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.305476:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513072.305479:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513072.305481:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.305484:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccaa0 00000800:00000200:30.0:1616513072.305487:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.322030:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.322085:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.322095:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513072.322098:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.322104:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.125@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.322114:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.125@tcp1 of length 224 into portal 26 MB=0x60198fea6f700 00000400:00000200:26.0:1616513072.322121:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.125@tcp1 of length 224/224 into md 0xf97c49c9 [1] + 0 00000400:00000200:26.0:1616513072.322126:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0e58 00000400:00000200:26.0:1616513072.322128:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.322131:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513072.322136:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0e58 00000800:00000200:26.0:1616513072.322139:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.322142:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.322144:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513072.322158:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690606479275776 00010000:00000200:8.0:1616513072.322169:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1690606479275776/t0(0) o400->e0121669-bad6-5f46-dc73-c8053b65119c@10.31.140.125@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.322183:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690606479275776, offset 192 00000400:00000200:8.0:1616513072.322189:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.125@tcp1 00000400:00000200:8.0:1616513072.322198:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.125@tcp1 routed destination 00000400:00000200:8.0:1616513072.322213:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.125@tcp1(10.31.140.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.322220:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.322225:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.322228:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.322231:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.322236:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.322243:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513072.322243:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.322253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513072.322299:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.322306:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.322309:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.322315:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513072.322318:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513072.322321:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70550 00000800:00000200:28.0:1616513072.322325:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.322328:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513072.322328:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513072.322329:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513072.322337:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.125@tcp1 : PUT - for me 00000400:00000200:14.0:1616513072.322345:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.125@tcp1 of length 224 into portal 12 MB=0x60198fea6fd00 00000400:00000200:14.0:1616513072.322352:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.125@tcp1 of length 224/224 into md 0xf96450b9 [1] + 23208 00000400:00000200:14.0:1616513072.322357:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.322360:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513072.322367:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.322370:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.322372:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.322450:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690606479277312 00010000:00000200:18.0:1616513072.322462:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa1b00 x1690606479277312/t0(0) o400->e931ae3b-3a3a-e910-bc7f-7439b90eb9bb@10.31.140.125@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.322477:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690606479277312, offset 192 00000400:00000200:18.0:1616513072.322483:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.125@tcp1 00000400:00000200:18.0:1616513072.322491:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.125@tcp1 routed destination 00000400:00000200:18.0:1616513072.322505:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.125@tcp1(10.31.140.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.322512:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513072.322518:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.322520:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513072.322522:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513072.322527:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.322532:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.322541:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.322546:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513072.322549:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.322551:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513072.322555:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.322559:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdba0 00000800:00000200:2.0:1616513072.322563:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.342950:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513072.343005:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.343015:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513072.343024:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.22@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.343035:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.22@tcp1 of length 224 into portal 26 MB=0x5ffd5757b1e80 00000400:00000200:12.0:1616513072.343042:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.22@tcp1 of length 224/224 into md 0xf97c49e9 [1] + 0 00000400:00000200:12.0:1616513072.343046:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0f68 00000400:00000200:12.0:1616513072.343049:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.343052:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513072.343056:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0f68 00000800:00000200:12.0:1616513072.343059:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.343059:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.343062:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.343066:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.343069:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.343070:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:8.0:1616513072.343075:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667147673216 00000400:00000200:0.0:1616513072.343079:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.22@tcp1 : PUT - for me 00010000:00000200:8.0:1616513072.343084:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688667147673216/t0(0) o400->20afae92-f3b3-2aba-0e5e-71d15f6fd137@10.31.136.22@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513072.343089:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.22@tcp1 of length 224 into portal 12 MB=0x5ffd5757b2000 00000400:00000200:0.0:1616513072.343095:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.22@tcp1 of length 224/224 into md 0x865b9f31 [1] + 55296 00000100:00000200:8.0:1616513072.343100:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667147673216, offset 192 00000400:00000200:0.0:1616513072.343100:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.343103:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513072.343110:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513072.343111:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.22@tcp1 00000800:00000200:0.0:1616513072.343113:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513072.343118:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.22@tcp1 routed destination 00000800:00000200:0.0:1616513072.343118:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513072.343132:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.22@tcp1(10.31.136.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.343138:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.343143:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.343146:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.343147:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.343152:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.343157:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513072.343158:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513072.343176:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667147673600 00010000:00000200:24.0:1616513072.343188:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d2d00 x1688667147673600/t0(0) o400->dccf286d-11b8-30b9-e7e5-11eaefb3d159@10.31.136.22@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.343202:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667147673600, offset 192 00000400:00000200:24.0:1616513072.343209:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.22@tcp1 00000400:00000200:24.0:1616513072.343217:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.22@tcp1 routed destination 00000800:00000200:16.0:1616513072.343230:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513072.343234:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.22@tcp1(10.31.136.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513072.343237:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513072.343240:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513072.343240:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513072.343241:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513072.343246:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513072.343247:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513072.343249:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513072.343251:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513072.343252:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71cb0 00000800:00000200:24.0:1616513072.343256:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513072.343256:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.343264:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.343276:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.343280:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.343282:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513072.343288:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.343292:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2d48 00000800:00000200:30.0:1616513072.343295:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513072.343298:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.373601:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.373657:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.373668:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.373678:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.179@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.373689:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.179@tcp1 of length 224 into portal 26 MB=0x5ffdf3334d800 00000400:00000200:28.0:1616513072.373696:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.179@tcp1 of length 224/224 into md 0xf97c49f9 [1] + 0 00000400:00000200:28.0:1616513072.373701:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1980 00000400:00000200:28.0:1616513072.373704:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.373707:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513072.373711:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1980 00000800:00000200:0.0:1616513072.373713:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.373715:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.373724:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.373726:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513072.373742:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708985444352 00010000:00000200:8.0:1616513072.373751:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688708985444352/t0(0) o400->c313ecb4-56f4-f9e4-8205-59d856497292@10.31.132.179@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.373764:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688708985444352, offset 192 00000400:00000200:8.0:1616513072.373769:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.179@tcp1 00000400:00000200:8.0:1616513072.373776:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.179@tcp1 routed destination 00000400:00000200:8.0:1616513072.373789:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.179@tcp1(10.31.132.179@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.373795:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.373800:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.373802:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.373804:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.373809:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.373814:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513072.373814:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513072.373826:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.373833:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.373836:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513072.373844:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.373850:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70d48 00000800:00000200:12.0:1616513072.373853:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.373856:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.373929:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513072.373941:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.373949:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513072.373951:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513072.373958:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.179@tcp1 : PUT - for me 00000400:00000200:0.0:1616513072.373967:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.179@tcp1 of length 224 into portal 12 MB=0x5ffdf3334d980 00000400:00000200:0.0:1616513072.373974:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.179@tcp1 of length 224/224 into md 0x865b9f31 [1] + 55520 00000400:00000200:0.0:1616513072.373979:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.373981:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513072.373988:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.373991:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.373993:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.374054:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708985444736 00010000:00000200:24.0:1616513072.374066:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d7500 x1688708985444736/t0(0) o400->3ec52b1b-3efc-ad1a-a84f-5e12c1a6749c@10.31.132.179@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.374080:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688708985444736, offset 192 00000400:00000200:24.0:1616513072.374087:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.179@tcp1 00000400:00000200:24.0:1616513072.374095:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.179@tcp1 routed destination 00000400:00000200:24.0:1616513072.374109:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.179@tcp1(10.31.132.179@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.374116:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513072.374122:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.374125:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513072.374127:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513072.374132:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.374139:0:48639:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.374194:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.374201:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513072.374203:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513072.374205:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.374210:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.374214:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:26.0:1616513072.374218:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.392733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513072.392788:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513072.392788:0:48641:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.392797:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:30.0:1616513072.392801:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513072.392806:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.81@tcp1 : PUT - for me 00000800:00000200:30.0:1616513072.392811:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513072.392815:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.81@tcp1 of length 224 into portal 26 MB=0x5ffd4e455f440 00000400:00000200:30.0:1616513072.392820:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.81@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.392822:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.81@tcp1 of length 224/224 into md 0xf97c4a19 [1] + 0 00000400:00000200:12.0:1616513072.392827:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d1a90 00000400:00000200:30.0:1616513072.392829:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.81@tcp1 of length 224 into portal 12 MB=0x5ffd4e455f4c0 00000400:00000200:12.0:1616513072.392830:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.392833:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513072.392837:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d1a90 00000800:00000200:12.0:1616513072.392841:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:30.0:1616513072.392843:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.81@tcp1 of length 224/224 into md 0xf96450b9 [1] + 23432 00000800:00000200:12.0:1616513072.392844:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.392845:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.392848:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513072.392852:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513072.392855:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:8.0:1616513072.392856:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664712541248 00000800:00000200:14.0:1616513072.392858:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513072.392862:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.392865:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:8.0:1616513072.392866:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688664712541248/t0(0) o400->16357870-2781-14f6-32a8-24210cf838d9@10.31.136.81@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:30.0:1616513072.392867:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513072.392881:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664712541248, offset 192 00000400:00000200:8.0:1616513072.392887:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.81@tcp1 00000400:00000200:8.0:1616513072.392894:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.81@tcp1 routed destination 00000400:00000200:8.0:1616513072.392908:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.81@tcp1(10.31.136.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.392915:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.392921:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.392923:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.392925:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:18.0:1616513072.392928:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664712541376 00000800:00000200:8.0:1616513072.392930:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.392935:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513072.392937:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:18.0:1616513072.392940:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7bf00 x1688664712541376/t0(0) o400->826afbaf-e32c-190e-624a-b051c6b91e17@10.31.136.81@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.392955:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664712541376, offset 192 00000400:00000200:18.0:1616513072.392961:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.81@tcp1 00000400:00000200:18.0:1616513072.392969:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.81@tcp1 routed destination 00000400:00000200:18.0:1616513072.392983:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.81@tcp1(10.31.136.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513072.392991:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.392998:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513072.392999:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513072.393001:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513072.393005:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.393007:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513072.393008:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513072.393010:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513072.393013:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70770 00000800:00000200:18.0:1616513072.393015:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513072.393019:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513072.393020:0:48641:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513072.393022:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513072.393077:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.393085:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513072.393087:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.393094:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.393098:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cca18 00000800:00000200:2.0:1616513072.393102:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.393105:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.426029:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513072.426085:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.426095:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513072.426104:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.11@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.426115:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.11@tcp1 of length 440 into portal 12 MB=0x603d0eaee7480 00000400:00000200:12.0:1616513072.426121:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.21.11@tcp1 of length 440/440 into md 0x865b9f31 [1] + 55744 00000400:00000200:12.0:1616513072.426126:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.426129:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513072.426136:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.426139:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513072.426140:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.426144:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.426201:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693045689840768 00010000:00000200:24.0:1616513072.426289:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57d580 x1693045689840768/t0(0) o41->818c9703-1177-c410-c401-4735e4ab1220@10.31.21.11@tcp1:698/0 lens 440/368 e 0 to 0 dl 1616513078 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513072.426304:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693045689840768, offset 192 00000400:00000200:24.0:1616513072.426311:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.11@tcp1 00000400:00000200:24.0:1616513072.426319:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.11@tcp1 routed destination 00000400:00000200:24.0:1616513072.426334:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.11@tcp1(10.31.21.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.426341:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.426346:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.426349:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.426351:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.426358:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.426364:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.426419:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.426428:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.426430:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.426447:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.426451:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf27f8 00000800:00000200:28.0:1616513072.426455:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.426458:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.462430:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513072.462486:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.462496:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.462505:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.131@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.462517:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.131@tcp1 of length 224 into portal 26 MB=0x603e437ec4980 00000400:00000200:16.0:1616513072.462524:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.131@tcp1 of length 224/224 into md 0xf97c4a29 [1] + 0 00000400:00000200:16.0:1616513072.462529:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d06e8 00000400:00000200:16.0:1616513072.462532:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.462535:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513072.462540:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d06e8 00000800:00000200:12.0:1616513072.462540:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.462544:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.462546:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.462548:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.462551:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:8.0:1616513072.462559:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693128585922944 00010000:00000200:8.0:1616513072.462570:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1693128585922944/t0(0) o400->9e2aad2d-fe48-48b2-9519-2246d9526426@10.31.132.131@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.462583:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693128585922944, offset 192 00000400:00000200:8.0:1616513072.462590:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.131@tcp1 00000400:00000200:8.0:1616513072.462598:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.131@tcp1 routed destination 00000800:00000200:28.0:1616513072.462606:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513072.462612:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.131@tcp1(10.31.132.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513072.462616:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.462619:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513072.462625:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.131@tcp1 : PUT - for me 00000800:00000200:8.0:1616513072.462625:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.462628:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.462630:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513072.462634:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.131@tcp1 of length 224 into portal 12 MB=0x603e437ec4a40 00000800:00000200:8.0:1616513072.462635:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513072.462641:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.131@tcp1 of length 224/224 into md 0x865b9f31 [1] + 56184 00000100:00000200:8.0:1616513072.462641:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513072.462646:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.462649:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513072.462652:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513072.462656:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.462659:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513072.462661:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.462666:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.462721:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693128585923136 00000800:00000200:14.0:1616513072.462725:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.462732:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:24.0:1616513072.462735:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ef500 x1693128585923136/t0(0) o400->7ddca444-5cfb-8121-a71b-376138e1049a@10.31.132.131@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513072.462735:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513072.462736:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513072.462742:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513072.462748:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f715c8 00000100:00000200:24.0:1616513072.462751:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693128585923136, offset 192 00000800:00000200:14.0:1616513072.462752:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513072.462758:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.131@tcp1 00000400:00000200:24.0:1616513072.462765:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.131@tcp1 routed destination 00000400:00000200:24.0:1616513072.462784:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.131@tcp1(10.31.132.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.462791:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.462797:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.462799:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.462801:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.462806:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.462811:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.462822:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.462827:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.462830:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513072.462832:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513072.462837:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.462841:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2000 00000800:00000200:12.0:1616513072.462848:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.489525:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.489589:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.489599:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.489609:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.176@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.489619:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.176@tcp1 of length 224 into portal 26 MB=0x5ffd3d4c8de40 00000400:00000200:2.0:1616513072.489625:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.176@tcp1 of length 224/224 into md 0xf97c4a39 [1] + 0 00000400:00000200:2.0:1616513072.489630:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0330 00000400:00000200:2.0:1616513072.489633:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.489636:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513072.489642:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0330 00000800:00000200:26.0:1616513072.489643:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.489645:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.489648:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.489650:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513072.489663:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660156669504 00010000:00000200:8.0:1616513072.489674:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688660156669504/t0(0) o400->e3aeb34b-30a5-6fbf-eaef-cf6018114a42@10.31.128.176@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.489687:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660156669504, offset 192 00000400:00000200:8.0:1616513072.489693:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.176@tcp1 00000400:00000200:8.0:1616513072.489701:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.176@tcp1 routed destination 00000400:00000200:8.0:1616513072.489716:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.176@tcp1(10.31.128.176@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.489723:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.489728:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.489731:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.489733:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.489738:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.489742:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513072.489744:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513072.489752:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.489757:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.489760:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513072.489766:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.489770:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70198 00000800:00000200:30.0:1616513072.489773:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513072.489776:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.490015:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513072.490087:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.490096:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513072.490106:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.176@tcp1 : PUT - for me 00000400:00000200:0.0:1616513072.490115:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.176@tcp1 of length 224 into portal 12 MB=0x5ffd3d4c8e1c0 00000400:00000200:0.0:1616513072.490121:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.176@tcp1 of length 224/224 into md 0x865b9f31 [1] + 56408 00000400:00000200:0.0:1616513072.490127:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.490129:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513072.490136:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.490139:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513072.490139:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513072.490143:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.490198:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660156670400 00010000:00000200:24.0:1616513072.490211:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d0d80 x1688660156670400/t0(0) o400->408ae192-7b28-60ea-c768-3dc5fc77d752@10.31.128.176@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.490224:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660156670400, offset 192 00000400:00000200:24.0:1616513072.490230:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.176@tcp1 00000400:00000200:24.0:1616513072.490237:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.176@tcp1 routed destination 00000400:00000200:24.0:1616513072.490251:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.176@tcp1(10.31.128.176@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.490257:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.490262:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.490265:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.490267:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.490272:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.490277:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.490336:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.490343:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.490346:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513072.490353:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.490357:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000800:00000200:16.0:1616513072.490361:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.490364:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.494360:0:48651:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.494415:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.494424:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513072.494451:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.158@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.494462:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.158@tcp1 of length 224 into portal 26 MB=0x6019e0bd774c0 00000400:00000200:26.0:1616513072.494468:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.158@tcp1 of length 224/224 into md 0xf97c4a49 [1] + 0 00000800:00000200:2.0:1616513072.494469:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.494474:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d14b8 00000400:00000200:26.0:1616513072.494476:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.494479:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513072.494484:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d14b8 00000800:00000200:26.0:1616513072.494488:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.494490:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.494492:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513072.494496:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690628175393984 00010000:00000200:8.0:1616513072.494502:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1690628175393984/t0(0) o400->90585914-e6f7-d2c8-9a3e-8fa9216acc66@10.31.140.158@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.494513:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690628175393984, offset 192 00000400:00000200:8.0:1616513072.494518:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.158@tcp1 00000400:00000200:8.0:1616513072.494525:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.158@tcp1 routed destination 00000400:00000200:8.0:1616513072.494538:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.158@tcp1(10.31.140.158@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.494543:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.494548:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.494551:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.494552:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.494555:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513072.494560:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.494560:0:48651:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513072.494569:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513072.494572:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.494574:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513072.494579:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513072.494583:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70908 00000800:00000200:30.0:1616513072.494585:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513072.494587:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.494660:0:48651:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.494672:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.494680:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513072.494681:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513072.494688:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.158@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.494697:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.158@tcp1 of length 224 into portal 12 MB=0x6019e0bd778c0 00000400:00000200:2.0:1616513072.494704:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.158@tcp1 of length 224/224 into md 0xf96450b9 [1] + 23656 00000400:00000200:2.0:1616513072.494709:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.494712:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513072.494720:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.494722:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.494725:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.494787:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690628175395008 00010000:00000200:18.0:1616513072.494799:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138b180 x1690628175395008/t0(0) o400->a4727e39-8d52-b345-e7d9-97e3e2769798@10.31.140.158@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.494813:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690628175395008, offset 192 00000400:00000200:18.0:1616513072.494819:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.158@tcp1 00000400:00000200:18.0:1616513072.494827:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.158@tcp1 routed destination 00000400:00000200:18.0:1616513072.494842:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.158@tcp1(10.31.140.158@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.494856:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.494861:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.494864:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.494866:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.494871:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.494877:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.494931:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.494938:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.494941:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.494948:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.494951:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc5d8 00000800:00000200:28.0:1616513072.494955:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.494958:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.522118:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513072.522173:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.522183:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.522192:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.173@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.522201:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.173@tcp1 of length 224 into portal 26 MB=0x6026e0b73bd80 00000400:00000200:16.0:1616513072.522208:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.173@tcp1 of length 224/224 into md 0xf97c4a79 [1] + 0 00000400:00000200:16.0:1616513072.522213:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0b28 00000400:00000200:16.0:1616513072.522216:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.522218:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513072.522223:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0b28 00000800:00000200:16.0:1616513072.522226:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.522226:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.522229:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513072.522231:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513072.522244:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691521522056576 00010000:00000200:8.0:1616513072.522254:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1691521522056576/t0(0) o400->00ad7ff2-0607-f6f4-2144-daa21c2738c1@10.31.129.173@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513072.522255:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:8.0:1616513072.522268:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691521522056576, offset 192 00000400:00000200:8.0:1616513072.522274:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.173@tcp1 00000400:00000200:8.0:1616513072.522282:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.173@tcp1 routed destination 00000400:00000200:8.0:1616513072.522297:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.173@tcp1(10.31.129.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.522304:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.522309:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513072.522310:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.522313:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.522315:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513072.522319:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.522320:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513072.522327:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.522327:0:48654:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513072.522328:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.173@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.522337:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.173@tcp1 of length 224 into portal 12 MB=0x6026e0b73c2c0 00000800:00000200:30.0:1616513072.522339:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513072.522343:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.173@tcp1 of length 224/224 into md 0x865b9f31 [1] + 56632 00000800:00000200:30.0:1616513072.522344:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513072.522347:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.522348:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.522352:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:30.0:1616513072.522353:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513072.522359:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:30.0:1616513072.522360:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f704c8 00000800:00000200:28.0:1616513072.522362:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:30.0:1616513072.522364:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513072.522364:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:30.0:1616513072.522367:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513072.522368:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.522427:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691521522057920 00010000:00000200:24.0:1616513072.522452:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0e8000 x1691521522057920/t0(0) o400->4f3cb6ae-c2c2-f20f-3bda-5772ba37be5c@10.31.129.173@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.522472:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691521522057920, offset 192 00000400:00000200:24.0:1616513072.522479:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.173@tcp1 00000400:00000200:24.0:1616513072.522488:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.173@tcp1 routed destination 00000400:00000200:24.0:1616513072.522503:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.173@tcp1(10.31.129.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.522510:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.522515:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.522518:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.522520:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.522525:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.522530:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.522541:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.522546:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.522549:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513072.522551:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513072.522556:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.522560:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:12.0:1616513072.522564:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.531852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513072.531907:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.531916:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513072.531925:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.100@tcp1 : PUT - for me 00000400:00000200:0.0:1616513072.531935:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.100@tcp1 of length 224 into portal 12 MB=0x6001f15185400 00000400:00000200:0.0:1616513072.531941:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.100@tcp1 of length 224/224 into md 0x865b9f31 [1] + 56856 00000400:00000200:0.0:1616513072.531947:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.531950:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513072.531956:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.531960:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.531961:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513072.531964:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.532020:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688983358166016 00010000:00000200:24.0:1616513072.532033:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3180 x1688983358166016/t0(0) o400->b2443e44-8d22-9e45-0e46-ac86670751d1@10.31.142.100@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.532047:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688983358166016, offset 192 00000400:00000200:24.0:1616513072.532054:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.100@tcp1 00000800:00000200:7.2:1616513072.532055:0:48655:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513072.532061:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.100@tcp1 routed destination 00000400:00000200:24.0:1616513072.532077:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.100@tcp1(10.31.142.100@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.532083:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513072.532089:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.532092:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513072.532094:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513072.532099:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513072.532110:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.532119:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513072.532128:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.100@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.532137:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.100@tcp1 of length 224 into portal 26 MB=0x6001f15185300 00000400:00000200:26.0:1616513072.532144:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.100@tcp1 of length 224/224 into md 0xf97c4a89 [1] + 0 00000400:00000200:26.0:1616513072.532149:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0198 00000400:00000200:26.0:1616513072.532151:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.532154:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513072.532159:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0198 00000800:00000200:26.0:1616513072.532162:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513072.532164:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513072.532165:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513072.532171:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513072.532172:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513072.532174:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513072.532175:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513072.532177:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688983358165760 00000400:00000200:2.0:1616513072.532181:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.532185:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3298 00010000:00000200:8.0:1616513072.532186:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688983358165760/t0(0) o400->504492ae-6ab5-3e4d-e2b2-05c1d90fae18@10.31.142.100@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513072.532189:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513072.532199:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688983358165760, offset 192 00000400:00000200:8.0:1616513072.532205:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.100@tcp1 00000400:00000200:8.0:1616513072.532211:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.100@tcp1 routed destination 00000400:00000200:8.0:1616513072.532225:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.100@tcp1(10.31.142.100@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.532231:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.532237:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.532240:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.532242:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.532246:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.532251:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513072.532252:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513072.532304:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.532311:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.532314:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513072.532321:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.532327:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70aa0 00000800:00000200:16.0:1616513072.532330:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.532333:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.543070:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513072.543126:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.543136:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513072.543146:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.68@tcp1 : PUT - for me 00000400:00000200:14.0:1616513072.543155:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.68@tcp1 of length 224 into portal 26 MB=0x5ffd338baef40 00000400:00000200:14.0:1616513072.543164:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.68@tcp1 of length 224/224 into md 0xf97c4a99 [1] + 0 00000400:00000200:14.0:1616513072.543169:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d83a23d0880 00000400:00000200:14.0:1616513072.543171:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.543174:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513072.543179:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83a23d0880 00000800:00000200:2.0:1616513072.543180:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.543182:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.543185:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.543187:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513072.543194:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657538510656 00010000:00000200:8.0:1616513072.543201:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688657538510656/t0(0) o400->5ab31469-0c7c-8ebf-85fe-7a7b27abf695@10.31.133.68@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.543212:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657538510656, offset 192 00000400:00000200:8.0:1616513072.543217:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.68@tcp1 00000400:00000200:8.0:1616513072.543223:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.68@tcp1 routed destination 00000400:00000200:8.0:1616513072.543236:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.68@tcp1(10.31.133.68@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.543242:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.543246:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.543249:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.543251:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.543254:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513072.543260:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.543260:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.543313:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.543320:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513072.543323:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513072.543325:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513072.543329:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:2.0:1616513072.543334:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513072.543335:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71760 00000800:00000200:6.0:1616513072.543335:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513072.543339:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513072.543344:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.53@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.543354:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.53@tcp1 of length 224 into portal 12 MB=0x6001f2dc10b00 00000400:00000200:2.0:1616513072.543361:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.53@tcp1 of length 224/224 into md 0xf96450b9 [1] + 23880 00000400:00000200:2.0:1616513072.543366:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.543369:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513072.543377:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.543380:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.543382:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.543421:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513072.543469:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688983771876096 00000800:00000200:28.0:1616513072.543474:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:18.0:1616513072.543481:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138b600 x1688983771876096/t0(0) o400->069c399e-de25-93ac-dec9-69e51d9b967a@10.31.129.53@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513072.543483:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.543491:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.53@tcp1 : PUT - for me 00000100:00000200:18.0:1616513072.543495:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688983771876096, offset 192 00000400:00000200:28.0:1616513072.543500:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.53@tcp1 of length 224 into portal 26 MB=0x6001f2dc105c0 00000400:00000200:18.0:1616513072.543502:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.53@tcp1 00000400:00000200:28.0:1616513072.543506:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.53@tcp1 of length 224/224 into md 0xf97c4ab9 [1] + 0 00000400:00000200:18.0:1616513072.543509:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.53@tcp1 routed destination 00000400:00000200:28.0:1616513072.543511:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f760 00000400:00000200:28.0:1616513072.543514:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.543517:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513072.543521:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f760 00000400:00000200:18.0:1616513072.543524:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.53@tcp1(10.31.129.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513072.543529:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:8.0:1616513072.543530:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688983771874752 00000800:00000200:18.0:1616513072.543531:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513072.543531:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.543532:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:8.0:1616513072.543534:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688983771874752/t0(0) o400->a032f1d1-6011-60cb-7b68-98b89b8011c5@10.31.129.53@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513072.543535:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513072.543537:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.543540:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.543542:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513072.543545:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688983771874752, offset 192 00000800:00000200:18.0:1616513072.543546:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513072.543549:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.53@tcp1 00000800:00000200:11.2:1616513072.543551:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513072.543555:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.53@tcp1 routed destination 00000400:00000200:8.0:1616513072.543566:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.53@tcp1(10.31.129.53@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.543571:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.543575:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.543577:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.543579:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.543582:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513072.543585:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.543587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.543596:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.543600:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513072.543602:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:6.0:1616513072.543602:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513072.543604:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513072.543608:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513072.543611:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513072.543612:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71a90 00000400:00000200:16.0:1616513072.543614:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513072.543615:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513072.543621:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.543625:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccbb0 00000800:00000200:16.0:1616513072.543636:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.543639:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.543986:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513072.543999:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.544006:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513072.544009:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513072.544015:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.68@tcp1 : PUT - for me 00000400:00000200:0.0:1616513072.544024:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.68@tcp1 of length 224 into portal 12 MB=0x5ffd338baf140 00000400:00000200:0.0:1616513072.544030:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.68@tcp1 of length 224/224 into md 0x865b9f31 [1] + 57080 00000400:00000200:0.0:1616513072.544035:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.544038:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513072.544044:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.544047:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.544049:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.544109:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657538511168 00010000:00000200:24.0:1616513072.544122:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d2400 x1688657538511168/t0(0) o400->c332bf28-5601-e839-99de-3610629e8fb3@10.31.133.68@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.544136:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657538511168, offset 192 00000400:00000200:24.0:1616513072.544142:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.68@tcp1 00000400:00000200:24.0:1616513072.544150:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.68@tcp1 routed destination 00000400:00000200:24.0:1616513072.544164:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.68@tcp1(10.31.133.68@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.544171:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.544177:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.544179:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.544183:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.544188:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.544193:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.544203:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.544206:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.544208:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513072.544212:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.544213:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.544216:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2198 00000800:00000200:16.0:1616513072.544218:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.558448:0:48658:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.558502:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.558514:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513072.558523:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.197@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.558533:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.197@tcp1 of length 224 into portal 12 MB=0x6018d4cd5d900 00000400:00000200:26.0:1616513072.558539:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.197@tcp1 of length 224/224 into md 0xf96450b9 [1] + 24104 00000400:00000200:26.0:1616513072.558545:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.558547:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513072.558554:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.558558:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.558558:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513072.558564:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.558575:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690556251363584 00010000:00000200:18.0:1616513072.558587:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c696300 x1690556251363584/t0(0) o400->0b4b6a76-c4f2-9607-e3b4-a1fcbbd3224a@10.31.128.197@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513072.558600:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513072.558601:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690556251363584, offset 192 00000400:00000200:18.0:1616513072.558608:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.197@tcp1 00000400:00000200:18.0:1616513072.558616:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.197@tcp1 routed destination 00000400:00000200:18.0:1616513072.558631:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.197@tcp1(10.31.128.197@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.558637:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513072.558643:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.558646:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513072.558648:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513072.558653:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513072.558655:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513072.558657:0:48658:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513072.558664:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:6.0:1616513072.558668:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513072.558673:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.197@tcp1 : PUT - for me 00000800:00000200:6.0:1616513072.558674:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513072.558676:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.558682:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.197@tcp1 of length 224 into portal 26 MB=0x6018d4cd5d400 00000400:00000200:6.0:1616513072.558683:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.558689:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.197@tcp1 of length 224/224 into md 0xf97c4ad9 [1] + 0 00000400:00000200:28.0:1616513072.558694:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f8f8 00000400:00000200:28.0:1616513072.558697:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:6.0:1616513072.558697:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccdd0 00000100:00000200:28.0:1616513072.558700:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:6.0:1616513072.558701:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513072.558705:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f8f8 00000800:00000200:6.0:1616513072.558705:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513072.558708:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.558710:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.558711:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.558714:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513072.558731:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690556251362304 00010000:00000200:8.0:1616513072.558739:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1690556251362304/t0(0) o400->2b40407e-33f4-837d-0440-06d806aad6d1@10.31.128.197@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.558750:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690556251362304, offset 192 00000400:00000200:8.0:1616513072.558755:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.197@tcp1 00000400:00000200:8.0:1616513072.558762:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.197@tcp1 routed destination 00000400:00000200:8.0:1616513072.558774:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.197@tcp1(10.31.128.197@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.558780:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.558785:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.558787:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.558789:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.558793:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.558798:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513072.558798:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513072.558851:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.558858:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.558861:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513072.558867:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.558873:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70ee0 00000800:00000200:12.0:1616513072.558876:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.558879:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:11.0:1616513072.570498:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924048, portal 4 00000100:00000200:11.0:1616513072.570503:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924048, offset 0 00000400:00000200:11.0:1616513072.570511:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:11.0:1616513072.570519:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:11.0:1616513072.570532:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513072.570540:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:11.0:1616513072.570548:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513072.570551:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:11.0:1616513072.570553:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:11.0:1616513072.570560:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:6.0:1616513072.570563:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924064, portal 4 00000100:00000200:6.0:1616513072.570568:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924064, offset 0 00000400:00000200:6.0:1616513072.570573:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:6.0:1616513072.570579:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:6.0:1616513072.570589:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513072.570594:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:6.0:1616513072.570600:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513072.570602:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:6.0:1616513072.570604:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:27.2:1616513072.570607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:6.0:1616513072.570608:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:27.0:1616513072.570628:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513072.570633:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513072.570636:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513072.570639:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513072.570644:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513072.570651:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8f30a8da00 x1693067725924048/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513079 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.2:1616513072.570658:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:27.0:1616513072.570663:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3af68 00000800:00000200:27.0:1616513072.570668:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:1.0:1616513072.570679:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513072.570685:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513072.570688:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513072.570695:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513072.570703:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de138ba80 x1693067725924064/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513079 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513072.570715:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01870 00000800:00000200:1.0:1616513072.570721:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513072.570723:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:27.2:1616513072.570781:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513072.570790:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513072.570797:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513072.570798:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513072.570805:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513072.570814:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6266d0 00000400:00000200:27.0:1616513072.570821:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716538d [1] + 192 00000400:00000200:27.0:1616513072.570826:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513072.570831:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8f30a8da00 x1693067725924048/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513079 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.2:1616513072.570838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:27.0:1616513072.570846:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513072.570848:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:1.0:1616513072.570848:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:27.0:1616513072.570850:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:11.0:1616513072.570850:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8f30a8da00 x1693067725924048/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513079 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513072.570854:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000100:00000200:11.0:1616513072.570860:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8f30a8da00 x1693067725924048/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513079 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513072.570863:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:11.0:1616513072.570868:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3aaa0 00000400:00000200:1.0:1616513072.570872:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6266e0 00000400:00000200:1.0:1616513072.570879:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff945 [1] + 192 00000400:00000200:1.0:1616513072.570884:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513072.570888:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de138ba80 x1693067725924064/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513079 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513072.570903:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513072.570914:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513072.570917:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513072.570919:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:6.0:1616513072.570920:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de138ba80 x1693067725924064/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513079 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:6.0:1616513072.570930:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de138ba80 x1693067725924064/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513079 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513072.570937:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01430 00000800:00000200:27.2:1616513072.601356:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513072.601374:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513072.601381:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513072.601388:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513072.601395:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e7a0 00000400:00000200:27.0:1616513072.601401:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 87136 00000400:00000200:27.0:1616513072.601406:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513072.601410:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:27.0:1616513072.601418:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513072.601421:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513072.601423:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:31.0:1616513072.601426:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513072.601452:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:31.0:1616513072.601460:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:31.0:1616513072.601468:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e820 00000400:00000200:31.0:1616513072.601475:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 87360 00000400:00000200:31.0:1616513072.601480:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:31.0:1616513072.601482:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:31.0:1616513072.601492:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513072.601495:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:31.0:1616513072.601497:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:3.0:1616513072.601500:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:15.0:1616513072.601502:0:252828:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938528 00000100:00000200:27.0:1616513072.601505:0:171364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938656 00000800:00000200:3.0:1616513072.601512:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513072.601514:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00010000:00000200:27.0:1616513072.601517:0:171364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8006cdd580 x1681364774938656/t0(0) o400->boslfs02-MDT0001-lwp-OST002e_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:15.0:1616513072.601517:0:252828:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d841ded6c00 x1681364774938528/t0(0) o400->boslfs02-MDT0001-lwp-OST0032_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:31.0:1616513072.601520:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:3.0:1616513072.601521:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:7.0:1616513072.601522:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:31.0:1616513072.601527:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:3.0:1616513072.601529:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 26 MB=0x5f9313e25e780 00000100:00000200:27.0:1616513072.601530:0:171364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938656, offset 192 00000800:00000200:7.0:1616513072.601530:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:31.0:1616513072.601532:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e8a0 00000100:00000200:15.0:1616513072.601532:0:252828:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938528, offset 192 00000800:00000200:19.0:1616513072.601535:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000400:00000200:27.0:1616513072.601536:0:171364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:3.0:1616513072.601536:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0xf97c4ae9 [1] + 0 00000400:00000200:7.0:1616513072.601537:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:31.0:1616513072.601540:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 87584 00000400:00000200:3.0:1616513072.601541:0:6475:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e440 00000400:00000200:27.0:1616513072.601542:0:171364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:19.0:1616513072.601542:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:31.0:1616513072.601544:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:3.0:1616513072.601544:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:15.0:1616513072.601547:0:252828:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:3.0:1616513072.601547:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:31.0:1616513072.601549:0:6476:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:3.0:1616513072.601553:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e440 00000400:00000200:27.0:1616513072.601554:0:171364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513072.601559:0:171364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:15.0:1616513072.601563:0:252828:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:19.0:1616513072.601564:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:15.0:1616513072.601577:0:252828:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000100:00000200:8.0:1616513072.601578:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938496 00000800:00000200:31.0:1616513072.601581:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:15.0:1616513072.601583:0:252828:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:3.0:1616513072.601583:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (26)++ 00000400:00000200:7.0:1616513072.601585:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e800 00000800:00000200:31.0:1616513072.601586:0:6476:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:27.0:1616513072.601586:0:171364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:19.0:1616513072.601588:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e920 00000800:00000200:31.0:1616513072.601589:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:27.0:1616513072.601589:0:171364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:15.0:1616513072.601589:0:252828:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513072.601590:0:171364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (29)++ 00000100:00000200:23.0:1616513072.601590:0:174000:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938784 00000400:00000200:7.0:1616513072.601590:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 87808 00000800:00000200:3.0:1616513072.601590:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:31.0:1616513072.601591:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:15.0:1616513072.601592:0:252828:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:3.0:1616513072.601593:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:27.0:1616513072.601594:0:171364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (27)-- 00000400:00000200:7.0:1616513072.601594:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:31.0:1616513072.601595:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:15.0:1616513072.601595:0:252828:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (27)++ 00000400:00000200:19.0:1616513072.601596:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 88032 00010000:00000200:8.0:1616513072.601599:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1681364774938496/t0(0) o400->2a9c6460-7cbc-99eb-d035-661c67cc9089@10.242.46.231@o2ib1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:15.0:1616513072.601600:0:252828:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (28)-- 00000400:00000200:19.0:1616513072.601601:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:7.0:1616513072.601601:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:23.0:1616513072.601602:0:174000:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8006cdb180 x1681364774938784/t0(0) o400->boslfs02-MDT0001-lwp-OST0036_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:19.0:1616513072.601606:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.0:1616513072.601607:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:7.0:1616513072.601609:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)-- 00000100:00000200:8.0:1616513072.601612:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681364774938496, offset 192 00000800:00000200:7.0:1616513072.601614:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:31.0:1616513072.601616:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:7.0:1616513072.601616:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:7.0:1616513072.601618:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:3.0:1616513072.601619:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)++ 00000400:00000200:8.0:1616513072.601620:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:7.0:1616513072.601621:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:7.0:1616513072.601622:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:3.0:1616513072.601622:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (27)-- 00000400:00000200:31.0:1616513072.601623:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:7.0:1616513072.601624:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (26)-- 00000400:00000200:3.0:1616513072.601624:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:7.0:1616513072.601625:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513072.601627:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:31.0:1616513072.601629:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e880 00000100:00000200:23.0:1616513072.601629:0:174000:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938784, offset 192 00000800:00000200:19.0:1616513072.601629:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:7.0:1616513072.601629:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:3.0:1616513072.601629:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:19.0:1616513072.601632:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (26)-- 00000400:00000200:31.0:1616513072.601633:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 88256 00000100:00000200:27.0:1616513072.601634:0:171364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938624 00000800:00000200:19.0:1616513072.601634:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:19.0:1616513072.601637:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000400:00000200:31.0:1616513072.601639:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:7.0:1616513072.601639:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774c38 00010000:00000200:27.0:1616513072.601640:0:171364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8f30a89f80 x1681364774938624/t0(0) o400->boslfs02-MDT0001-lwp-OST0002_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:19.0:1616513072.601641:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:8.0:1616513072.601642:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:23.0:1616513072.601645:0:174000:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:15.0:1616513072.601648:0:252828:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938912 00000800:00000200:7.0:1616513072.601648:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000400:00000200:3.0:1616513072.601648:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249e48 00000800:00000200:8.0:1616513072.601649:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:7.0:1616513072.601649:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000100:00000200:31.0:1616513072.601650:0:6476:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.0:1616513072.601652:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:19.0:1616513072.601653:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:23.0:1616513072.601654:0:174000:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00010000:00000200:15.0:1616513072.601654:0:252828:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8dd8bfb600 x1681364774938912/t0(0) o400->boslfs02-MDT0001-lwp-OST003a_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:27.0:1616513072.601656:0:171364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938624, offset 192 00000800:00000200:8.0:1616513072.601656:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.601659:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:27.0:1616513072.601661:0:171364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:8.0:1616513072.601661:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (26)++ 00000100:00000200:15.0:1616513072.601665:0:252828:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938912, offset 192 00000800:00000200:8.0:1616513072.601666:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (27)-- 00000400:00000200:23.0:1616513072.601668:0:174000:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:15.0:1616513072.601671:0:252828:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:8.0:1616513072.601673:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:3.0:1616513072.601673:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:31.0:1616513072.601674:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:23.0:1616513072.601674:0:174000:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000100:00000200:11.0:1616513072.601675:0:170089:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938752 00000800:00000200:3.0:1616513072.601675:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:7.0:1616513072.601676:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:31.0:1616513072.601677:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:31.0:1616513072.601678:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)-- 00000400:00000200:27.0:1616513072.601679:0:171364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:3.0:1616513072.601679:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513072.601680:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:31.0:1616513072.601684:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00010000:00000200:11.0:1616513072.601684:0:170089:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8006cdde80 x1681364774938752/t0(0) o400->boslfs02-MDT0001-lwp-OST0022_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:27.0:1616513072.601691:0:171364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513072.601696:0:171364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000100:00000200:11.0:1616513072.601696:0:170089:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938752, offset 192 00000400:00000200:15.0:1616513072.601697:0:252828:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:11.0:1616513072.601702:0:170089:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:15.0:1616513072.601710:0:252828:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:19.0:1616513072.601716:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e7c0 00000800:00000200:15.0:1616513072.601716:0:252828:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513072.601718:0:171364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513072.601718:0:174000:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:11.0:1616513072.601719:0:170089:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:27.0:1616513072.601720:0:171364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:23.0:1616513072.601720:0:174000:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (26)++ 00000400:00000200:31.0:1616513072.601721:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:19.0:1616513072.601721:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 88480 00000400:00000200:3.0:1616513072.601721:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:27.0:1616513072.601722:0:171364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:27.0:1616513072.601725:0:171364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:23.0:1616513072.601725:0:174000:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (29)++ 00000400:00000200:19.0:1616513072.601725:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:19.0:1616513072.601727:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:23.0:1616513072.601730:0:174000:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)-- 00000400:00000200:11.0:1616513072.601730:0:170089:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513072.601736:0:252828:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513072.601736:0:170089:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:7.0:1616513072.601736:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e900 00000800:00000200:19.0:1616513072.601737:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)++ 00000400:00000200:31.0:1616513072.601738:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e7e0 00000800:00000200:19.0:1616513072.601739:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:15.0:1616513072.601739:0:252828:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)++ 00000400:00000200:7.0:1616513072.601739:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 88704 00000400:00000200:3.0:1616513072.601739:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e840 00000800:00000200:19.0:1616513072.601740:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:15.0:1616513072.601741:0:252828:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:11.0:1616513072.601741:0:170089:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:7.0:1616513072.601742:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:19.0:1616513072.601743:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:11.0:1616513072.601743:0:170089:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:15.0:1616513072.601744:0:252828:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)-- 00000400:00000200:31.0:1616513072.601745:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 88928 00000100:00000200:27.0:1616513072.601745:0:171364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938560 00000800:00000200:11.0:1616513072.601745:0:170089:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (30)++ 00000400:00000200:31.0:1616513072.601748:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:11.0:1616513072.601748:0:170089:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (31)-- 00010000:00000200:27.0:1616513072.601750:0:171364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8dd8bf8d80 x1681364774938560/t0(0) o400->boslfs02-MDT0001-lwp-OST0006_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:3.0:1616513072.601750:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 89152 00000800:00000200:19.0:1616513072.601751:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000100:00000200:7.0:1616513072.601754:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:3.0:1616513072.601754:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:19.0:1616513072.601757:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:19.0:1616513072.601762:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e8c0 00000100:00000200:31.0:1616513072.601764:0:6476:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.0:1616513072.601764:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (30)++ 00000800:00000200:7.0:1616513072.601766:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:7.0:1616513072.601768:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:7.0:1616513072.601769:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:31.0:1616513072.601770:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)++ 00000100:00000200:3.0:1616513072.601770:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.0:1616513072.601771:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:31.0:1616513072.601772:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:31.0:1616513072.601773:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000400:00000200:7.0:1616513072.601773:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513072.601775:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000400:00000200:7.0:1616513072.601776:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513072.601778:0:171364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938560, offset 192 00000800:00000200:3.0:1616513072.601778:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:31.0:1616513072.601779:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000100:00000200:15.0:1616513072.601779:0:252828:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938880 00000400:00000200:19.0:1616513072.601780:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 89376 00000400:00000200:7.0:1616513072.601780:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71980 00000800:00000200:3.0:1616513072.601781:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:3.0:1616513072.601782:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000100:00000200:11.0:1616513072.601783:0:170089:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938592 00000400:00000200:19.0:1616513072.601784:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:15.0:1616513072.601785:0:252828:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8f30a8de80 x1681364774938880/t0(0) o400->boslfs02-MDT0001-lwp-OST0012_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:31.0:1616513072.601786:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513072.601786:0:171364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:23.0:1616513072.601786:0:170090:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938688 00000800:00000200:7.0:1616513072.601786:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)-- 00000100:00000200:19.0:1616513072.601787:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:3.0:1616513072.601787:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)++ 00010000:00000200:11.0:1616513072.601788:0:170089:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cdc566c00 x1681364774938592/t0(0) o400->boslfs02-MDT0001-lwp-OST002a_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:27.0:1616513072.601791:0:171364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00010000:00000200:23.0:1616513072.601792:0:170090:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8da4dd607080 x1681364774938688/t0(0) o400->boslfs02-MDT0001-lwp-OST001e_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513072.601795:0:252828:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938880, offset 192 00000100:00000200:11.0:1616513072.601797:0:170089:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938592, offset 192 00000400:00000200:15.0:1616513072.601800:0:252828:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:3.0:1616513072.601801:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:7.0:1616513072.601802:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000400:00000200:27.0:1616513072.601803:0:171364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513072.601803:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (29)-- 00000400:00000200:11.0:1616513072.601804:0:170089:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:7.0:1616513072.601804:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:23.0:1616513072.601806:0:170090:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938688, offset 192 00000800:00000200:19.0:1616513072.601807:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:27.0:1616513072.601808:0:171364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:7.0:1616513072.601808:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:23.0:1616513072.601811:0:170090:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:15.0:1616513072.601811:0:252828:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:7.0:1616513072.601813:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249a08 00000400:00000200:15.0:1616513072.601823:0:252828:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513072.601829:0:252828:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:11.0:1616513072.601829:0:170089:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:3.0:1616513072.601831:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:11.0:1616513072.601840:0:170089:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513072.601844:0:170089:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:31.0:1616513072.601845:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e860 00000800:00000200:27.0:1616513072.601845:0:171364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513072.601845:0:6474:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:27.0:1616513072.601847:0:171364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (30)++ 00000400:00000200:23.0:1616513072.601847:0:170090:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:19.0:1616513072.601848:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:27.0:1616513072.601849:0:171364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (31)++ 00000400:00000200:31.0:1616513072.601850:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 89600 00000800:00000200:19.0:1616513072.601850:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:27.0:1616513072.601851:0:171364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:19.0:1616513072.601852:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (30)++ 00000400:00000200:31.0:1616513072.601853:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:19.0:1616513072.601854:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (30)-- 00000100:00000200:31.0:1616513072.601855:0:6476:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:19.0:1616513072.601856:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:23.0:1616513072.601860:0:170090:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:19.0:1616513072.601861:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:19.0:1616513072.601864:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e2a8 00000800:00000200:7.0:1616513072.601864:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:23.0:1616513072.601866:0:170090:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:7.0:1616513072.601866:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:31.0:1616513072.601867:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:15.0:1616513072.601867:0:252828:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513072.601868:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:11.0:1616513072.601868:0:170089:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513072.601869:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:15.0:1616513072.601869:0:252828:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (30)++ 00000800:00000200:7.0:1616513072.601869:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:3.0:1616513072.601869:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e980 00000800:00000200:31.0:1616513072.601870:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:11.0:1616513072.601870:0:170089:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:23.0:1616513072.601871:0:170090:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513072.601871:0:252828:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:19.0:1616513072.601872:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:23.0:1616513072.601874:0:170090:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (31)++ 00000800:00000200:15.0:1616513072.601874:0:252828:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:11.0:1616513072.601874:0:170089:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (31)++ 00000400:00000200:7.0:1616513072.601874:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:3.0:1616513072.601874:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 89824 00000100:00000200:27.0:1616513072.601876:0:171364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938816 00000800:00000200:11.0:1616513072.601876:0:170089:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:23.0:1616513072.601878:0:170090:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (31)++ 00000400:00000200:3.0:1616513072.601878:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:7.0:1616513072.601879:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e8e0 00000100:00000200:3.0:1616513072.601880:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:27.0:1616513072.601881:0:171364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8dd8bf9200 x1681364774938816/t0(0) o400->boslfs02-MDT0001-lwp-OST001a_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:23.0:1616513072.601881:0:170090:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (32)-- 00000100:00000200:15.0:1616513072.601881:0:252828:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938720 00000800:00000200:19.0:1616513072.601885:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:3.0:1616513072.601885:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (31)++ 00000400:00000200:19.0:1616513072.601886:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:15.0:1616513072.601886:0:252828:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cdc565100 x1681364774938720/t0(0) o400->boslfs02-MDT0001-lwp-OST0026_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:7.0:1616513072.601886:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 90048 00000800:00000200:3.0:1616513072.601887:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (31)-- 00000400:00000200:7.0:1616513072.601889:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:3.0:1616513072.601889:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (30)-- 00000100:00000200:27.0:1616513072.601890:0:171364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938816, offset 192 00000100:00000200:7.0:1616513072.601890:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:19.0:1616513072.601891:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:3.0:1616513072.601891:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:7.0:1616513072.601896:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (30)++ 00000800:00000200:3.0:1616513072.601896:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (30)-- 00000400:00000200:27.0:1616513072.601898:0:171364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:7.0:1616513072.601898:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (30)-- 00000400:00000200:3.0:1616513072.601898:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:31.0:1616513072.601899:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939008 00000800:00000200:7.0:1616513072.601899:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000100:00000200:23.0:1616513072.601900:0:170091:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938848 00000400:00000200:19.0:1616513072.601900:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774550 00000800:00000200:7.0:1616513072.601901:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000100:00000200:15.0:1616513072.601902:0:252828:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938720, offset 192 00000400:00000200:27.0:1616513072.601903:0:171364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:3.0:1616513072.601903:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:7.0:1616513072.601904:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00010000:00000200:23.0:1616513072.601906:0:170091:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8f30a8da00 x1681364774938848/t0(0) o400->boslfs02-MDT0001-lwp-OST0016_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513072.601907:0:252828:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:3.0:1616513072.601908:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3aaa0 00010000:00000200:31.0:1616513072.601910:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8da4dd605e80 x1681364774939008/t0(0) o400->boslfs02-MDT0001-lwp-OST000a_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:27.0:1616513072.601914:0:171364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000100:00000200:23.0:1616513072.601916:0:170091:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938848, offset 192 00000800:00000200:19.0:1616513072.601918:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:27.0:1616513072.601919:0:171364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:19.0:1616513072.601920:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000400:00000200:23.0:1616513072.601921:0:170091:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:19.0:1616513072.601922:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (29)-- 00000400:00000200:15.0:1616513072.601922:0:252828:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:7.0:1616513072.601922:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000100:00000200:31.0:1616513072.601923:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939008, offset 192 00000800:00000200:19.0:1616513072.601924:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:19.0:1616513072.601925:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:19.0:1616513072.601927:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (28)-- 00000400:00000200:19.0:1616513072.601929:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:31.0:1616513072.601930:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:19.0:1616513072.601934:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:15.0:1616513072.601934:0:252828:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:19.0:1616513072.601936:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249cb0 00000800:00000200:3.0:1616513072.601938:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:15.0:1616513072.601940:0:252828:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513072.601941:0:171364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513072.601941:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:27.0:1616513072.601943:0:171364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (27)++ 00000400:00000200:23.0:1616513072.601943:0:170091:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:27.0:1616513072.601944:0:171364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:3.0:1616513072.601944:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000800:00000200:27.0:1616513072.601947:0:171364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)-- 00000400:00000200:23.0:1616513072.601955:0:170091:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513072.601960:0:170091:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:31.0:1616513072.601963:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:31.0:1616513072.601976:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513072.601982:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:31.0:1616513072.601983:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:19.0:1616513072.601984:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)++ 00000400:00000200:7.0:1616513072.601984:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e940 00000800:00000200:23.0:1616513072.601985:0:170091:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513072.601985:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:15.0:1616513072.601985:0:252828:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513072.601987:0:170091:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (27)++ 00000400:00000200:19.0:1616513072.601987:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:15.0:1616513072.601987:0:252828:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (27)++ 00000400:00000200:3.0:1616513072.601987:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:31.0:1616513072.601988:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:7.0:1616513072.601988:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 90272 00000800:00000200:23.0:1616513072.601989:0:170091:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (29)++ 00000400:00000200:7.0:1616513072.601990:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:31.0:1616513072.601991:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (30)++ 00000800:00000200:23.0:1616513072.601992:0:170091:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (30)-- 00000400:00000200:19.0:1616513072.601992:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:15.0:1616513072.601992:0:252828:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (30)++ 00000100:00000200:7.0:1616513072.601992:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:3.0:1616513072.601993:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e960 00000800:00000200:31.0:1616513072.601995:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (31)++ 00000800:00000200:15.0:1616513072.601995:0:252828:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (31)-- 00000400:00000200:19.0:1616513072.601996:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774cc0 00000800:00000200:7.0:1616513072.601996:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (31)++ 00000800:00000200:31.0:1616513072.601998:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (32)-- 00000800:00000200:19.0:1616513072.601998:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (32)-- 00000800:00000200:7.0:1616513072.601999:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (32)-- 00000800:00000200:19.0:1616513072.602000:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)++ 00000400:00000200:3.0:1616513072.602000:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 90496 00000800:00000200:19.0:1616513072.602002:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:7.0:1616513072.602002:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (30)-- 00000400:00000200:19.0:1616513072.602003:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1616513072.602003:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:7.0:1616513072.602004:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:7.0:1616513072.602005:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (29)-- 00000100:00000200:3.0:1616513072.602005:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:7.0:1616513072.602006:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:23.0:1616513072.602007:0:170091:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938944 00000400:00000200:19.0:1616513072.602008:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:19.0:1616513072.602010:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3aff0 00000400:00000200:7.0:1616513072.602010:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:3.0:1616513072.602010:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)++ 00010000:00000200:23.0:1616513072.602012:0:170091:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8f30a89200 x1681364774938944/t0(0) o400->boslfs02-MDT0001-lwp-OST000e_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:7.0:1616513072.602012:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e550 00000800:00000200:3.0:1616513072.602012:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:19.0:1616513072.602013:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:3.0:1616513072.602014:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:7.0:1616513072.602016:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:19.0:1616513072.602018:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000100:00000200:23.0:1616513072.602022:0:170091:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938944, offset 192 00000800:00000200:19.0:1616513072.602024:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.231@o2ib1 00000800:00000200:11.0:1616513072.602025:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)++ 00000100:00000200:31.0:1616513072.602026:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774938976 00000400:00000200:23.0:1616513072.602026:0:170091:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:19.0:1616513072.602026:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:19.0:1616513072.602028:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:19.0:1616513072.602029:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:11.0:1616513072.602029:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (28)-- 00010000:00000200:31.0:1616513072.602031:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8da4dd604800 x1681364774938976/t0(0) o400->boslfs02-MDT0001-lwp-OST003e_UUID@10.242.46.231@o2ib1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:23.0:1616513072.602031:0:170091:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:19.0:1616513072.602031:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:11.0:1616513072.602031:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:11.0:1616513072.602035:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:11.0:1616513072.602038:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249100 00000800:00000200:19.0:1616513072.602040:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (26)-- 00000100:00000200:31.0:1616513072.602042:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774938976, offset 192 00000400:00000200:19.0:1616513072.602042:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.0:1616513072.602042:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:23.0:1616513072.602043:0:170091:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513072.602044:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (26)-- 00000400:00000200:7.0:1616513072.602045:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:31.0:1616513072.602047:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:19.0:1616513072.602047:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:11.0:1616513072.602048:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:23.0:1616513072.602049:0:170091:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:19.0:1616513072.602049:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ff58 00000800:00000200:11.0:1616513072.602049:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000400:00000200:7.0:1616513072.602049:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:11.0:1616513072.602051:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (25)-- 00000400:00000200:7.0:1616513072.602051:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775188 00000400:00000200:31.0:1616513072.602052:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:11.0:1616513072.602053:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:11.0:1616513072.602057:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:11.0:1616513072.602058:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2000 00000400:00000200:31.0:1616513072.602064:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513072.602069:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)-- 00000800:00000200:31.0:1616513072.602070:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:7.0:1616513072.602070:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:23.0:1616513072.602071:0:170091:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513072.602072:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:23.0:1616513072.602073:0:170091:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:31.0:1616513072.602074:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513072.602075:0:170091:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (24)++ 00000800:00000200:31.0:1616513072.602077:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:23.0:1616513072.602077:0:170091:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:31.0:1616513072.602079:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:11.0:1616513072.602079:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:31.0:1616513072.602081:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:19.0:1616513072.602081:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.231@o2ib1 00000800:00000200:11.0:1616513072.602081:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:19.0:1616513072.602083:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:11.0:1616513072.602083:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:7.0:1616513072.602083:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:19.0:1616513072.602084:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (25)-- 00000400:00000200:11.0:1616513072.602084:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:19.0:1616513072.602086:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.2:1616513072.602088:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:11.0:1616513072.602088:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:11.0:1616513072.602090:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e5d8 00000800:00000200:11.0:1616513072.602092:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:31.0:1616513072.602094:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513072.602104:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513072.602105:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:31.0:1616513072.602106:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:31.0:1616513072.602112:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:31.0:1616513072.602114:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000800:00000200:31.0:1616513072.602117:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:7.2:1616513072.607927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.607987:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.607997:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.608006:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.124@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.608016:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.124@tcp1 of length 224 into portal 26 MB=0x603fc1384f140 00000400:00000200:2.0:1616513072.608023:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.124@tcp1 of length 224/224 into md 0xf97c4b19 [1] + 0 00000400:00000200:2.0:1616513072.608028:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e880 00000400:00000200:2.0:1616513072.608031:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.608034:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513072.608038:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e880 00000800:00000200:2.0:1616513072.608041:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.608043:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.608044:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.608050:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513072.608053:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513072.608053:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513072.608055:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693231054385472 00000400:00000200:26.0:1616513072.608062:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.124@tcp1 : PUT - for me 00010000:00000200:8.0:1616513072.608064:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1693231054385472/t0(0) o400->d0398c04-0eaf-ce40-eb00-2d74570c34fe@10.31.138.124@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513072.608072:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.124@tcp1 of length 224 into portal 12 MB=0x603fc1384f1c0 00000400:00000200:26.0:1616513072.608084:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.124@tcp1 of length 224/224 into md 0xf96450b9 [1] + 24328 00000100:00000200:8.0:1616513072.608084:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693231054385472, offset 192 00000400:00000200:26.0:1616513072.608089:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513072.608089:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.124@tcp1 00000100:00000200:26.0:1616513072.608092:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513072.608095:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.124@tcp1 routed destination 00000800:00000200:26.0:1616513072.608099:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.608102:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.608104:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513072.608109:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.124@tcp1(10.31.138.124@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.608115:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.608120:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.608123:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.608125:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.608129:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.608133:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513072.608134:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513072.608164:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693231054385600 00010000:00000200:18.0:1616513072.608176:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693f00 x1693231054385600/t0(0) o400->404925fa-5010-abe4-2fd7-8e8baf319488@10.31.138.124@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513072.608186:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:18.0:1616513072.608189:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693231054385600, offset 192 00000800:00000200:14.0:1616513072.608193:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513072.608195:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.124@tcp1 00000400:00000200:14.0:1616513072.608196:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:6.0:1616513072.608196:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513072.608201:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.124@tcp1 routed destination 00000400:00000200:14.0:1616513072.608203:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513072.608209:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70660 00000800:00000200:14.0:1616513072.608212:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:18.0:1616513072.608215:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.124@tcp1(10.31.138.124@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.608221:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.608227:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.608229:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.608231:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.608236:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.608245:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.608306:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.608313:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513072.608316:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513072.608322:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513072.608323:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513072.608327:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdcb0 00000800:00000200:0.0:1616513072.608331:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.635711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.635767:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.635777:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.635786:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.203@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.635796:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.203@tcp1 of length 224 into portal 26 MB=0x5ffd4de19a280 00000400:00000200:2.0:1616513072.635802:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.203@tcp1 of length 224/224 into md 0xf97c4b29 [1] + 0 00000400:00000200:2.0:1616513072.635808:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f6d8 00000400:00000200:2.0:1616513072.635811:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.635813:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513072.635818:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f6d8 00000800:00000200:26.0:1616513072.635821:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.635821:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.635824:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.635826:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513072.635838:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664607924864 00010000:00000200:8.0:1616513072.635848:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688664607924864/t0(0) o400->5e757498-95d5-79d9-2963-1af97b49fc26@10.31.132.203@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.635860:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664607924864, offset 192 00000400:00000200:8.0:1616513072.635866:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.203@tcp1 00000400:00000200:8.0:1616513072.635873:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.203@tcp1 routed destination 00000400:00000200:8.0:1616513072.635886:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.203@tcp1(10.31.132.203@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.635895:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.635900:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.635902:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.635904:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.635909:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.635914:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513072.635915:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:6.0:1616513072.635924:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513072.635929:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513072.635930:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513072.635936:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513072.635940:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71540 00000800:00000200:6.0:1616513072.635943:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513072.635945:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.636526:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.636537:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.636545:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513072.636547:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.636554:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.203@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.636563:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.203@tcp1 of length 224 into portal 12 MB=0x5ffd4de19a480 00000400:00000200:26.0:1616513072.636570:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.203@tcp1 of length 224/224 into md 0xf96450b9 [1] + 24552 00000400:00000200:26.0:1616513072.636575:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.636578:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513072.636585:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.636588:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.636590:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.636652:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664607925376 00010000:00000200:18.0:1616513072.636664:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c692880 x1688664607925376/t0(0) o400->b530049f-55d1-68f0-2889-178cc51fceb6@10.31.132.203@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.636679:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664607925376, offset 192 00000400:00000200:18.0:1616513072.636685:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.203@tcp1 00000400:00000200:18.0:1616513072.636693:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.203@tcp1 routed destination 00000400:00000200:18.0:1616513072.636709:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.203@tcp1(10.31.132.203@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.636716:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.636721:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.636724:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.636726:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.636731:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.636738:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.636792:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.636799:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513072.636802:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513072.636809:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513072.636813:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cda90 00000800:00000200:12.0:1616513072.636817:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.636820:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.649625:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.649682:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.649692:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.649703:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.107@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.649714:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.107@tcp1 of length 224 into portal 26 MB=0x601477499df00 00000400:00000200:28.0:1616513072.649721:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.107@tcp1 of length 224/224 into md 0xf97c4b49 [1] + 0 00000400:00000200:28.0:1616513072.649726:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3fed0 00000400:00000200:28.0:1616513072.649728:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.649731:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513072.649736:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fed0 00000800:00000200:0.0:1616513072.649736:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.649739:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.649742:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.649744:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513072.649753:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690256270810880 00010000:00000200:8.0:1616513072.649761:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1690256270810880/t0(0) o400->3c079d98-db48-38c6-8231-419ca271066d@10.31.142.107@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.649773:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690256270810880, offset 192 00000400:00000200:8.0:1616513072.649778:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.107@tcp1 00000400:00000200:8.0:1616513072.649785:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.107@tcp1 routed destination 00000400:00000200:8.0:1616513072.649797:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.107@tcp1(10.31.142.107@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.649803:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.649808:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.649810:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.649812:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.649816:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513072.649822:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.649822:0:48668:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513072.649834:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513072.649838:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513072.649840:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513072.649846:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513072.649850:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70110 00000800:00000200:6.0:1616513072.649853:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513072.649855:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.649869:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513072.649922:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.649931:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513072.649940:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.107@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.649949:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.107@tcp1 of length 224 into portal 12 MB=0x601477499e000 00000400:00000200:12.0:1616513072.649955:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.107@tcp1 of length 224/224 into md 0x865b9f31 [1] + 57304 00000400:00000200:12.0:1616513072.649960:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.649963:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513072.649970:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.649973:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.649975:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.649978:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.650033:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690256270811136 00010000:00000200:24.0:1616513072.650045:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57de80 x1690256270811136/t0(0) o400->6692726b-56b9-4310-002b-283c691b3ff8@10.31.142.107@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.650058:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690256270811136, offset 192 00000400:00000200:24.0:1616513072.650064:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.107@tcp1 00000400:00000200:24.0:1616513072.650071:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.107@tcp1 routed destination 00000400:00000200:24.0:1616513072.650085:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.107@tcp1(10.31.142.107@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.650091:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.650097:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.650099:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.650101:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.650106:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.650111:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.650121:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.650127:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513072.650130:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513072.650132:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513072.650136:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513072.650140:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2b28 00000800:00000200:0.0:1616513072.650144:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.661488:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513072.661543:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.661554:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513072.661563:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.163@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.661573:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.163@tcp1 of length 224 into portal 26 MB=0x604191e6fc000 00000400:00000200:12.0:1616513072.661580:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.163@tcp1 of length 224/224 into md 0xf97c4b59 [1] + 0 00000400:00000200:12.0:1616513072.661585:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3ed48 00000400:00000200:12.0:1616513072.661587:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.661590:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513072.661595:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ed48 00000800:00000200:12.0:1616513072.661597:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.661598:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.661600:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.661603:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513072.661612:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693355791597568 00010000:00000200:8.0:1616513072.661621:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1693355791597568/t0(0) o400->f4a88395-facf-06ba-8b4f-bf40254015cc@10.31.129.163@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.661633:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693355791597568, offset 192 00000400:00000200:8.0:1616513072.661639:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.163@tcp1 00000400:00000200:8.0:1616513072.661646:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.163@tcp1 routed destination 00000400:00000200:8.0:1616513072.661660:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.163@tcp1(10.31.129.163@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.661666:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.661671:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.661674:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.661676:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.661681:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513072.661687:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.661687:0:48669:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.661743:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.661750:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513072.661752:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.661759:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.661765:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70cc0 00000800:00000200:2.0:1616513072.661768:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.661771:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.661835:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.661889:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.661898:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.661907:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.163@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.661915:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.163@tcp1 of length 224 into portal 12 MB=0x604191e6fc400 00000400:00000200:28.0:1616513072.661922:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.163@tcp1 of length 224/224 into md 0x865b9f31 [1] + 57528 00000400:00000200:28.0:1616513072.661927:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.661930:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513072.661936:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.661939:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.661941:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.661944:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513072.662001:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693355791598592 00010000:00000200:24.0:1616513072.662013:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ed100 x1693355791598592/t0(0) o400->5707a491-d51c-7f4f-c3f1-c95f353740db@10.31.129.163@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.662027:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693355791598592, offset 192 00000400:00000200:24.0:1616513072.662033:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.163@tcp1 00000400:00000200:24.0:1616513072.662041:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.163@tcp1 routed destination 00000400:00000200:24.0:1616513072.662056:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.163@tcp1(10.31.129.163@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.662062:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.662068:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.662071:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.662073:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.662078:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.662083:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.662093:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.662098:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.662101:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513072.662102:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.662108:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.662112:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000800:00000200:16.0:1616513072.662115:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.672591:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513072.672647:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.672656:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513072.672665:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.216@tcp1 : PUT - for me 00000400:00000200:0.0:1616513072.672675:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.216@tcp1 of length 224 into portal 26 MB=0x5fffd448e2280 00000400:00000200:0.0:1616513072.672682:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.216@tcp1 of length 224/224 into md 0xf97c4b69 [1] + 0 00000400:00000200:0.0:1616513072.672687:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e6e8 00000400:00000200:0.0:1616513072.672692:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513072.672694:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513072.672699:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e6e8 00000800:00000200:28.0:1616513072.672700:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.672702:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.672705:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513072.672707:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.672710:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513072.672710:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513072.672714:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688838125527680 00000400:00000200:28.0:1616513072.672719:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.216@tcp1 : PUT - for me 00010000:00000200:8.0:1616513072.672722:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688838125527680/t0(0) o400->6cbbf34b-3d79-5250-56c1-d0202dab1be9@10.31.142.216@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513072.672728:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.216@tcp1 of length 224 into portal 12 MB=0x5fffd448e24c0 00000400:00000200:28.0:1616513072.672735:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.216@tcp1 of length 224/224 into md 0x865b9f31 [1] + 57752 00000100:00000200:8.0:1616513072.672739:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688838125527680, offset 192 00000400:00000200:28.0:1616513072.672740:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.672742:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513072.672749:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513072.672750:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.216@tcp1 00000800:00000200:28.0:1616513072.672752:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.672754:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513072.672756:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.216@tcp1 routed destination 00000400:00000200:8.0:1616513072.672768:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.216@tcp1(10.31.142.216@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.672774:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.672779:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.672781:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.672783:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.672787:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513072.672792:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.672794:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513072.672804:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513072.672808:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:24.0:1616513072.672811:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688838125528256 00000400:00000200:6.0:1616513072.672811:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513072.672816:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513072.672820:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71870 00010000:00000200:24.0:1616513072.672822:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eb180 x1688838125528256/t0(0) o400->29ee5b95-f120-df8a-b414-c9f464f8e4dd@10.31.142.216@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:6.0:1616513072.672823:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513072.672825:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513072.672836:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688838125528256, offset 192 00000400:00000200:24.0:1616513072.672843:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.216@tcp1 00000400:00000200:24.0:1616513072.672850:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.216@tcp1 routed destination 00000400:00000200:24.0:1616513072.672864:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.216@tcp1(10.31.142.216@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.672870:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.672875:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.672878:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513072.672880:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.672884:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.672889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.672944:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.672951:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.672954:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513072.672955:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.672961:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.672965:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf24c8 00000800:00000200:16.0:1616513072.672969:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.682837:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513072.682850:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.682859:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513072.682868:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.205@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.682877:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.205@tcp1 of length 224 into portal 26 MB=0x5ffd4f6a40540 00000400:00000200:12.0:1616513072.682883:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.205@tcp1 of length 224/224 into md 0xf97c4b79 [1] + 0 00000400:00000200:12.0:1616513072.682889:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3ee58 00000400:00000200:12.0:1616513072.682891:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.682894:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513072.682898:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ee58 00000800:00000200:28.0:1616513072.682902:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513072.682902:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513072.682904:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.682907:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513072.682911:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665019647296 00010000:00000200:8.0:1616513072.682917:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688665019647296/t0(0) o400->dcf99e2d-e871-b7bc-8565-3339a2eacbae@10.31.136.205@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.682926:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665019647296, offset 192 00000400:00000200:8.0:1616513072.682931:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.205@tcp1 00000400:00000200:8.0:1616513072.682936:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.205@tcp1 routed destination 00000400:00000200:8.0:1616513072.682948:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.205@tcp1(10.31.136.205@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.682953:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.682957:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.682959:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.682960:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.682963:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513072.682967:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.682971:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.683028:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.683035:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513072.683038:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.683044:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.683050:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70a18 00000800:00000200:2.0:1616513072.683053:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.683056:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.683062:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:6.0:1616513072.683063:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513072.683070:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.205@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.683078:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.205@tcp1 of length 224 into portal 12 MB=0x5ffd4f6a405c0 00000400:00000200:2.0:1616513072.683085:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.205@tcp1 of length 224/224 into md 0xf96450b9 [1] + 24776 00000400:00000200:2.0:1616513072.683090:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.683092:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513072.683099:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.683101:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.683103:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.683165:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665019647424 00010000:00000200:18.0:1616513072.683177:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138bf00 x1688665019647424/t0(0) o400->c58cd2ff-babd-b9c3-6ce2-17653d5632c8@10.31.136.205@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.683192:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665019647424, offset 192 00000400:00000200:18.0:1616513072.683199:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.205@tcp1 00000400:00000200:18.0:1616513072.683206:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.205@tcp1 routed destination 00000400:00000200:18.0:1616513072.683221:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.205@tcp1(10.31.136.205@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.683228:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.683233:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.683236:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.683238:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.683243:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.683248:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513072.683302:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.683309:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513072.683312:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513072.683318:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513072.683322:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc440 00000800:00000200:0.0:1616513072.683332:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.683335:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.719475:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.719530:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.719540:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.719549:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.203@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.719560:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.203@tcp1 of length 224 into portal 12 MB=0x5ffd52cf9d300 00000400:00000200:28.0:1616513072.719566:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.203@tcp1 of length 224/224 into md 0x865b9f31 [1] + 57976 00000400:00000200:28.0:1616513072.719572:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.719575:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513072.719582:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.719585:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.719585:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.719589:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.719654:0:48675:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513072.719666:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665931240192 00010000:00000200:24.0:1616513072.719679:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ef980 x1688665931240192/t0(0) o400->56a7607b-4e8b-564c-b016-726623fa2628@10.31.136.203@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.719693:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665931240192, offset 192 00000400:00000200:24.0:1616513072.719700:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.203@tcp1 00000400:00000200:24.0:1616513072.719708:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.203@tcp1 routed destination 00000800:00000200:14.0:1616513072.719711:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.719722:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513072.719724:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.203@tcp1(10.31.136.203@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513072.719725:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513072.719730:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513072.719733:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.203@tcp1 : PUT - for me 00000800:00000200:24.0:1616513072.719736:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.719739:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513072.719741:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513072.719742:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.203@tcp1 of length 224 into portal 26 MB=0x5ffd52cf9d280 00000800:00000200:24.0:1616513072.719746:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513072.719749:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.203@tcp1 of length 224/224 into md 0xf97c4b99 [1] + 0 00000800:00000200:7.2:1616513072.719751:0:48675:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513072.719754:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e3b8 00000400:00000200:14.0:1616513072.719757:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.719760:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513072.719764:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e3b8 00000800:00000200:14.0:1616513072.719768:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513072.719771:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513072.719773:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.719775:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.719782:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513072.719784:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513072.719790:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513072.719793:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:14.0:1616513072.719795:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.719798:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513072.719851:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665931240064 00010000:00000200:8.0:1616513072.719862:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688665931240064/t0(0) o400->99ce7c45-98c3-6f37-d2cc-8c4d17cd5bea@10.31.136.203@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.719888:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665931240064, offset 192 00000400:00000200:8.0:1616513072.719894:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.203@tcp1 00000400:00000200:8.0:1616513072.719901:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.203@tcp1 routed destination 00000400:00000200:8.0:1616513072.719915:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.203@tcp1(10.31.136.203@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.719921:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.719926:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.719929:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.719931:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.719936:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.719941:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513072.719942:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513072.719994:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513072.720001:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513072.720004:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513072.720011:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513072.720016:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70ff0 00000800:00000200:0.0:1616513072.720020:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513072.720023:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.783265:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513072.783279:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513072.783289:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513072.783295:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:6.0:1616513072.783299:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.10@tcp1 : PUT - for me 00000400:00000200:6.0:1616513072.783310:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.10@tcp1 of length 224 into portal 26 MB=0x5ffdcff7b8140 00000400:00000200:6.0:1616513072.783317:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.10@tcp1 of length 224/224 into md 0xf97c4ba9 [1] + 0 00000400:00000200:6.0:1616513072.783322:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3fc28 00000400:00000200:6.0:1616513072.783325:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513072.783328:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:6.0:1616513072.783333:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fc28 00000800:00000200:6.0:1616513072.783336:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513072.783339:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513072.783341:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513072.783357:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699527725376 00010000:00000200:8.0:1616513072.783369:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688699527725376/t0(0) o400->0c7a03b5-f6a6-b269-30e3-ac43e5f976b4@10.31.136.10@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.783395:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699527725376, offset 192 00000400:00000200:8.0:1616513072.783401:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.10@tcp1 00000800:00000200:11.2:1616513072.783407:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:8.0:1616513072.783410:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.10@tcp1 routed destination 00000400:00000200:8.0:1616513072.783426:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.10@tcp1(10.31.136.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.783430:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.783433:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513072.783461:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.783467:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.783469:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513072.783471:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.88@tcp1 : PUT - for me 00000800:00000200:8.0:1616513072.783471:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.783476:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513072.783480:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.88@tcp1 of length 224 into portal 26 MB=0x603669a6dfb00 00000800:00000200:7.2:1616513072.783482:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513072.783483:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513072.783486:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.88@tcp1 of length 224/224 into md 0xf97c4bb9 [1] + 0 00000400:00000200:12.0:1616513072.783491:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f980 00000400:00000200:12.0:1616513072.783494:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513072.783498:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513072.783501:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513072.783510:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f980 00000800:00000200:28.0:1616513072.783513:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513072.783514:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513072.783517:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513072.783519:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513072.783521:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513072.783521:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692589072710400 00000400:00000200:28.0:1616513072.783522:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.88@tcp1 : PUT - for me 00010000:00000200:8.0:1616513072.783526:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1692589072710400/t0(0) o400->caafb5d8-d29d-670d-2c04-05b651218fb1@10.31.140.88@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513072.783530:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.88@tcp1 of length 224 into portal 12 MB=0x603669a6dfb80 00000400:00000200:28.0:1616513072.783536:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.88@tcp1 of length 224/224 into md 0x865b9f31 [1] + 58200 00000800:00000200:26.0:1616513072.783536:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:8.0:1616513072.783541:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692589072710400, offset 192 00000400:00000200:28.0:1616513072.783542:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.783543:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:8.0:1616513072.783546:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.88@tcp1 00000100:00000200:28.0:1616513072.783548:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:26.0:1616513072.783548:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513072.783551:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.88@tcp1 routed destination 00000400:00000200:26.0:1616513072.783555:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.783560:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70ff0 00000800:00000200:26.0:1616513072.783564:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513072.783565:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.88@tcp1(10.31.140.88@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513072.783567:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513072.783569:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.783570:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513072.783572:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.783574:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513072.783574:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.783576:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.783578:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513072.783578:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513072.783580:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513072.783581:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513072.783585:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513072.783587:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.783594:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.10@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.783600:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.10@tcp1 of length 224 into portal 12 MB=0x5ffdcff7b82c0 00000400:00000200:28.0:1616513072.783604:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.10@tcp1 of length 224/224 into md 0x865b9f31 [1] + 58424 00000400:00000200:28.0:1616513072.783608:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.783610:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513072.783615:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513072.783617:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513072.783619:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.783621:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.783629:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.783631:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:24.0:1616513072.783631:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692589072710528 00000400:00000200:28.0:1616513072.783636:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.783640:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71870 00000100:00000200:4.0:1616513072.783642:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699527725760 00000800:00000200:28.0:1616513072.783643:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:24.0:1616513072.783644:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ebf00 x1692589072710528/t0(0) o400->dee1a039-9dc6-32f1-ff4e-1491c12d6f74@10.31.140.88@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513072.783646:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:4.0:1616513072.783655:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eba80 x1688699527725760/t0(0) o400->9a21ead5-d1ef-6499-92db-4592b44d037b@10.31.136.10@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513072.783659:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692589072710528, offset 192 00000400:00000200:24.0:1616513072.783665:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.88@tcp1 00000100:00000200:4.0:1616513072.783669:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699527725760, offset 192 00000400:00000200:24.0:1616513072.783673:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.88@tcp1 routed destination 00000400:00000200:4.0:1616513072.783675:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.10@tcp1 00000400:00000200:4.0:1616513072.783682:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.10@tcp1 routed destination 00000400:00000200:24.0:1616513072.783688:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.88@tcp1(10.31.140.88@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513072.783695:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:4.0:1616513072.783696:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.10@tcp1(10.31.136.10@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.783703:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513072.783704:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.783708:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513072.783708:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513072.783710:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513072.783711:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.783713:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513072.783716:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513072.783717:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513072.783721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513072.783723:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513072.783731:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513072.783733:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513072.783736:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:4.0:1616513072.783738:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513072.783739:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:6.0:1616513072.783739:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513072.783740:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513072.783743:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513072.783745:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513072.783746:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:4.0:1616513072.783748:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bba90 00000400:00000200:14.0:1616513072.783753:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2aa0 00000800:00000200:4.0:1616513072.783753:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513072.783757:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.784280:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.784290:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.784296:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513072.784299:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.784304:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.15@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.784313:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.15@tcp1 of length 224 into portal 26 MB=0x60393d8892340 00000800:00000200:11.2:1616513072.784318:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513072.784320:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.15@tcp1 of length 224/224 into md 0xf97c4bc9 [1] + 0 00000400:00000200:26.0:1616513072.784324:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f7e8 00000400:00000200:26.0:1616513072.784327:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513072.784329:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513072.784330:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513072.784335:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f7e8 00000800:00000200:26.0:1616513072.784338:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513072.784339:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513072.784340:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513072.784341:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513072.784342:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513072.784348:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.15@tcp1 : PUT - for me 00000100:00000200:12.0:1616513072.784356:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783388205888 00000400:00000200:16.0:1616513072.784357:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.15@tcp1 of length 224 into portal 12 MB=0x60393d8892680 00000400:00000200:16.0:1616513072.784363:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.15@tcp1 of length 224/224 into md 0x865b9f31 [1] + 58648 00010000:00000200:12.0:1616513072.784366:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1692783388205888/t0(0) o400->2b33e6f2-fafb-126e-cd84-36d27a8291e2@10.31.140.15@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513072.784369:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.784371:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513072.784378:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513072.784380:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783388205888, offset 192 00000800:00000200:16.0:1616513072.784381:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.784383:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:12.0:1616513072.784385:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.15@tcp1 00000400:00000200:12.0:1616513072.784392:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.15@tcp1 routed destination 00000100:00000200:4.0:1616513072.784393:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783388206720 00010000:00000200:4.0:1616513072.784400:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e0000 x1692783388206720/t0(0) o400->782f0885-f49d-1859-9327-34711524682d@10.31.140.15@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513072.784407:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.15@tcp1(10.31.140.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513072.784412:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783388206720, offset 192 00000800:00000200:12.0:1616513072.784414:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:4.0:1616513072.784417:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.15@tcp1 00000800:00000200:12.0:1616513072.784420:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.784422:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:4.0:1616513072.784423:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.15@tcp1 routed destination 00000800:00000200:12.0:1616513072.784424:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513072.784429:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.784434:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513072.784444:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:12.0:1616513072.784446:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:6.0:1616513072.784447:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:4.0:1616513072.784447:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.15@tcp1(10.31.140.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:6.0:1616513072.784449:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513072.784451:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:6.0:1616513072.784453:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:4.0:1616513072.784453:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:6.0:1616513072.784457:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240dd0 00000800:00000200:4.0:1616513072.784458:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513072.784459:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513072.784460:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.784462:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.784465:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.784471:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.784481:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.784485:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.784486:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513072.784489:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513072.784491:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.784494:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bba90 00000800:00000200:28.0:1616513072.784497:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.798243:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513072.798298:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513072.798307:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513072.798316:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.129@tcp1 : PUT - for me 00000400:00000200:20.0:1616513072.798327:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.129@tcp1 of length 224 into portal 26 MB=0x604a35d2a9a80 00000400:00000200:20.0:1616513072.798333:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.129@tcp1 of length 224/224 into md 0xf97c4be9 [1] + 0 00000400:00000200:20.0:1616513072.798338:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3fa90 00000400:00000200:20.0:1616513072.798341:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513072.798345:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513072.798350:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fa90 00000800:00000200:20.0:1616513072.798353:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.798353:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513072.798356:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513072.798357:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513072.798411:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949549517440 00010000:00000200:12.0:1616513072.798422:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1693949549517440/t0(0) o400->34e76dc5-6071-d702-d93e-e0bfe11dbd89@10.31.140.129@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513072.798448:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949549517440, offset 192 00000400:00000200:12.0:1616513072.798454:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.129@tcp1 00000400:00000200:12.0:1616513072.798462:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.129@tcp1 routed destination 00000400:00000200:12.0:1616513072.798477:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.129@tcp1(10.31.140.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513072.798477:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.798484:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513072.798489:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.798493:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513072.798495:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513072.798499:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:12.0:1616513072.798516:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513072.798538:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.798548:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.798557:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.129@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.798566:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.129@tcp1 of length 224 into portal 12 MB=0x604a35d2a9d80 00000400:00000200:2.0:1616513072.798573:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.129@tcp1 of length 224/224 into md 0xf96450b9 [1] + 25000 00000400:00000200:2.0:1616513072.798578:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.798580:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513072.798588:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513072.798591:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513072.798591:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513072.798598:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513072.798599:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513072.798601:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513072.798602:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.798609:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.798614:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240000 00000800:00000200:26.0:1616513072.798617:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513072.798654:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949549518208 00010000:00000200:18.0:1616513072.798666:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138c380 x1693949549518208/t0(0) o400->f3f3373b-54f9-3db2-8230-dc89b9608b9d@10.31.140.129@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513072.798680:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949549518208, offset 192 00000400:00000200:18.0:1616513072.798687:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.129@tcp1 00000400:00000200:18.0:1616513072.798694:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.129@tcp1 routed destination 00000400:00000200:18.0:1616513072.798710:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.129@tcp1(10.31.140.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.798716:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.798722:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.798725:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.798729:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.798734:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.798739:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.798749:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513072.798756:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513072.798759:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513072.798766:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513072.798770:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd320 00000800:00000200:4.0:1616513072.798774:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513072.798776:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.812698:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513072.812710:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513072.812720:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:6.0:1616513072.812729:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.120@tcp1 : PUT - for me 00000400:00000200:6.0:1616513072.812738:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.120@tcp1 of length 224 into portal 26 MB=0x6012e12b50c80 00000400:00000200:6.0:1616513072.812745:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.120@tcp1 of length 224/224 into md 0xf97c4bf9 [1] + 0 00000400:00000200:6.0:1616513072.812750:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f188 00000400:00000200:6.0:1616513072.812752:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513072.812756:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:6.0:1616513072.812761:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f188 00000800:00000200:6.0:1616513072.812764:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.812764:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513072.812767:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513072.812769:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513072.812776:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690147254242432 00010000:00000200:12.0:1616513072.812782:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1690147254242432/t0(0) o400->d464eee0-fadd-befe-393d-238f7e6ebc71@10.31.140.120@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513072.812794:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690147254242432, offset 192 00000400:00000200:12.0:1616513072.812798:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.120@tcp1 00000400:00000200:12.0:1616513072.812804:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.120@tcp1 routed destination 00000400:00000200:12.0:1616513072.812818:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.120@tcp1(10.31.140.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.812823:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513072.812828:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.812830:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.812832:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513072.812835:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513072.812839:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.812839:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.812892:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.812899:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513072.812901:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.812908:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.812914:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240a18 00000800:00000200:26.0:1616513072.812917:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.812920:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.813328:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513072.813384:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.813393:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.813402:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.120@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.813411:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.120@tcp1 of length 224 into portal 12 MB=0x6012e12b51340 00000400:00000200:16.0:1616513072.813417:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.120@tcp1 of length 224/224 into md 0x865b9f31 [1] + 58872 00000400:00000200:16.0:1616513072.813423:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.813425:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513072.813431:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513072.813456:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.813461:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.813465:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513072.813495:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690147254244160 00010000:00000200:4.0:1616513072.813508:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e7980 x1690147254244160/t0(0) o400->d96e627f-dc13-f2a0-02aa-c67fd10276f4@10.31.140.120@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513072.813522:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690147254244160, offset 192 00000400:00000200:4.0:1616513072.813528:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.120@tcp1 00000400:00000200:4.0:1616513072.813536:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.120@tcp1 routed destination 00000400:00000200:4.0:1616513072.813550:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.120@tcp1(10.31.140.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.813557:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.813562:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.813565:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.813569:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.813574:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.813577:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.813592:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513072.813595:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513072.813597:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513072.813603:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513072.813606:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba3b8 00000800:00000200:4.0:1616513072.813609:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513072.813611:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.828211:0:48687:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.828267:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.828276:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:6.0:1616513072.828278:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513072.828285:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.80@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.828295:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.80@tcp1 of length 224 into portal 26 MB=0x60182bf10c040 00000400:00000200:2.0:1616513072.828304:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.80@tcp1 of length 224/224 into md 0xf97c4c19 [1] + 0 00000400:00000200:2.0:1616513072.828309:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e220 00000400:00000200:2.0:1616513072.828312:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.828314:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513072.828319:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e220 00000800:00000200:2.0:1616513072.828322:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.828324:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.828326:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513072.828391:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510923186240 00010000:00000200:12.0:1616513072.828401:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1690510923186240/t0(0) o400->65edb78c-0d5b-ddb7-1a20-5414406b97e5@10.31.133.80@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513072.828415:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510923186240, offset 192 00000400:00000200:12.0:1616513072.828421:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.80@tcp1 00000400:00000200:12.0:1616513072.828429:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.80@tcp1 routed destination 00000400:00000200:12.0:1616513072.828458:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.80@tcp1(10.31.133.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.828464:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513072.828471:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.828473:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.828475:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513072.828480:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.828485:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513072.828487:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513072.828542:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.828549:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513072.828552:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.828558:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.828566:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240c38 00000800:00000200:26.0:1616513072.828570:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.828573:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.828989:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513072.829045:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513072.829054:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513072.829063:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.80@tcp1 : PUT - for me 00000400:00000200:20.0:1616513072.829072:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.80@tcp1 of length 224 into portal 12 MB=0x60182bf10c200 00000400:00000200:20.0:1616513072.829078:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.80@tcp1 of length 224/224 into md 0x865b9f31 [1] + 59096 00000400:00000200:20.0:1616513072.829084:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513072.829086:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513072.829093:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513072.829096:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513072.829098:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.829100:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513072.829157:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510923186688 00010000:00000200:4.0:1616513072.829169:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06300 x1690510923186688/t0(0) o400->0932db64-aa86-3d48-984f-94d23584815a@10.31.133.80@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513072.829184:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510923186688, offset 192 00000400:00000200:4.0:1616513072.829190:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.80@tcp1 00000400:00000200:4.0:1616513072.829197:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.80@tcp1 routed destination 00000400:00000200:4.0:1616513072.829212:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.80@tcp1(10.31.133.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.829219:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513072.829224:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513072.829227:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.829229:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.829234:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.829239:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.829254:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513072.829258:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513072.829260:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513072.829265:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513072.829268:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba3b8 00000800:00000200:4.0:1616513072.829271:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513072.829273:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.837254:0:48688:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513072.837265:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513072.837274:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513072.837277:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:6.0:1616513072.837282:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.98@tcp1 : PUT - for me 00000800:00000200:2.0:1616513072.837286:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:6.0:1616513072.837292:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.98@tcp1 of length 224 into portal 26 MB=0x5f906ce1cd440 00000400:00000200:2.0:1616513072.837295:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.98@tcp1 : PUT - for me 00000400:00000200:6.0:1616513072.837299:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.98@tcp1 of length 224/224 into md 0xf97c4c29 [1] + 0 00000400:00000200:6.0:1616513072.837303:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3fa08 00000400:00000200:2.0:1616513072.837305:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.98@tcp1 of length 224 into portal 12 MB=0x5f906ce1cd4c0 00000400:00000200:6.0:1616513072.837306:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513072.837308:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:6.0:1616513072.837312:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fa08 00000800:00000200:6.0:1616513072.837315:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:6.0:1616513072.837317:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513072.837317:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.98@tcp1 of length 224/224 into md 0xf96450b9 [1] + 25224 00000800:00000200:6.0:1616513072.837319:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513072.837322:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513072.837322:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.837325:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513072.837332:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.837335:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.837337:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513072.837377:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681182506669120 00010000:00000200:12.0:1616513072.837393:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1681182506669120/t0(0) o400->269ef27e-0f4f-ed07-6ef3-109e1f96b0a0@10.31.128.98@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513072.837407:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681182506669120, offset 192 00000400:00000200:12.0:1616513072.837415:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.98@tcp1 00000100:00000200:18.0:1616513072.837417:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681182506669248 00000400:00000200:12.0:1616513072.837423:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.98@tcp1 routed destination 00010000:00000200:18.0:1616513072.837430:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138c800 x1681182506669248/t0(0) o400->89f393ec-abbd-83eb-1945-bca0d9236db6@10.31.128.98@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513072.837451:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.98@tcp1(10.31.128.98@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.837457:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:18.0:1616513072.837459:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681182506669248, offset 192 00000800:00000200:12.0:1616513072.837463:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513072.837465:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.98@tcp1 00000800:00000200:12.0:1616513072.837466:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.837468:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513072.837473:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.98@tcp1 routed destination 00000800:00000200:12.0:1616513072.837473:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.837478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513072.837480:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513072.837488:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.98@tcp1(10.31.128.98@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.837494:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.837500:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.837503:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.837505:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.837509:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.837516:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513072.837531:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.837538:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513072.837540:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513072.837541:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513072.837547:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513072.837553:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782402a8 00000800:00000200:14.0:1616513072.837556:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513072.837570:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513072.837577:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513072.837580:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513072.837586:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513072.837590:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd100 00000800:00000200:20.0:1616513072.837594:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513072.837597:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.838766:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.838835:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.838844:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513072.838853:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.81@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.838863:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.81@tcp1 of length 224 into portal 26 MB=0x60182fe68e8c0 00000800:00000200:11.2:1616513072.838868:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513072.838869:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.81@tcp1 of length 224/224 into md 0xf97c4c49 [1] + 0 00000400:00000200:26.0:1616513072.838874:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f320 00000400:00000200:26.0:1616513072.838877:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.838880:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513072.838884:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513072.838884:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f320 00000800:00000200:26.0:1616513072.838887:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.838890:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.838892:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513072.838894:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513072.838896:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513072.838902:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.81@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.838912:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.81@tcp1 of length 224 into portal 12 MB=0x60182fe68ea40 00000400:00000200:28.0:1616513072.838918:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.81@tcp1 of length 224/224 into md 0x865b9f31 [1] + 59320 00000400:00000200:28.0:1616513072.838923:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.838925:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513072.838933:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.838936:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.838938:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.838940:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513072.838945:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511985928384 00000100:00000200:8.0:1616513072.838951:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511985928768 00010000:00000200:12.0:1616513072.838955:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1690511985928384/t0(0) o400->a5d1d5ad-80e2-4844-8b39-e696ecb8b746@10.31.133.81@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:8.0:1616513072.838962:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eb600 x1690511985928768/t0(0) o400->a44eec90-e909-451a-7e90-1d6633e84811@10.31.133.81@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513072.838968:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511985928384, offset 192 00000400:00000200:12.0:1616513072.838973:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.81@tcp1 00000100:00000200:8.0:1616513072.838976:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511985928768, offset 192 00000400:00000200:12.0:1616513072.838980:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.81@tcp1 routed destination 00000400:00000200:8.0:1616513072.838982:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.81@tcp1 00000400:00000200:8.0:1616513072.838989:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.81@tcp1 routed destination 00000400:00000200:12.0:1616513072.838994:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.81@tcp1(10.31.133.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.839000:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513072.839003:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.81@tcp1(10.31.133.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.839009:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.839009:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513072.839012:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.839014:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.839014:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.839017:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513072.839018:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513072.839019:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.839023:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513072.839023:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513072.839024:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513072.839027:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513072.839030:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513072.839033:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513072.839034:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513072.839039:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513072.839043:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240440 00000800:00000200:6.0:1616513072.839045:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513072.839047:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513072.839079:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513072.839086:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513072.839089:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513072.839095:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513072.839099:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70110 00000800:00000200:20.0:1616513072.839103:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513072.839106:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.851378:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513072.851433:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.851452:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.851460:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.87@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.851469:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.87@tcp1 of length 224 into portal 12 MB=0x6054a5ff8de80 00000400:00000200:2.0:1616513072.851475:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.11.87@tcp1 of length 224/224 into md 0xf96450b9 [1] + 25448 00000400:00000200:2.0:1616513072.851480:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.851482:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513072.851485:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.851489:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.851491:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.851493:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.851503:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513072.851554:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694666856128128 00000800:00000200:28.0:1616513072.851563:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:18.0:1616513072.851566:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138cc80 x1694666856128128/t0(0) o400->075301c8-1b3b-8988-120e-30846dbaad6f@10.31.11.87@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513072.851572:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.851581:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.87@tcp1 : PUT - for me 00000100:00000200:18.0:1616513072.851581:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694666856128128, offset 192 00000400:00000200:18.0:1616513072.851587:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.87@tcp1 00000400:00000200:28.0:1616513072.851590:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.87@tcp1 of length 224 into portal 26 MB=0x6054a5ff8db00 00000400:00000200:18.0:1616513072.851595:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.87@tcp1 routed destination 00000400:00000200:28.0:1616513072.851597:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.11.87@tcp1 of length 224/224 into md 0xf97c4c59 [1] + 0 00000400:00000200:28.0:1616513072.851602:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3fe48 00000400:00000200:28.0:1616513072.851604:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.851607:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513072.851610:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.87@tcp1(10.31.11.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513072.851611:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fe48 00000800:00000200:28.0:1616513072.851616:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513072.851617:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513072.851617:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.851619:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.851621:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513072.851622:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.851625:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513072.851627:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513072.851632:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.851637:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513072.851651:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513072.851656:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513072.851658:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513072.851664:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513072.851667:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccf68 00000100:00000200:12.0:1616513072.851671:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694666856127232 00000800:00000200:6.0:1616513072.851671:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513072.851673:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:12.0:1616513072.851681:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1694666856127232/t0(0) o400->88fa82b7-c32d-cc88-a1e1-b2d8ee28a133@10.31.11.87@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513072.851694:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694666856127232, offset 192 00000400:00000200:12.0:1616513072.851700:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.87@tcp1 00000400:00000200:12.0:1616513072.851707:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.87@tcp1 routed destination 00000400:00000200:12.0:1616513072.851721:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.87@tcp1(10.31.11.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.851727:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513072.851734:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.851736:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513072.851738:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.851743:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.851748:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513072.851749:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513072.851804:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513072.851811:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513072.851813:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513072.851820:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513072.851827:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240c38 00000800:00000200:20.0:1616513072.851830:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513072.851834:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.854490:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.854544:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.854554:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513072.854563:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.149@tcp1 : PUT - for me 00000400:00000200:4.0:1616513072.854572:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.149@tcp1 of length 224 into portal 26 MB=0x603d0945f63c0 00000400:00000200:4.0:1616513072.854580:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.149@tcp1 of length 224/224 into md 0xf97c4c69 [1] + 0 00000400:00000200:4.0:1616513072.854585:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3ecc0 00000400:00000200:4.0:1616513072.854588:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513072.854591:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513072.854594:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ecc0 00000800:00000200:4.0:1616513072.854597:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.854598:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513072.854600:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513072.854604:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513072.854607:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.854608:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.854617:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.149@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.854626:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.149@tcp1 of length 224 into portal 12 MB=0x603d0945f6600 00000400:00000200:28.0:1616513072.854632:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.149@tcp1 of length 224/224 into md 0x865b9f31 [1] + 59544 00000400:00000200:28.0:1616513072.854637:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.854639:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513072.854646:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.854649:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.854651:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513072.854657:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693044237624256 00000100:00000200:8.0:1616513072.854660:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693044237624832 00010000:00000200:12.0:1616513072.854667:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1693044237624256/t0(0) o400->38f8fead-6e40-e8d5-5cf0-74ee16d5e8b5@10.242.104.149@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:8.0:1616513072.854668:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eda00 x1693044237624832/t0(0) o400->a3bb5799-414e-7876-7ce0-854e915538c3@10.242.104.149@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513072.854680:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693044237624832, offset 192 00000100:00000200:12.0:1616513072.854682:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693044237624256, offset 192 00000400:00000200:8.0:1616513072.854685:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.149@tcp1 00000400:00000200:12.0:1616513072.854690:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.149@tcp1 00000400:00000200:8.0:1616513072.854691:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.149@tcp1 routed destination 00000400:00000200:12.0:1616513072.854700:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.149@tcp1 routed destination 00000400:00000200:8.0:1616513072.854705:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.149@tcp1(10.242.104.149@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.854711:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513072.854715:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.149@tcp1(10.242.104.149@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.854721:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.854722:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.854724:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.854727:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.854727:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513072.854730:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.854731:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513072.854732:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513072.854736:0:48690:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513072.854737:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.854741:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513072.854743:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513072.854791:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.854804:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513072.854807:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.854814:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513072.854814:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513072.854818:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71540 00000800:00000200:26.0:1616513072.854822:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513072.854822:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.854824:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.854825:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513072.854826:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.854831:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.854837:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240000 00000800:00000200:16.0:1616513072.854840:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.862508:0:48691:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513072.862519:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513072.862527:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:6.0:1616513072.862536:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.126@tcp1 : PUT - for me 00000400:00000200:6.0:1616513072.862544:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.126@tcp1 of length 224 into portal 26 MB=0x603a3d1441dc0 00000400:00000200:6.0:1616513072.862550:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.126@tcp1 of length 224/224 into md 0xf97c4c89 [1] + 0 00000400:00000200:6.0:1616513072.862555:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f078 00000400:00000200:6.0:1616513072.862557:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513072.862560:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:6.0:1616513072.862563:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f078 00000800:00000200:6.0:1616513072.862566:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513072.862569:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513072.862570:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.862573:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.862583:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513072.862592:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.126@tcp1 : PUT - for me 00000400:00000200:14.0:1616513072.862601:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.126@tcp1 of length 224 into portal 12 MB=0x603a3d1441e40 00000400:00000200:14.0:1616513072.862607:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.126@tcp1 of length 224/224 into md 0xf96450b9 [1] + 25672 00000400:00000200:14.0:1616513072.862612:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.862615:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513072.862622:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.862625:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.862627:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513072.862627:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692851985718720 00000800:00000200:14.0:1616513072.862630:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:12.0:1616513072.862638:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1692851985718720/t0(0) o400->f16d34ba-287d-81d7-d8d4-0629f7300bb9@10.31.143.126@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513072.862652:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692851985718720, offset 192 00000400:00000200:12.0:1616513072.862658:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.126@tcp1 00000400:00000200:12.0:1616513072.862665:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.126@tcp1 routed destination 00000400:00000200:12.0:1616513072.862690:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.126@tcp1(10.31.143.126@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513072.862696:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692851985718848 00000800:00000200:12.0:1616513072.862696:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513072.862701:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.862704:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.862706:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:18.0:1616513072.862707:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa3a80 x1692851985718848/t0(0) o400->b4e4811f-1ef1-b671-9025-32bd29650b2d@10.31.143.126@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513072.862710:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.862714:0:48691:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513072.862718:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513072.862721:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692851985718848, offset 192 00000400:00000200:18.0:1616513072.862729:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.126@tcp1 00000400:00000200:18.0:1616513072.862736:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.126@tcp1 routed destination 00000400:00000200:18.0:1616513072.862750:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.126@tcp1(10.31.143.126@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.862756:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.862761:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.862763:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.862765:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513072.862768:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513072.862770:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513072.862775:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513072.862776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513072.862777:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513072.862777:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.862784:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.862789:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241188 00000800:00000200:2.0:1616513072.862792:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513072.862830:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.862837:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.862840:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.862846:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.862851:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd7e8 00000800:00000200:28.0:1616513072.862855:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.862858:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.867898:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.867952:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.867962:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513072.867971:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.117@tcp1 : PUT - for me 00000400:00000200:4.0:1616513072.867981:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.117@tcp1 of length 224 into portal 26 MB=0x5fffae389df40 00000400:00000200:4.0:1616513072.867987:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.117@tcp1 of length 224/224 into md 0xf97c4c99 [1] + 0 00000400:00000200:4.0:1616513072.867992:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3ebb0 00000400:00000200:4.0:1616513072.867995:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513072.867998:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513072.868002:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ebb0 00000800:00000200:16.0:1616513072.868005:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513072.868005:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513072.868007:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513072.868012:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513072.868015:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.868016:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.868024:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.117@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.868034:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.117@tcp1 of length 224 into portal 12 MB=0x5fffae389dfc0 00000400:00000200:16.0:1616513072.868040:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.117@tcp1 of length 224/224 into md 0x865b9f31 [1] + 59768 00000400:00000200:16.0:1616513072.868046:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.868048:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513072.868055:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.868058:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.868060:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513072.868063:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688827907923776 00000100:00000200:8.0:1616513072.868073:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688827907923904 00010000:00000200:12.0:1616513072.868074:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688827907923776/t0(0) o400->c291518d-f94a-a576-43a8-873e7bcd9cc4@10.31.138.117@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:8.0:1616513072.868083:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e4c80 x1688827907923904/t0(0) o400->f31c87f9-b7c5-e5e1-3519-96cfb5426089@10.31.138.117@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513072.868088:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688827907923776, offset 192 00000400:00000200:12.0:1616513072.868095:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.117@tcp1 00000100:00000200:8.0:1616513072.868098:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688827907923904, offset 192 00000400:00000200:12.0:1616513072.868102:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.117@tcp1 routed destination 00000400:00000200:8.0:1616513072.868104:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.117@tcp1 00000400:00000200:8.0:1616513072.868111:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.117@tcp1 routed destination 00000400:00000200:12.0:1616513072.868117:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.117@tcp1(10.31.138.117@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.868124:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513072.868125:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.117@tcp1(10.31.138.117@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.868132:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.868132:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513072.868135:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.868137:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.868137:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.868140:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.868141:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.868142:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513072.868146:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513072.868148:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513072.868149:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513072.868150:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513072.868202:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.868209:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513072.868211:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.868218:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513072.868223:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513072.868224:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:26.0:1616513072.868227:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.868230:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513072.868230:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513072.868233:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513072.868234:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513072.868239:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513072.868243:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70660 00000800:00000200:20.0:1616513072.868247:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.870056:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513072.870111:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.870120:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.870129:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.5@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.870137:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.5@tcp1 of length 224 into portal 26 MB=0x5ffd422439100 00000400:00000200:16.0:1616513072.870144:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.5@tcp1 of length 224/224 into md 0xf97c4ca9 [1] + 0 00000400:00000200:16.0:1616513072.870148:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e660 00000400:00000200:16.0:1616513072.870151:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.870154:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513072.870158:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e660 00000800:00000200:16.0:1616513072.870161:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.870162:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.870164:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513072.870166:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513072.870220:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661456556288 00010000:00000200:12.0:1616513072.870231:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688661456556288/t0(0) o400->d0ad09cf-3ce5-f809-7bb8-742a0dfffcc6@10.31.133.5@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513072.870246:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661456556288, offset 192 00000400:00000200:12.0:1616513072.870252:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.5@tcp1 00000400:00000200:12.0:1616513072.870259:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.5@tcp1 routed destination 00000400:00000200:12.0:1616513072.870274:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.5@tcp1(10.31.133.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.870280:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513072.870286:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.870289:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.870291:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513072.870296:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.870301:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513072.870302:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:6.0:1616513072.870313:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513072.870318:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513072.870320:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513072.870325:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513072.870330:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241dc0 00000800:00000200:6.0:1616513072.870333:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513072.870335:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:23.0:1616513072.870543:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:23.0:1616513072.870551:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513072.870804:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513072.870863:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.870872:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513072.870880:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.5@tcp1 : PUT - for me 00000400:00000200:4.0:1616513072.870888:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.5@tcp1 of length 224 into portal 12 MB=0x5ffd422439280 00000400:00000200:4.0:1616513072.870894:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.5@tcp1 of length 224/224 into md 0x865b9f31 [1] + 59992 00000400:00000200:4.0:1616513072.870899:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513072.870901:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513072.870908:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513072.870912:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513072.870914:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513072.870916:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661456556672 00000800:00000200:20.0:1616513072.870917:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513072.870922:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8542eeb180 x1688661456556672/t0(0) o400->9ad9993d-fcef-89ee-ab8d-4588af352b72@10.31.133.5@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513072.870932:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661456556672, offset 192 00000400:00000200:8.0:1616513072.870936:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.5@tcp1 00000400:00000200:8.0:1616513072.870942:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.5@tcp1 routed destination 00000400:00000200:8.0:1616513072.870954:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.5@tcp1(10.31.133.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.870959:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.870963:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.870965:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.870967:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.870969:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.870975:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.870985:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.870990:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.870992:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513072.870995:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513072.870998:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.871002:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71980 00000800:00000200:28.0:1616513072.871006:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.902538:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.902594:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.902605:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513072.902614:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.69@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.902624:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.69@tcp1 of length 224 into portal 26 MB=0x5ffd5705fde80 00000400:00000200:26.0:1616513072.902632:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.69@tcp1 of length 224/224 into md 0xf97c4cb9 [1] + 0 00000400:00000200:26.0:1616513072.902637:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e198 00000400:00000200:26.0:1616513072.902640:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.902643:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513072.902648:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e198 00000800:00000200:14.0:1616513072.902649:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.902651:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.902654:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513072.902657:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513072.902660:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.902661:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513072.902669:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.69@tcp1 : PUT - for me 00000400:00000200:14.0:1616513072.902680:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.69@tcp1 of length 224 into portal 12 MB=0x5ffd5705fdf80 00000400:00000200:14.0:1616513072.902687:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.69@tcp1 of length 224/224 into md 0xf96450b9 [1] + 25896 00000400:00000200:14.0:1616513072.902692:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513072.902695:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513072.902703:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.902707:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.902709:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513072.902713:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667062001280 00010000:00000200:12.0:1616513072.902723:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688667062001280/t0(0) o400->171ed33b-320f-b623-b73d-4ca6668cf2b2@10.31.136.69@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513072.902736:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667062001280, offset 192 00000400:00000200:12.0:1616513072.902743:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.69@tcp1 00000400:00000200:12.0:1616513072.902750:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.69@tcp1 routed destination 00000400:00000200:12.0:1616513072.902765:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.69@tcp1(10.31.136.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513072.902768:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667062001536 00000800:00000200:12.0:1616513072.902771:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513072.902777:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.902779:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:18.0:1616513072.902780:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5580 x1688667062001536/t0(0) o400->e157e41a-febf-6711-b67a-5925d4da4dc3@10.31.136.69@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513072.902781:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513072.902786:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.902791:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:18.0:1616513072.902793:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667062001536, offset 192 00000100:00000200:12.0:1616513072.902793:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513072.902799:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.69@tcp1 00000400:00000200:18.0:1616513072.902807:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.69@tcp1 routed destination 00000400:00000200:18.0:1616513072.902821:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.69@tcp1(10.31.136.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.902827:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.902832:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.902835:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.902837:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.902841:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513072.902844:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513072.902849:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513072.902850:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513072.902853:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.902854:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513072.902858:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513072.902863:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782407f8 00000800:00000200:4.0:1616513072.902864:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513072.902866:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513072.902870:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513072.902872:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513072.902878:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513072.902881:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd870 00000800:00000200:4.0:1616513072.902885:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513072.902887:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.902893:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513072.902900:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.139@tcp1 : PUT - for me 00000400:00000200:4.0:1616513072.902907:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.139@tcp1 of length 224 into portal 26 MB=0x5ffd9184d7580 00000400:00000200:4.0:1616513072.902913:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.139@tcp1 of length 224/224 into md 0xf97c4cc9 [1] + 0 00000800:00000200:20.0:1616513072.902917:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513072.902917:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e088 00000400:00000200:4.0:1616513072.902919:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513072.902921:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513072.902925:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e088 00000800:00000200:4.0:1616513072.902927:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.902929:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513072.902931:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513072.902936:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688682764268928 00010000:00000200:12.0:1616513072.902942:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688682764268928/t0(0) o400->dcae5647-90df-6318-747e-8320adb1110b@10.31.129.139@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513072.902953:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688682764268928, offset 192 00000400:00000200:12.0:1616513072.902958:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.139@tcp1 00000400:00000200:12.0:1616513072.902964:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.139@tcp1 routed destination 00000400:00000200:12.0:1616513072.902986:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.139@tcp1(10.31.129.139@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.902992:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513072.902997:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.903000:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.903001:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513072.903004:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513072.903009:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.903009:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513072.903019:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.903023:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513072.903026:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513072.903029:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513072.903031:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513072.903036:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ff0 00000800:00000200:14.0:1616513072.903039:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.903068:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.903121:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.903130:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513072.903132:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513072.903139:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.139@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.903147:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.139@tcp1 of length 224 into portal 12 MB=0x5ffd9184d7980 00000400:00000200:28.0:1616513072.903152:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.139@tcp1 of length 224/224 into md 0x865b9f31 [1] + 60216 00000400:00000200:28.0:1616513072.903157:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.903161:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513072.903168:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.903170:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.903172:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513072.903187:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688682764269952 00010000:00000200:8.0:1616513072.903198:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ee300 x1688682764269952/t0(0) o400->38fc30b0-fccd-0682-9d82-79eeaf32ad96@10.31.129.139@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513072.903211:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688682764269952, offset 192 00000400:00000200:8.0:1616513072.903217:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.139@tcp1 00000400:00000200:8.0:1616513072.903224:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.139@tcp1 routed destination 00000400:00000200:8.0:1616513072.903238:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.139@tcp1(10.31.129.139@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.903244:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.903249:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.903252:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.903254:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.903258:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.903263:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.903271:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513072.903275:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513072.903276:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513072.903281:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513072.903281:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513072.903283:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70ee0 00000800:00000200:4.0:1616513072.903285:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.918407:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513072.918464:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513072.918473:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513072.918482:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.127@tcp1 : PUT - for me 00000400:00000200:20.0:1616513072.918492:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.127@tcp1 of length 224 into portal 26 MB=0x602bed5437b80 00000400:00000200:20.0:1616513072.918499:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.127@tcp1 of length 224/224 into md 0xf97c4ce9 [1] + 0 00000400:00000200:20.0:1616513072.918504:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3fd38 00000400:00000200:20.0:1616513072.918506:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513072.918509:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513072.918513:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fd38 00000800:00000200:28.0:1616513072.918517:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513072.918517:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513072.918519:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513072.918525:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.918528:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513072.918528:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513072.918537:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.127@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.918547:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.127@tcp1 of length 224 into portal 12 MB=0x602bed5437c00 00000400:00000200:28.0:1616513072.918553:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.127@tcp1 of length 224/224 into md 0x865b9f31 [1] + 60440 00000400:00000200:28.0:1616513072.918558:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.918561:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513072.918570:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.918573:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.918575:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513072.918586:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691868505275392 00000100:00000200:12.0:1616513072.918587:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691868505275264 00010000:00000200:8.0:1616513072.918596:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf81b00 x1691868505275392/t0(0) o400->611d6d2d-2340-0152-991e-bc6d40fcd8ea@10.31.142.127@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:12.0:1616513072.918598:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1691868505275264/t0(0) o400->730aefce-2fd3-9a60-f6fc-f4c07dd0a0f6@10.31.142.127@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.918609:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691868505275392, offset 192 00000100:00000200:12.0:1616513072.918612:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691868505275264, offset 192 00000400:00000200:8.0:1616513072.918615:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.127@tcp1 00000400:00000200:12.0:1616513072.918620:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.127@tcp1 00000400:00000200:8.0:1616513072.918622:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.127@tcp1 routed destination 00000400:00000200:12.0:1616513072.918631:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.127@tcp1 routed destination 00000400:00000200:8.0:1616513072.918636:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.127@tcp1(10.31.142.127@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.918642:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513072.918646:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.127@tcp1(10.31.142.127@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.918657:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513072.918657:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.918660:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.918661:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513072.918663:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.918665:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.918666:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513072.918668:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.918672:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513072.918674:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513072.918678:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513072.918679:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513072.918730:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.918738:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513072.918741:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.918747:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:4.0:1616513072.918748:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513072.918751:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71a90 00000800:00000200:4.0:1616513072.918754:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513072.918755:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513072.918757:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.918758:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513072.918759:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513072.918763:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513072.918767:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240b28 00000800:00000200:4.0:1616513072.918770:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.920477:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513072.920484:0:48697:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513072.920495:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513072.920504:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:28.0:1616513072.920510:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513072.920513:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.220@tcp1 : PUT - for me 00000800:00000200:28.0:1616513072.920518:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:10.0:1616513072.920522:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.220@tcp1 of length 224 into portal 26 MB=0x6000b6e247680 00000400:00000200:28.0:1616513072.920528:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.220@tcp1 : PUT - for me 00000400:00000200:10.0:1616513072.920528:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.220@tcp1 of length 224/224 into md 0xf97c4cf9 [1] + 0 00000400:00000200:10.0:1616513072.920533:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f5c8 00000400:00000200:10.0:1616513072.920535:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.920537:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.220@tcp1 of length 224 into portal 12 MB=0x6000b6e247700 00000100:00000200:10.0:1616513072.920538:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513072.920542:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f5c8 00000400:00000200:28.0:1616513072.920543:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.220@tcp1 of length 224/224 into md 0x865b9f31 [1] + 60664 00000800:00000200:10.0:1616513072.920545:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513072.920548:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:10.0:1616513072.920548:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513072.920550:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:28.0:1616513072.920551:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513072.920552:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513072.920558:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.920561:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.920563:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.920566:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513072.920567:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688898952787712 00010000:00000200:8.0:1616513072.920588:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf80480 x1688898952787712/t0(0) o400->5afb2893-04e4-68b5-83a9-3cdff71873b0@10.31.136.220@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513072.920598:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688898952787712, offset 192 00000400:00000200:8.0:1616513072.920602:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.220@tcp1 00000400:00000200:8.0:1616513072.920608:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.220@tcp1 routed destination 00000400:00000200:8.0:1616513072.920620:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.220@tcp1(10.31.136.220@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.920625:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:12.0:1616513072.920628:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688898952787584 00000800:00000200:8.0:1616513072.920629:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.920632:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.920633:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.920636:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:12.0:1616513072.920638:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688898952787584/t0(0) o400->81fa5f41-368b-a5d0-0a83-df9f29f4373f@10.31.136.220@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513072.920640:0:48697:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513072.920651:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688898952787584, offset 192 00000400:00000200:12.0:1616513072.920656:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.220@tcp1 00000400:00000200:12.0:1616513072.920664:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.220@tcp1 routed destination 00000400:00000200:12.0:1616513072.920677:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.220@tcp1(10.31.136.220@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.920684:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513072.920689:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.920691:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513072.920692:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513072.920693:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513072.920698:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513072.920699:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513072.920701:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513072.920702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513072.920704:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513072.920708:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.920712:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71760 00000800:00000200:26.0:1616513072.920716:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513072.920719:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513072.920765:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513072.920771:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513072.920774:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513072.920780:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513072.920786:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240d48 00000800:00000200:20.0:1616513072.920789:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513072.920792:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.949249:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513072.949305:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.949317:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.949326:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.30@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.949337:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.30@tcp1 of length 224 into portal 26 MB=0x6041a0f169000 00000400:00000200:16.0:1616513072.949344:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.30@tcp1 of length 224/224 into md 0xf97c4d19 [1] + 0 00000400:00000200:16.0:1616513072.949348:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3ea18 00000400:00000200:16.0:1616513072.949351:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.949354:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513072.949359:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ea18 00000800:00000200:28.0:1616513072.949361:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.949362:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.949365:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.949367:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513072.949451:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359829061632 00010000:00000200:12.0:1616513072.949461:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1693359829061632/t0(0) o400->1126cb8e-63ed-ac50-05ee-1a663e765098@10.31.129.30@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513072.949461:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513072.949472:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513072.949475:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693359829061632, offset 192 00000800:00000200:10.0:1616513072.949479:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513072.949481:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.30@tcp1 00000400:00000200:10.0:1616513072.949487:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.30@tcp1 : PUT - for me 00000400:00000200:12.0:1616513072.949488:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.30@tcp1 routed destination 00000400:00000200:10.0:1616513072.949495:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.30@tcp1 of length 224 into portal 12 MB=0x6041a0f169380 00000400:00000200:10.0:1616513072.949500:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.30@tcp1 of length 224/224 into md 0xf96450b9 [1] + 26120 00000400:00000200:12.0:1616513072.949503:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.30@tcp1(10.31.129.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513072.949505:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513072.949507:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513072.949509:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513072.949513:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513072.949514:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513072.949516:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513072.949517:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:10.0:1616513072.949518:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513072.949521:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513072.949521:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513072.949526:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.949530:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513072.949532:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513072.949579:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359829062528 00000800:00000200:26.0:1616513072.949585:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:18.0:1616513072.949591:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bde300 x1693359829062528/t0(0) o400->7149e3e7-0b9d-941a-56cc-5cb113b853fd@10.31.129.30@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:26.0:1616513072.949592:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513072.949595:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513072.949601:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:18.0:1616513072.949605:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693359829062528, offset 192 00000400:00000200:26.0:1616513072.949607:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241b18 00000800:00000200:26.0:1616513072.949610:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513072.949612:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.30@tcp1 00000800:00000200:26.0:1616513072.949613:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:18.0:1616513072.949620:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.30@tcp1 routed destination 00000400:00000200:18.0:1616513072.949635:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.30@tcp1(10.31.129.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.949642:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.949647:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.949652:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.949654:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.949659:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.949663:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513072.949718:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513072.949727:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513072.949729:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513072.949736:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513072.949740:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd8f8 00000800:00000200:20.0:1616513072.949744:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513072.949747:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.956609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.956665:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.956674:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.956683:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.150@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.956692:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.150@tcp1 of length 224 into portal 26 MB=0x5ffe1084675c0 00000400:00000200:28.0:1616513072.956698:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.150@tcp1 of length 224/224 into md 0xf97c4d39 [1] + 0 00000400:00000200:28.0:1616513072.956703:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e990 00000400:00000200:28.0:1616513072.956708:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.956711:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513072.956715:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e990 00000800:00000200:28.0:1616513072.956718:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.956719:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.956721:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.956726:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.956729:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.956730:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.956738:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.150@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.956747:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.150@tcp1 of length 224 into portal 12 MB=0x5ffe108467740 00000400:00000200:16.0:1616513072.956753:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.150@tcp1 of length 224/224 into md 0x865b9f31 [1] + 60888 00000400:00000200:16.0:1616513072.956758:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.956761:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513072.956768:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.956771:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.956773:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513072.956778:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716855113152 00000100:00000200:8.0:1616513072.956788:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716855113536 00010000:00000200:12.0:1616513072.956789:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688716855113152/t0(0) o400->d4fbfac8-12c1-01d3-b257-3c27bfc3046d@10.31.140.150@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:8.0:1616513072.956798:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e1200 x1688716855113536/t0(0) o400->0236b85c-a87e-8b06-cd87-e7e02df7ed73@10.31.140.150@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513072.956803:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688716855113152, offset 192 00000400:00000200:12.0:1616513072.956809:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.150@tcp1 00000100:00000200:8.0:1616513072.956812:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688716855113536, offset 192 00000400:00000200:12.0:1616513072.956816:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.150@tcp1 routed destination 00000400:00000200:8.0:1616513072.956819:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.150@tcp1 00000400:00000200:8.0:1616513072.956826:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.150@tcp1 routed destination 00000400:00000200:12.0:1616513072.956831:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.150@tcp1(10.31.140.150@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513072.956838:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513072.956840:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.150@tcp1(10.31.140.150@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.956846:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513072.956847:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.956850:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513072.956852:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.956852:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.956854:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513072.956856:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513072.956856:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.956861:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513072.956863:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513072.956864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513072.956866:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.956919:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513072.956926:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.956930:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513072.956931:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513072.956938:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:10.0:1616513072.956939:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513072.956942:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70aa0 00000800:00000200:10.0:1616513072.956946:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:4.0:1616513072.956946:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513072.956949:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513072.956955:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513072.956955:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513072.956961:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782405d8 00000800:00000200:10.0:1616513072.956964:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513072.961793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.961848:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.961857:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513072.961861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513072.961866:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.25@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.961877:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.25@tcp1 of length 224 into portal 12 MB=0x6039434f4d580 00000400:00000200:26.0:1616513072.961885:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.25@tcp1 of length 224/224 into md 0xf96450b9 [1] + 26344 00000400:00000200:26.0:1616513072.961891:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.961893:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513072.961900:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.961903:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513072.961903:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513072.961910:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513072.961913:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.961914:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.961923:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.4@tcp1 : PUT - for me 00000800:00000200:16.0:1616513072.961925:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513072.961932:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.4@tcp1 of length 224 into portal 26 MB=0x5ffd4f08b7680 00000800:00000200:16.0:1616513072.961934:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513072.961939:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.4@tcp1 of length 224/224 into md 0xf97c4d49 [1] + 0 00000400:00000200:16.0:1616513072.961943:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.4@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.961943:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e7f8 00000400:00000200:2.0:1616513072.961946:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.961949:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513072.961952:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.4@tcp1 of length 224 into portal 12 MB=0x5ffd4f08b7700 00000400:00000200:2.0:1616513072.961953:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e7f8 00000800:00000200:2.0:1616513072.961956:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513072.961958:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.4@tcp1 of length 224/224 into md 0x865b9f31 [1] + 61112 00000800:00000200:2.0:1616513072.961959:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.961961:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513072.961964:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.961966:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513072.961973:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.961976:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513072.961978:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.961980:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:18.0:1616513072.961981:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692784938767744 00000100:00000200:8.0:1616513072.961985:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664917374720 00000800:00000200:16.0:1616513072.961987:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513072.961988:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:8.0:1616513072.961992:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1680 x1688664917374720/t0(0) o400->fe012dbb-9ae1-1147-dca0-91fc8917097a@10.31.136.4@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:18.0:1616513072.961993:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c697980 x1692784938767744/t0(0) o400->6e0e1f77-24d2-9df6-791c-d0a6fbb1d1d6@10.31.140.25@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513072.961995:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.25@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.962001:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.25@tcp1 of length 224 into portal 26 MB=0x6039434f4d380 00000100:00000200:8.0:1616513072.962003:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664917374720, offset 192 00000400:00000200:16.0:1616513072.962007:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.25@tcp1 of length 224/224 into md 0xf97c4d69 [1] + 0 00000400:00000200:8.0:1616513072.962008:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.4@tcp1 00000400:00000200:16.0:1616513072.962011:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f870 00000400:00000200:16.0:1616513072.962013:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513072.962014:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692784938767744, offset 192 00000400:00000200:8.0:1616513072.962014:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.4@tcp1 routed destination 00000100:00000200:16.0:1616513072.962015:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513072.962024:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f870 00000800:00000200:16.0:1616513072.962026:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513072.962026:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.4@tcp1(10.31.136.4@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513072.962028:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513072.962029:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.25@tcp1 00000800:00000200:16.0:1616513072.962030:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513072.962031:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:12.0:1616513072.962033:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664917374592 00000400:00000200:18.0:1616513072.962036:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.25@tcp1 routed destination 00000800:00000200:8.0:1616513072.962036:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.962038:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513072.962039:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:12.0:1616513072.962043:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688664917374592/t0(0) o400->d14260b3-9a9d-390a-550e-841d643fbf0c@10.31.136.4@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:8.0:1616513072.962043:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.962048:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513072.962052:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.25@tcp1(10.31.140.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.962058:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:12.0:1616513072.962058:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664917374592, offset 192 00000800:00000200:18.0:1616513072.962064:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513072.962064:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.4@tcp1 00000800:00000200:18.0:1616513072.962067:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.962069:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513072.962071:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.4@tcp1 routed destination 00000800:00000200:18.0:1616513072.962074:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513072.962077:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692784938767232 00000800:00000200:11.2:1616513072.962079:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513072.962086:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.4@tcp1(10.31.136.4@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:15.0:1616513072.962091:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1692784938767232/t0(0) o400->28769519-1081-1760-2006-c3efe73e868b@10.31.140.25@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513072.962093:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513072.962099:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513072.962109:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513072.962111:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:12.0:1616513072.962116:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:10.0:1616513072.962117:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000100:00000200:15.0:1616513072.962118:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692784938767232, offset 192 00000100:00000200:12.0:1616513072.962123:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513072.962123:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:15.0:1616513072.962124:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.25@tcp1 00000400:00000200:10.0:1616513072.962125:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513072.962125:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513072.962127:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513072.962131:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:10.0:1616513072.962132:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:4.0:1616513072.962132:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513072.962133:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:15.0:1616513072.962134:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.25@tcp1 routed destination 00000400:00000200:4.0:1616513072.962135:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513072.962135:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513072.962136:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513072.962136:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f704c8 00000400:00000200:26.0:1616513072.962139:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:10.0:1616513072.962139:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513072.962142:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513072.962144:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241f58 00000400:00000200:4.0:1616513072.962146:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cda08 00000800:00000200:26.0:1616513072.962147:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513072.962150:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.25@tcp1(10.31.140.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513072.962155:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:15.0:1616513072.962156:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513072.962163:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513072.962166:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513072.962168:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513072.962173:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.962178:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513072.962183:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513072.962188:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.962192:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513072.962195:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513072.962197:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513072.962201:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513072.962206:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775980 00000800:00000200:16.0:1616513072.962209:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.980195:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513072.980251:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513072.980260:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513072.980268:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.237@tcp1 : PUT - for me 00000400:00000200:20.0:1616513072.980278:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.237@tcp1 of length 224 into portal 12 MB=0x5ffdc98d37c00 00000400:00000200:20.0:1616513072.980285:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.237@tcp1 of length 224/224 into md 0x865b9f31 [1] + 61336 00000400:00000200:20.0:1616513072.980290:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513072.980292:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513072.980299:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513072.980302:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513072.980304:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513072.980308:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513072.980320:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697805437952 00010000:00000200:8.0:1616513072.980331:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06780 x1688697805437952/t0(0) o400->048d909d-86a9-5e85-c0d2-ea0be21bd179@10.31.141.237@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513072.980353:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:8.0:1616513072.980354:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697805437952, offset 192 00000400:00000200:8.0:1616513072.980360:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.237@tcp1 00000400:00000200:8.0:1616513072.980367:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.237@tcp1 routed destination 00000400:00000200:8.0:1616513072.980381:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.237@tcp1(10.31.141.237@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513072.980388:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513072.980393:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.980396:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513072.980398:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513072.980403:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.980409:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.980418:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513072.980427:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.237@tcp1 : PUT - for me 00000400:00000200:14.0:1616513072.980458:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.237@tcp1 of length 224 into portal 26 MB=0x5ffdc98d37b80 00000800:00000200:2.0:1616513072.980463:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513072.980471:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.237@tcp1 of length 224/224 into md 0xf97c4d89 [1] + 0 00000800:00000200:2.0:1616513072.980472:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513072.980474:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513072.980476:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e110 00000400:00000200:14.0:1616513072.980479:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513072.980481:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:14.0:1616513072.980482:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513072.980485:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70908 00000400:00000200:14.0:1616513072.980487:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e110 00000800:00000200:2.0:1616513072.980489:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513072.980491:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513072.980492:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.980493:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513072.980495:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513072.980510:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697805437824 00010000:00000200:15.0:1616513072.980520:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688697805437824/t0(0) o400->25edbe19-0591-3144-d3ed-7c6ee43ff0c7@10.31.141.237@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513072.980534:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697805437824, offset 192 00000400:00000200:15.0:1616513072.980540:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.237@tcp1 00000400:00000200:15.0:1616513072.980548:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.237@tcp1 routed destination 00000400:00000200:15.0:1616513072.980563:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.237@tcp1(10.31.141.237@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513072.980569:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513072.980575:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513072.980577:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513072.980580:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513072.980585:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.980589:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513072.980592:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513072.980641:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.980648:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513072.980651:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513072.980657:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513072.980663:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774990 00000800:00000200:28.0:1616513072.980667:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513072.980669:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513072.985495:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513072.985550:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513072.985560:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513072.985569:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.192@tcp1 : PUT - for me 00000400:00000200:26.0:1616513072.985579:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.192@tcp1 of length 224 into portal 26 MB=0x605893a267d00 00000400:00000200:26.0:1616513072.985586:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.192@tcp1 of length 224/224 into md 0xf97c4d99 [1] + 0 00000400:00000200:26.0:1616513072.985590:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3fb18 00000400:00000200:26.0:1616513072.985593:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513072.985596:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513072.985601:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fb18 00000800:00000200:26.0:1616513072.985604:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513072.985604:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513072.985606:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513072.985611:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513072.985614:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.985615:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513072.985623:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.192@tcp1 : PUT - for me 00000400:00000200:2.0:1616513072.985633:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.192@tcp1 of length 224 into portal 12 MB=0x605893a267fc0 00000400:00000200:2.0:1616513072.985639:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.192@tcp1 of length 224/224 into md 0xf96450b9 [1] + 26568 00000400:00000200:2.0:1616513072.985646:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513072.985649:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513072.985656:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513072.985659:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513072.985661:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513072.985669:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694936804523264 00010000:00000200:15.0:1616513072.985680:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1694936804523264/t0(0) o400->7c3405cb-782e-23ae-8072-26e103827ba2@10.31.140.192@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513072.985707:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694936804523264, offset 192 00000400:00000200:15.0:1616513072.985713:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.192@tcp1 00000100:00000200:18.0:1616513072.985718:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694936804523968 00000400:00000200:15.0:1616513072.985722:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.192@tcp1 routed destination 00010000:00000200:18.0:1616513072.985730:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138d100 x1694936804523968/t0(0) o400->c0c3bece-bdd2-df39-b86b-e8d88e1d6d9f@10.31.140.192@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513072.985738:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.192@tcp1(10.31.140.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513072.985744:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694936804523968, offset 192 00000800:00000200:15.0:1616513072.985745:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513072.985751:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.192@tcp1 00000800:00000200:15.0:1616513072.985752:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513072.985755:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513072.985757:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513072.985758:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.192@tcp1 routed destination 00000800:00000200:15.0:1616513072.985762:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513072.985767:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513072.985769:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513072.985779:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.192@tcp1(10.31.140.192@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513072.985786:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513072.985791:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513072.985794:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513072.985796:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513072.985800:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513072.985812:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513072.985818:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513072.985820:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513072.985821:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513072.985827:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513072.985832:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775d38 00000800:00000200:18.0:1616513072.985835:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513072.985847:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513072.985901:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513072.985908:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513072.985911:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513072.985917:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513072.985921:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd6d8 00000800:00000200:4.0:1616513072.985925:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513072.985927:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513072.993014:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513072.993069:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513072.993078:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513072.993087:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.240@tcp1 : PUT - for me 00000400:00000200:28.0:1616513072.993096:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.240@tcp1 of length 224 into portal 26 MB=0x60182ef6af600 00000400:00000200:28.0:1616513072.993103:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.240@tcp1 of length 224/224 into md 0xf97c4da9 [1] + 0 00000400:00000200:28.0:1616513072.993107:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e770 00000400:00000200:28.0:1616513072.993110:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513072.993113:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513072.993118:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e770 00000800:00000200:28.0:1616513072.993121:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513072.993123:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513072.993124:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.993130:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513072.993133:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.993134:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513072.993143:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.240@tcp1 : PUT - for me 00000400:00000200:16.0:1616513072.993151:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.240@tcp1 of length 224 into portal 12 MB=0x60182ef6af6c0 00000400:00000200:16.0:1616513072.993158:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.240@tcp1 of length 224/224 into md 0x865b9f31 [1] + 61560 00000400:00000200:16.0:1616513072.993163:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513072.993165:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513072.993172:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513072.993175:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513072.993177:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513072.993181:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511734404608 00010000:00000200:15.0:1616513072.993191:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1690511734404608/t0(0) o400->a640c3e6-3b87-28d0-2969-774830db1978@10.31.130.240@tcp1:29/0 lens 224/224 e 0 to 0 dl 1616513164 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513072.993194:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511734404800 00000100:00000200:15.0:1616513072.993204:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511734404608, offset 192 00010000:00000200:8.0:1616513072.993206:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd2880 x1690511734404800/t0(0) o400->4ec1ed91-0c49-00e2-348b-ced7b4b77796@10.31.130.240@tcp1:698/0 lens 224/224 e 0 to 0 dl 1616513078 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513072.993210:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.240@tcp1 00000400:00000200:15.0:1616513072.993217:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.240@tcp1 routed destination 00000100:00000200:8.0:1616513072.993221:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511734404800, offset 192 00000400:00000200:8.0:1616513072.993228:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.240@tcp1 00000400:00000200:15.0:1616513072.993232:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.240@tcp1(10.31.130.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513072.993236:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.240@tcp1 routed destination 00000800:00000200:15.0:1616513072.993239:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513072.993245:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513072.993248:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513072.993250:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513072.993251:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.240@tcp1(10.31.130.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513072.993255:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513072.993258:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513072.993260:0:48704:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513072.993261:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513072.993264:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513072.993267:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513072.993269:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513072.993274:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513072.993279:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513072.993313:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513072.993321:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513072.993324:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513072.993331:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513072.993335:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513072.993337:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77746e8 00000800:00000200:14.0:1616513072.993341:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513072.993342:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513072.993344:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513072.993345:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513072.993346:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513072.993351:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513072.993355:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70198 00000800:00000200:20.0:1616513072.993359:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.004159:0:48705:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.004215:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.004224:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.004233:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.236@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.004241:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.236@tcp1 of length 224 into portal 26 MB=0x5ffe45c45f4c0 00000400:00000200:26.0:1616513073.004248:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.236@tcp1 of length 224/224 into md 0xf97c4dc9 [1] + 0 00000400:00000200:26.0:1616513073.004258:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3eff0 00000400:00000200:26.0:1616513073.004261:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.004264:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513073.004268:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eff0 00000800:00000200:18.0:1616513073.004270:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.004271:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.004274:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.004276:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.004290:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731149268160 00010000:00000200:15.0:1616513073.004300:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688731149268160/t0(0) o400->c5988460-4f9f-0980-2def-5a1c5a6d009c@10.31.141.236@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.004314:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688731149268160, offset 192 00000400:00000200:15.0:1616513073.004320:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.236@tcp1 00000400:00000200:15.0:1616513073.004327:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.236@tcp1 routed destination 00000400:00000200:15.0:1616513073.004343:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.236@tcp1(10.31.141.236@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.004350:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.004356:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.004359:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.004361:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.004366:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.004371:0:48705:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.004372:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513073.004425:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.004432:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.004449:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.004456:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.004462:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775210 00000800:00000200:14.0:1616513073.004466:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.004469:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.006141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.006195:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.006204:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.006213:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.236@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.006222:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.236@tcp1 of length 224 into portal 12 MB=0x5ffe45c45f780 00000400:00000200:18.0:1616513073.006228:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.236@tcp1 of length 224/224 into md 0xf96450b9 [1] + 26792 00000400:00000200:18.0:1616513073.006234:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.006236:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513073.006244:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.006247:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.006248:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.006252:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.006310:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731149268864 00010000:00000200:22.0:1616513073.006322:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d5580 x1688731149268864/t0(0) o400->e4ca5721-c012-299b-e1d8-5e9f7145ed0d@10.31.141.236@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.006337:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688731149268864, offset 192 00000400:00000200:22.0:1616513073.006343:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.236@tcp1 00000400:00000200:22.0:1616513073.006350:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.236@tcp1 routed destination 00000400:00000200:22.0:1616513073.006365:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.236@tcp1(10.31.141.236@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.006372:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.006378:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.006380:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.006383:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.006389:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.006437:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513073.006491:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513073.006498:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513073.006501:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513073.006507:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513073.006510:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba08 00000800:00000200:4.0:1616513073.006514:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513073.006517:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.015024:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.015079:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.015089:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.015098:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.51@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.015107:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.51@tcp1 of length 224 into portal 26 MB=0x5ffe5c952a6c0 00000400:00000200:14.0:1616513073.015114:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.51@tcp1 of length 224/224 into md 0xf97c4de9 [1] + 0 00000400:00000200:14.0:1616513073.015119:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f298 00000400:00000200:14.0:1616513073.015122:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.015124:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.015130:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f298 00000800:00000200:14.0:1616513073.015133:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.015133:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.015136:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.015138:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.015151:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737273784000 00010000:00000200:15.0:1616513073.015163:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688737273784000/t0(0) o400->991c7e52-e690-3788-9920-25adedbcd316@10.31.140.51@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513073.015175:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513073.015177:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688737273784000, offset 192 00000400:00000200:15.0:1616513073.015184:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.51@tcp1 00000400:00000200:15.0:1616513073.015192:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.51@tcp1 routed destination 00000400:00000200:15.0:1616513073.015208:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.51@tcp1(10.31.140.51@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.015215:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.015221:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.015224:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.015226:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.015230:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:15.0:1616513073.015231:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:15.0:1616513073.015238:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513073.015240:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.015249:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.51@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.015259:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.51@tcp1 of length 224 into portal 12 MB=0x5ffe5c952aa40 00000400:00000200:26.0:1616513073.015265:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.51@tcp1 of length 224/224 into md 0xf96450b9 [1] + 27016 00000400:00000200:26.0:1616513073.015270:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.015273:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.015279:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513073.015283:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513073.015285:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513073.015292:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513073.015294:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.015295:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.015297:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.015303:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.015310:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775650 00000800:00000200:18.0:1616513073.015313:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.015344:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737273784896 00010000:00000200:22.0:1616513073.015356:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c692400 x1688737273784896/t0(0) o400->a5a9e925-9fb9-771c-eb37-c8e71d2bb425@10.31.140.51@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.015371:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688737273784896, offset 192 00000400:00000200:22.0:1616513073.015377:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.51@tcp1 00000400:00000200:22.0:1616513073.015385:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.51@tcp1 routed destination 00000400:00000200:22.0:1616513073.015400:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.51@tcp1(10.31.140.51@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.015407:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.015413:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.015415:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.015417:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.015422:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.015469:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.015528:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.015535:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.015537:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513073.015544:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.015548:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a110 00000800:00000200:28.0:1616513073.015554:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.015557:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.022903:0:48707:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.022957:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.022967:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.022975:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.23@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.022984:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.23@tcp1 of length 224 into portal 26 MB=0x5ffd442eb45c0 00000400:00000200:14.0:1616513073.022991:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.23@tcp1 of length 224/224 into md 0xf97c4e09 [1] + 0 00000400:00000200:14.0:1616513073.022996:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3ec38 00000400:00000200:14.0:1616513073.022999:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.023002:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.023006:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ec38 00000800:00000200:14.0:1616513073.023009:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.023012:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513073.023012:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.023019:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.023021:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.023022:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.023031:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.23@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.023040:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.23@tcp1 of length 224 into portal 12 MB=0x5ffd442eb4640 00000400:00000200:26.0:1616513073.023046:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.23@tcp1 of length 224/224 into md 0xf96450b9 [1] + 27240 00000400:00000200:26.0:1616513073.023051:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.023054:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.023060:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.023063:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.023065:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.023069:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662004417984 00010000:00000200:15.0:1616513073.023080:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688662004417984/t0(0) o400->dd913108-0170-7bd3-e0bf-a9928de730e3@10.31.133.23@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.023093:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662004417984, offset 192 00000400:00000200:15.0:1616513073.023099:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.23@tcp1 00000400:00000200:15.0:1616513073.023106:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.23@tcp1 routed destination 00000400:00000200:15.0:1616513073.023120:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.23@tcp1(10.31.133.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513073.023126:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662004418112 00000800:00000200:15.0:1616513073.023126:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:22.0:1616513073.023138:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c697080 x1688662004418112/t0(0) o400->4dd9624c-ffb4-2fc3-b641-f719115000c2@10.31.133.23@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513073.023139:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.023143:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.023145:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.023150:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513073.023152:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662004418112, offset 192 00000800:00000200:7.2:1616513073.023155:0:48707:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.023157:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513073.023158:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.23@tcp1 00000400:00000200:22.0:1616513073.023166:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.23@tcp1 routed destination 00000400:00000200:22.0:1616513073.023181:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.23@tcp1(10.31.133.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.023187:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.023194:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.023196:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.023199:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.023203:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513073.023214:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.023221:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.023224:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513073.023224:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513073.023230:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.023236:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774660 00000800:00000200:18.0:1616513073.023239:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.023249:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513073.023304:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513073.023311:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513073.023314:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513073.023321:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513073.023325:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a4c8 00000800:00000200:4.0:1616513073.023329:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513073.023332:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513073.023338:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513073.023340:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513073.023342:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513073.023346:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513073.023348:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.024447:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513073.024472:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.024504:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.024513:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513073.024521:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.22@tcp1 : PUT - for me 00000400:00000200:20.0:1616513073.024530:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.22@tcp1 of length 224 into portal 12 MB=0x60182bf7bc440 00000400:00000200:20.0:1616513073.024536:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.22@tcp1 of length 224/224 into md 0x865b9f31 [1] + 61784 00000400:00000200:20.0:1616513073.024541:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513073.024544:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513073.024546:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.024551:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513073.024554:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513073.024555:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.024558:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513073.024560:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513073.024565:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.22@tcp1 : PUT - for me 00000100:00000200:8.0:1616513073.024568:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510930199616 00000400:00000200:2.0:1616513073.024573:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.22@tcp1 of length 224 into portal 26 MB=0x60182bf7bc3c0 00010000:00000200:8.0:1616513073.024578:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03600 x1690510930199616/t0(0) o400->5f373f66-755f-177d-50d8-5321a152b520@10.31.133.22@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513073.024580:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.22@tcp1 of length 224/224 into md 0xf97c4e19 [1] + 0 00000400:00000200:2.0:1616513073.024585:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3f430 00000400:00000200:2.0:1616513073.024587:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.024590:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:8.0:1616513073.024591:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510930199616, offset 192 00000400:00000200:2.0:1616513073.024595:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f430 00000400:00000200:8.0:1616513073.024596:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.22@tcp1 00000800:00000200:2.0:1616513073.024598:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.024601:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.024602:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513073.024603:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.22@tcp1 routed destination 00000800:00000200:2.0:1616513073.024605:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513073.024617:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.22@tcp1(10.31.133.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513073.024623:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513073.024628:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513073.024631:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513073.024633:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513073.024637:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.024642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.024672:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510930199488 00010000:00000200:15.0:1616513073.024683:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1690510930199488/t0(0) o400->64769791-d8e0-9f22-0f19-61228336bba1@10.31.133.22@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513073.024694:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:15.0:1616513073.024696:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510930199488, offset 192 00000800:00000200:14.0:1616513073.024700:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:15.0:1616513073.024702:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.22@tcp1 00000400:00000200:14.0:1616513073.024703:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:15.0:1616513073.024710:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.22@tcp1 routed destination 00000400:00000200:14.0:1616513073.024710:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.024714:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f715c8 00000800:00000200:14.0:1616513073.024718:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.024720:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513073.024725:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.22@tcp1(10.31.133.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.024731:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513073.024737:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.024740:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513073.024742:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513073.024746:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.024751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513073.024752:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513073.024804:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.024812:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.024815:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.024822:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.024828:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774d48 00000800:00000200:16.0:1616513073.024831:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.024836:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:21.2F:1616513073.026798:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4ea00] (20)++ 00000800:00000200:11.0:1616513073.026855:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (21)++ 00000800:00000200:11.0:1616513073.026864:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.205@o2ib1 00000400:00000200:11.0:1616513073.026873:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.205@o2ib1 : PUT - for me 00000400:00000200:11.0:1616513073.026880:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.205@o2ib1 of length 224 into portal 26 MB=0x60386f0ea54c0 00000400:00000200:11.0:1616513073.026886:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.205@o2ib1 of length 224/224 into md 0xf97c4e29 [1] + 0 00000400:00000200:11.0:1616513073.026891:0:6475:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3e4c8 00000400:00000200:11.0:1616513073.026894:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:11.0:1616513073.026897:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:11.0:1616513073.026901:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e4c8 00000800:00000200:11.0:1616513073.026905:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4ea00] (22)++ 00000800:00000200:11.0:1616513073.026908:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4ea00] (23)-- 00000800:00000200:27.0:1616513073.026909:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (23)-- 00000800:00000200:11.0:1616513073.026913:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (21)-- 00000800:00000200:21.2:1616513073.026918:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4ea00] (20)++ 00000100:00000200:15.0:1616513073.026947:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692727962653888 00010000:00000200:15.0:1616513073.026957:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1692727962653888/t0(0) o400->3759e23b-3443-c286-154b-e60843c8912c@10.242.46.205@o2ib1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.026971:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692727962653888, offset 192 00000800:00000200:19.0:1616513073.026972:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (21)++ 00000400:00000200:15.0:1616513073.026977:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.205@o2ib1 00000400:00000200:15.0:1616513073.026984:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.205@o2ib1 local destination 00000800:00000200:19.0:1616513073.026987:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.205@o2ib1 00000400:00000200:15.0:1616513073.026996:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.205@o2ib1(10.242.46.205@o2ib1:10.242.46.205@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.027002:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.205@o2ib1 00000400:00000200:19.0:1616513073.027005:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.205@o2ib1 : PUT - for me 00000800:00000200:15.0:1616513073.027008:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91ec180] -> 10.242.46.205@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.027011:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4ea00] (22)++ 00000400:00000200:19.0:1616513073.027013:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.205@o2ib1 of length 224 into portal 12 MB=0x60386f0ea5680 00000800:00000200:15.0:1616513073.027013:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4ea00] (23)++ 00000800:00000200:15.0:1616513073.027017:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4ea00] (24)-- 00000400:00000200:19.0:1616513073.027019:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.205@o2ib1 of length 224/224 into md 0x3657608d [1] + 90720 00000100:00000200:15.0:1616513073.027023:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:19.0:1616513073.027025:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:19.0:1616513073.027027:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:31.0:1616513073.027032:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (23)++ 00000800:00000200:19.0:1616513073.027036:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4ea00] (24)++ 00000800:00000200:19.0:1616513073.027039:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4ea00] (25)-- 00000800:00000200:31.0:1616513073.027040:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4ea00] (25)-- 00000400:00000200:31.0:1616513073.027043:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:19.0:1616513073.027043:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (23)-- 00000800:00000200:19.0:1616513073.027046:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (22)-- 00000400:00000200:31.0:1616513073.027050:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.205@o2ib1: PUT: OK 00000400:00000200:31.0:1616513073.027056:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77757e8 00000800:00000200:31.0:1616513073.027059:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (21)-- 00000100:00000200:3.0:1616513073.027103:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692727962654336 00010000:00000200:3.0:1616513073.027116:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8dd8bfad00 x1692727962654336/t0(0) o400->db89f005-a603-f7cc-3df4-30f0aead2f2c@10.242.46.205@o2ib1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:3.0:1616513073.027131:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692727962654336, offset 192 00000400:00000200:3.0:1616513073.027137:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.205@o2ib1 00000400:00000200:3.0:1616513073.027144:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.205@o2ib1 local destination 00000400:00000200:3.0:1616513073.027159:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.205@o2ib1(10.242.46.205@o2ib1:10.242.46.205@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513073.027166:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.205@o2ib1 00000800:00000200:3.0:1616513073.027171:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91ec180] -> 10.242.46.205@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513073.027174:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4ea00] (20)++ 00000800:00000200:3.0:1616513073.027176:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4ea00] (21)++ 00000800:00000200:3.0:1616513073.027181:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4ea00] (22)-- 00000800:00000200:21.2:1616513073.027186:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4ea00] (21)++ 00000800:00000200:11.0:1616513073.027193:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (22)++ 00000800:00000200:11.0:1616513073.027197:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4ea00] (23)-- 00000400:00000200:11.0:1616513073.027199:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:27.0:1616513073.027200:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (22)-- 00000400:00000200:11.0:1616513073.027204:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.205@o2ib1: PUT: OK 00000400:00000200:11.0:1616513073.027207:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:11.0:1616513073.027210:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (21)-- 00000800:00000200:11.2:1616513073.028001:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.028056:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.028065:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513073.028074:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.32@tcp1 : PUT - for me 00000400:00000200:20.0:1616513073.028084:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.32@tcp1 of length 224 into portal 26 MB=0x600923c779740 00000400:00000200:20.0:1616513073.028091:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.32@tcp1 of length 224/224 into md 0xf97c4e39 [1] + 0 00000400:00000200:20.0:1616513073.028096:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841de3fdc0 00000400:00000200:20.0:1616513073.028098:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513073.028101:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513073.028106:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fdc0 00000800:00000200:20.0:1616513073.028110:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513073.028110:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513073.028112:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513073.028114:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513073.028120:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689477939959616 00010000:00000200:15.0:1616513073.028126:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1689477939959616/t0(0) o400->f4dd15f9-aa79-3152-1de0-9c026f8523c1@10.31.142.32@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.028137:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689477939959616, offset 192 00000400:00000200:15.0:1616513073.028142:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.32@tcp1 00000400:00000200:15.0:1616513073.028148:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.32@tcp1 routed destination 00000800:00000200:7.2:1616513073.028157:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:15.0:1616513073.028161:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.32@tcp1(10.31.142.32@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.028166:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.028171:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.028173:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.028175:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513073.028179:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:15.0:1616513073.028183:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513073.028211:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.028221:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.028230:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.32@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.028238:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.32@tcp1 of length 224 into portal 12 MB=0x600923c779940 00000400:00000200:2.0:1616513073.028245:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.32@tcp1 of length 224/224 into md 0xf96450b9 [1] + 27464 00000400:00000200:2.0:1616513073.028250:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.028252:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513073.028259:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513073.028261:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513073.028263:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.028266:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.028270:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.028273:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513073.028274:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.028278:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.028285:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77744c8 00000800:00000200:2.0:1616513073.028288:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.028320:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689477939960128 00010000:00000200:22.0:1616513073.028343:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138d580 x1689477939960128/t0(0) o400->4411ca1e-fca6-ba01-04af-073c8b4ca166@10.31.142.32@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.028357:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689477939960128, offset 192 00000400:00000200:22.0:1616513073.028363:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.32@tcp1 00000400:00000200:22.0:1616513073.028377:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.32@tcp1 routed destination 00000400:00000200:22.0:1616513073.028391:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.32@tcp1(10.31.142.32@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.028397:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.028402:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.028405:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.028407:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.028411:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.028417:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.028470:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.028477:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.028480:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.028486:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.028491:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a908 00000800:00000200:16.0:1616513073.028495:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.028498:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:3.0:1616513073.034519:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924080, portal 4 00000100:00000200:3.0:1616513073.034525:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924080, offset 0 00000400:00000200:3.0:1616513073.034532:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:3.0:1616513073.034538:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:3.0:1616513073.034551:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513073.034557:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513073.034564:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513073.034567:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:3.0:1616513073.034569:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:3.0:1616513073.034574:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513073.034583:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513073.034638:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513073.034645:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513073.034649:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513073.034655:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513073.034662:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8f30a8b180 x1693067725924080/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513073.034674:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2990 00000800:00000200:4.0:1616513073.034678:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513073.034681:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513073.034813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513073.034870:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513073.034879:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:16.0:1616513073.034888:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513073.034896:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6266f0 00000400:00000200:16.0:1616513073.034903:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d3391 [1] + 192 00000400:00000200:16.0:1616513073.034909:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513073.034914:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8f30a8b180 x1693067725924080/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513073.034924:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513073.034930:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513073.034933:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513073.034935:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:3.0:1616513073.034940:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8f30a8b180 x1693067725924080/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513073.034953:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8f30a8b180 x1693067725924080/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513073.034963:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:7.2:1616513073.051357:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.051413:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.051423:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.051432:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.216@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.051454:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.216@tcp1 of length 224 into portal 26 MB=0x5ffe892a683c0 00000400:00000200:18.0:1616513073.051461:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.216@tcp1 of length 224/224 into md 0xf97c4e49 [1] + 0 00000400:00000200:18.0:1616513073.051466:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba330 00000800:00000200:26.0:1616513073.051468:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513073.051469:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.051473:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.051478:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba330 00000800:00000200:18.0:1616513073.051482:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.051485:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.051487:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.051541:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688749241435072 00010000:00000200:15.0:1616513073.051552:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688749241435072/t0(0) o400->92b3cf87-0b7e-d06e-9ab7-b4a9de9b743f@10.31.136.216@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.051565:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688749241435072, offset 192 00000400:00000200:15.0:1616513073.051571:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.216@tcp1 00000400:00000200:15.0:1616513073.051579:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.216@tcp1 routed destination 00000400:00000200:15.0:1616513073.051593:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.216@tcp1(10.31.136.216@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.051599:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.051605:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.051608:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.051610:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.051614:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.051619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.051621:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513073.051673:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.051681:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.051684:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.051691:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.051697:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775c28 00000800:00000200:2.0:1616513073.051701:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.051704:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.052149:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.052159:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.052167:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513073.052169:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.052178:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.216@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.052187:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.216@tcp1 of length 224 into portal 12 MB=0x5ffe892a68500 00000400:00000200:26.0:1616513073.052194:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.216@tcp1 of length 224/224 into md 0xf96450b9 [1] + 27688 00000400:00000200:26.0:1616513073.052199:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.052202:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.052209:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.052212:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.052213:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.052277:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688749241435392 00010000:00000200:22.0:1616513073.052289:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693a80 x1688749241435392/t0(0) o400->6c1db612-839e-18ab-bd61-253793d28576@10.31.136.216@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.052304:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688749241435392, offset 192 00000400:00000200:22.0:1616513073.052311:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.216@tcp1 00000400:00000200:22.0:1616513073.052319:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.216@tcp1 routed destination 00000400:00000200:22.0:1616513073.052334:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.216@tcp1(10.31.136.216@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.052341:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.052358:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.052360:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.052363:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.052367:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.052374:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513073.052428:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513073.052444:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513073.052447:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513073.052453:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513073.052457:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a000 00000800:00000200:4.0:1616513073.052461:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513073.052463:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.085234:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513073.085292:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.085302:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513073.085312:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.252@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.085321:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.252@tcp1 of length 224 into portal 26 MB=0x5ffd325651140 00000400:00000200:28.0:1616513073.085328:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.252@tcp1 of length 224/224 into md 0xf97c4e89 [1] + 0 00000400:00000200:28.0:1616513073.085333:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb430 00000400:00000200:28.0:1616513073.085336:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.085339:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513073.085344:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb430 00000800:00000200:16.0:1616513073.085347:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.085348:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.085350:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.085352:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513073.085404:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657214116160 00010000:00000200:15.0:1616513073.085413:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688657214116160/t0(0) o400->3561c29f-ea49-bb35-9249-7f3ed426203f@10.31.130.252@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.085426:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657214116160, offset 192 00000400:00000200:15.0:1616513073.085445:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.252@tcp1 00000400:00000200:15.0:1616513073.085452:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.252@tcp1 routed destination 00000400:00000200:15.0:1616513073.085466:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.252@tcp1(10.31.130.252@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.085471:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.085477:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.085480:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.085482:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.085486:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.085492:0:48713:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.085493:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513073.085548:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.085556:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.085560:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.085567:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.085573:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775dc0 00000800:00000200:2.0:1616513073.085577:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.085580:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.086900:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513073.086954:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513073.086963:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513073.086971:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.252@tcp1 : PUT - for me 00000400:00000200:4.0:1616513073.086979:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.252@tcp1 of length 224 into portal 12 MB=0x5ffd325651600 00000400:00000200:4.0:1616513073.086985:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.252@tcp1 of length 224/224 into md 0x865b9f31 [1] + 62008 00000400:00000200:4.0:1616513073.086989:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513073.086992:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513073.086999:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513073.087002:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513073.087003:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513073.087006:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513073.087019:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657214117376 00010000:00000200:8.0:1616513073.087031:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8542eec380 x1688657214117376/t0(0) o400->4bd25548-a7b5-b34d-0493-7055d203c8f5@10.31.130.252@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513073.087043:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657214117376, offset 192 00000400:00000200:8.0:1616513073.087048:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.252@tcp1 00000400:00000200:8.0:1616513073.087055:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.252@tcp1 routed destination 00000400:00000200:8.0:1616513073.087067:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.252@tcp1(10.31.130.252@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513073.087073:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.087079:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513073.087081:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.087083:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.087087:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.087092:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.087146:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.087153:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.087156:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.087163:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.087166:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70770 00000800:00000200:16.0:1616513073.087170:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.087174:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.098613:0:6487:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924096, portal 4 00000100:00000200:20.0:1616513073.098618:0:6487:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924096, offset 0 00000400:00000200:20.0:1616513073.098625:0:6487:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:20.0:1616513073.098631:0:6487:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:20.0:1616513073.098643:0:6487:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.098650:0:6487:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:20.0:1616513073.098656:0:6487:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.098659:0:6487:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513073.098661:0:6487:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513073.098666:0:6487:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513073.098713:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513073.098736:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513073.098742:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513073.098745:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513073.098747:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:4.0:1616513073.098751:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513073.098759:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affc0d80 x1693067725924096/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513073.098770:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22990 00000800:00000200:4.0:1616513073.098774:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513073.098849:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513073.098903:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513073.098912:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:16.0:1616513073.098921:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513073.098929:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626700 00000400:00000200:16.0:1616513073.098936:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d33b1 [1] + 192 00000400:00000200:16.0:1616513073.098941:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513073.098948:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affc0d80 x1693067725924096/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513073.098956:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513073.098964:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513073.098967:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513073.098969:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:20.0:1616513073.098970:0:6487:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affc0d80 x1693067725924096/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513073.098983:0:6487:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affc0d80 x1693067725924096/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513073.098993:0:6487:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23760 00000800:00000200:7.2:1616513073.105231:0:48715:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.105289:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.105298:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.105307:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.38@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.105316:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.38@tcp1 of length 224 into portal 26 MB=0x5ffdc2ac19240 00000400:00000200:18.0:1616513073.105323:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.38@tcp1 of length 224/224 into md 0xf97c4e99 [1] + 0 00000400:00000200:18.0:1616513073.105328:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bbdc0 00000400:00000200:18.0:1616513073.105331:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.105334:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.105339:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbdc0 00000800:00000200:18.0:1616513073.105342:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.105343:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.105345:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.105347:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.105403:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695958770240 00010000:00000200:15.0:1616513073.105414:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688695958770240/t0(0) o400->7e557054-f93c-26e2-e150-8c7cac6ef88e@10.31.142.38@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.105429:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695958770240, offset 192 00000400:00000200:15.0:1616513073.105448:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.38@tcp1 00000400:00000200:15.0:1616513073.105456:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.38@tcp1 routed destination 00000400:00000200:15.0:1616513073.105471:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.38@tcp1(10.31.142.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.105478:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.105484:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.105487:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.105489:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.105494:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.105499:0:48715:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.105502:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513073.105567:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.105575:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.105578:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.105585:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.105591:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77754b8 00000800:00000200:2.0:1616513073.105595:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.105598:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.105601:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.105657:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.105666:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513073.105677:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.38@tcp1 : PUT - for me 00000400:00000200:20.0:1616513073.105686:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.38@tcp1 of length 224 into portal 12 MB=0x5ffdc2ac19380 00000400:00000200:20.0:1616513073.105692:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.38@tcp1 of length 224/224 into md 0x865b9f31 [1] + 62232 00000400:00000200:20.0:1616513073.105697:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513073.105700:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513073.105707:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513073.105710:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513073.105710:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513073.105715:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513073.105725:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695958770560 00010000:00000200:8.0:1616513073.105735:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03180 x1688695958770560/t0(0) o400->d652bde1-80ac-4dbc-c5f3-ef03ad2af3a8@10.31.142.38@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513073.105747:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695958770560, offset 192 00000400:00000200:8.0:1616513073.105752:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.38@tcp1 00000400:00000200:8.0:1616513073.105758:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.38@tcp1 routed destination 00000400:00000200:8.0:1616513073.105771:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.38@tcp1(10.31.142.38@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513073.105776:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.105781:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513073.105783:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.105785:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.105789:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.105794:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.105846:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.105860:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.105863:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513073.105866:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.105870:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.105874:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70d48 00000800:00000200:28.0:1616513073.105884:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.117433:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513073.117489:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513073.117498:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513073.117507:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.22@tcp1 : PUT - for me 00000400:00000200:4.0:1616513073.117516:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.22@tcp1 of length 224 into portal 26 MB=0x5ffe97a73f200 00000400:00000200:4.0:1616513073.117522:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.22@tcp1 of length 224/224 into md 0xf97c4eb9 [1] + 0 00000400:00000200:4.0:1616513073.117527:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb870 00000400:00000200:4.0:1616513073.117530:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513073.117533:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513073.117538:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000800:00000200:4.0:1616513073.117541:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513073.117542:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513073.117543:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513073.117549:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513073.117552:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513073.117553:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:15.0:1616513073.117561:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753130435072 00000400:00000200:20.0:1616513073.117562:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.22@tcp1 : PUT - for me 00000400:00000200:20.0:1616513073.117571:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.22@tcp1 of length 224 into portal 12 MB=0x5ffe97a73f280 00010000:00000200:15.0:1616513073.117572:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688753130435072/t0(0) o400->8cecb00c-1b60-0ea7-f966-525196956c40@10.31.142.22@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513073.117578:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.22@tcp1 of length 224/224 into md 0x865b9f31 [1] + 62456 00000400:00000200:20.0:1616513073.117583:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513073.117585:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513073.117586:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688753130435072, offset 192 00000800:00000200:20.0:1616513073.117592:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513073.117593:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.22@tcp1 00000800:00000200:20.0:1616513073.117595:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513073.117597:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:15.0:1616513073.117601:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.22@tcp1 routed destination 00000100:00000200:8.0:1616513073.117609:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753130435200 00000400:00000200:15.0:1616513073.117616:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.22@tcp1(10.31.142.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:8.0:1616513073.117619:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e00000 x1688753130435200/t0(0) o400->cdfbe6bb-e392-1226-ffdf-c9f053a6ee47@10.31.142.22@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513073.117623:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.117631:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:8.0:1616513073.117633:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688753130435200, offset 192 00000800:00000200:15.0:1616513073.117634:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.117636:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513073.117638:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.22@tcp1 00000800:00000200:15.0:1616513073.117642:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513073.117645:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.22@tcp1 routed destination 00000800:00000200:7.2:1616513073.117647:0:48716:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.117649:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:8.0:1616513073.117658:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.22@tcp1(10.31.142.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513073.117664:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.117669:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513073.117672:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.117674:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.117678:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.117683:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513073.117703:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.117711:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.117714:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.117721:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.117727:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775760 00000800:00000200:26.0:1616513073.117731:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.117734:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513073.117739:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.117746:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.117749:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513073.117750:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.117755:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.117759:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71cb0 00000800:00000200:28.0:1616513073.117763:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.119320:0:48716:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.119378:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.119388:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.119397:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.64@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.119406:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.64@tcp1 of length 224 into portal 26 MB=0x60194e6b8f140 00000400:00000200:2.0:1616513073.119413:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.64@tcp1 of length 224/224 into md 0xf97c4ed9 [1] + 0 00000400:00000200:2.0:1616513073.119418:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb3a8 00000400:00000200:2.0:1616513073.119421:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.119424:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.119429:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb3a8 00000800:00000200:2.0:1616513073.119432:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.119451:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513073.119451:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.119458:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.119461:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513073.119461:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513073.119470:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.64@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.119479:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.64@tcp1 of length 224 into portal 12 MB=0x60194e6b8f1c0 00000400:00000200:14.0:1616513073.119485:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.64@tcp1 of length 224/224 into md 0xf96450b9 [1] + 27912 00000400:00000200:14.0:1616513073.119490:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:15.0:1616513073.119491:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588897931584 00000100:00000200:14.0:1616513073.119493:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:15.0:1616513073.119501:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1690588897931584/t0(0) o400->f98596b5-f1da-edcb-8a88-27eb8a2ec403@10.31.142.64@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513073.119502:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.119505:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.119507:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.119515:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588897931584, offset 192 00000400:00000200:15.0:1616513073.119520:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.64@tcp1 00000400:00000200:15.0:1616513073.119528:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.64@tcp1 routed destination 00000400:00000200:15.0:1616513073.119542:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.64@tcp1(10.31.142.64@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.119549:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.119555:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.119557:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.119559:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.119564:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.119568:0:48716:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.119571:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513073.119576:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588897931712 00010000:00000200:22.0:1616513073.119589:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c8d80 x1690588897931712/t0(0) o400->eaec2bda-5f69-9bc7-96e7-743e9e986bde@10.31.142.64@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.119605:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588897931712, offset 192 00000400:00000200:22.0:1616513073.119611:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.64@tcp1 00000400:00000200:22.0:1616513073.119619:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.64@tcp1 routed destination 00000800:00000200:18.0:1616513073.119622:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.119630:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.119633:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.119633:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513073.119634:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.64@tcp1(10.31.142.64@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513073.119640:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513073.119641:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513073.119646:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774000 00000800:00000200:22.0:1616513073.119647:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513073.119649:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:22.0:1616513073.119650:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.119654:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.119659:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.119666:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.119732:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.119738:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.119740:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.119746:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.119750:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba90 00000800:00000200:16.0:1616513073.119753:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.119756:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.122051:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.122107:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.122117:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513073.122119:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.122125:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.122126:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.73@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.122133:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.73@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.122136:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.73@tcp1 of length 224 into portal 26 MB=0x5ffd47d71d900 00000400:00000200:14.0:1616513073.122141:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.73@tcp1 of length 224 into portal 12 MB=0x5ffd47d71d980 00000400:00000200:26.0:1616513073.122143:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.73@tcp1 of length 224/224 into md 0xf97c4ee9 [1] + 0 00000400:00000200:26.0:1616513073.122147:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb760 00000400:00000200:26.0:1616513073.122150:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.122152:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.73@tcp1 of length 224/224 into md 0xf96450b9 [1] + 28136 00000100:00000200:26.0:1616513073.122156:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.122156:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.122161:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb760 00000100:00000200:14.0:1616513073.122163:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.122164:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513073.122167:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513073.122168:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513073.122169:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.122171:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.122172:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.122174:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.122227:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662986316032 00000100:00000200:22.0:1616513073.122236:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662986316160 00010000:00000200:15.0:1616513073.122238:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688662986316032/t0(0) o400->34a35b29-954d-a637-375c-aff20aca361a@10.31.138.73@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:22.0:1616513073.122249:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cba80 x1688662986316160/t0(0) o400->2e1d77a3-4f88-2f55-cb3a-da931617096d@10.31.138.73@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513073.122253:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662986316032, offset 192 00000400:00000200:15.0:1616513073.122259:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.73@tcp1 00000100:00000200:22.0:1616513073.122263:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662986316160, offset 192 00000400:00000200:15.0:1616513073.122268:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.73@tcp1 routed destination 00000400:00000200:22.0:1616513073.122270:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.73@tcp1 00000400:00000200:22.0:1616513073.122278:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.73@tcp1 routed destination 00000400:00000200:15.0:1616513073.122295:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.73@tcp1(10.31.138.73@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513073.122296:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.73@tcp1(10.31.138.73@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.122302:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.122304:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513073.122309:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.122311:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.122312:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.122314:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513073.122315:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.122316:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513073.122319:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513073.122321:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513073.122325:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513073.122327:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513073.122330:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513073.122379:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.122386:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.122389:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513073.122390:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.122396:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.122402:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775ba0 00000800:00000200:4.0:1616513073.122402:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513073.122406:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513073.122409:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513073.122412:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513073.122418:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513073.122422:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a550 00000800:00000200:4.0:1616513073.122426:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513073.122430:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.129758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.129812:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.129822:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.129831:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.171@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.129840:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.171@tcp1 of length 224 into portal 26 MB=0x5ffe32bbe95c0 00000400:00000200:14.0:1616513073.129846:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.171@tcp1 of length 224/224 into md 0xf97c4f19 [1] + 0 00000400:00000200:14.0:1616513073.129853:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb320 00000400:00000200:14.0:1616513073.129856:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.129859:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.129864:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb320 00000800:00000200:2.0:1616513073.129866:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.129867:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.129870:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.129871:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.129927:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726040122816 00000800:00000200:7.2:1616513073.129928:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:15.0:1616513073.129937:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688726040122816/t0(0) o400->d1591251-d223-d43f-398e-2bfde4479f07@10.31.141.171@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.129950:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726040122816, offset 192 00000400:00000200:15.0:1616513073.129956:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.171@tcp1 00000400:00000200:15.0:1616513073.129963:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.171@tcp1 routed destination 00000400:00000200:15.0:1616513073.129978:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.171@tcp1(10.31.141.171@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.129984:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.129990:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513073.129991:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.129993:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.129995:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513073.129999:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513073.130001:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:15.0:1616513073.130006:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513073.130010:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.171@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.130021:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.171@tcp1 of length 224 into portal 12 MB=0x5ffe32bbe9940 00000400:00000200:26.0:1616513073.130028:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.171@tcp1 of length 224/224 into md 0xf96450b9 [1] + 28360 00000400:00000200:26.0:1616513073.130033:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.130036:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.130042:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513073.130045:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513073.130046:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513073.130053:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513073.130055:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.130056:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.130058:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.130065:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.130073:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775cb0 00000800:00000200:18.0:1616513073.130077:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.130104:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726040123712 00010000:00000200:22.0:1616513073.130120:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693600 x1688726040123712/t0(0) o400->5baa57eb-6361-73ea-3509-62ac57733ca7@10.31.141.171@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.130134:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726040123712, offset 192 00000400:00000200:22.0:1616513073.130140:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.171@tcp1 00000400:00000200:22.0:1616513073.130147:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.171@tcp1 routed destination 00000400:00000200:22.0:1616513073.130161:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.171@tcp1(10.31.141.171@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.130167:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.130172:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.130175:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.130177:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.130181:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.130189:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.130243:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.130249:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.130252:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.130257:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.130261:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b870 00000800:00000200:16.0:1616513073.130265:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.130267:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.130274:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.130276:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.130277:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.130278:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513073.130280:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.148915:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.148970:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.148980:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.148989:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.144@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.148998:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.144@tcp1 of length 224 into portal 12 MB=0x603e4af2f7b00 00000400:00000200:14.0:1616513073.149004:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.144@tcp1 of length 224/224 into md 0xf96450b9 [1] + 28584 00000400:00000200:14.0:1616513073.149009:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.149012:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.149018:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.149021:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.149023:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.149026:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513073.149035:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513073.149036:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.149045:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.19@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.149055:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.19@tcp1 of length 224 into portal 12 MB=0x5ffd4a5ded540 00000400:00000200:26.0:1616513073.149062:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.19@tcp1 of length 224/224 into md 0xf96450b9 [1] + 28808 00000400:00000200:26.0:1616513073.149067:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.149070:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.149078:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.149081:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.149082:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.149085:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513073.149089:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693130586815232 00000100:00000200:10.0:1616513073.149092:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663664547136 00010000:00000200:30.0:1616513073.149101:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c8000 x1693130586815232/t0(0) o400->0d5ee282-4f9e-e88c-9126-61dfe1048e3d@10.31.140.144@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513073.149104:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:10.0:1616513073.149105:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691f80 x1688663664547136/t0(0) o400->67146f9f-0b9b-79a1-ed41-6845fe8246fd@10.31.136.19@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513073.149113:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.149115:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:30.0:1616513073.149116:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693130586815232, offset 192 00000100:00000200:10.0:1616513073.149120:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663664547136, offset 192 00000400:00000200:28.0:1616513073.149122:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.144@tcp1 : PUT - for me 00000400:00000200:30.0:1616513073.149123:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.144@tcp1 00000800:00000200:8.0:1616513073.149124:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:10.0:1616513073.149129:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.19@tcp1 00000400:00000200:30.0:1616513073.149131:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.144@tcp1 routed destination 00000400:00000200:28.0:1616513073.149132:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.144@tcp1 of length 224 into portal 26 MB=0x603e4af2f7880 00000400:00000200:8.0:1616513073.149134:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.19@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.149139:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.144@tcp1 of length 224/224 into md 0xf97c4f29 [1] + 0 00000400:00000200:8.0:1616513073.149144:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.19@tcp1 of length 224 into portal 26 MB=0x5ffd4a5ded480 00000400:00000200:30.0:1616513073.149146:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.144@tcp1(10.31.140.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513073.149148:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba220 00000400:00000200:28.0:1616513073.149151:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.149153:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.19@tcp1 of length 224/224 into md 0xf97c4f39 [1] + 0 00000800:00000200:30.0:1616513073.149154:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:10.0:1616513073.149156:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.19@tcp1 routed destination 00000400:00000200:8.0:1616513073.149158:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bab28 00000100:00000200:28.0:1616513073.149161:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.149161:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513073.149165:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513073.149166:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba220 00000800:00000200:30.0:1616513073.149168:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:8.0:1616513073.149168:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:16.0:1616513073.149169:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:30.0:1616513073.149170:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:10.0:1616513073.149172:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.19@tcp1(10.31.136.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513073.149173:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bab28 00000800:00000200:30.0:1616513073.149175:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513073.149177:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513073.149178:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.149178:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513073.149179:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513073.149180:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513073.149181:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513073.149182:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:10.0:1616513073.149184:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513073.149185:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513073.149187:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513073.149189:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513073.149194:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.149199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513073.149231:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693130586814592 00000100:00000200:12.0:1616513073.149240:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663664546944 00010000:00000200:15.0:1616513073.149241:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1693130586814592/t0(0) o400->305e7520-9ace-edc6-db77-5dfdd9d79615@10.31.140.144@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:12.0:1616513073.149252:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688663664546944/t0(0) o400->be5d4a90-ef59-5727-a8dc-0d5ab647784d@10.31.136.19@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.149255:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693130586814592, offset 192 00000800:00000200:24.0:1616513073.149259:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513073.149259:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:15.0:1616513073.149261:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.144@tcp1 00000400:00000200:15.0:1616513073.149268:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.144@tcp1 routed destination 00000100:00000200:12.0:1616513073.149272:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663664546944, offset 192 00000400:00000200:12.0:1616513073.149278:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.19@tcp1 00000800:00000200:24.0:1616513073.149279:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513073.149280:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.149282:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513073.149282:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.149285:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513073.149286:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513073.149288:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.19@tcp1 routed destination 00000400:00000200:24.0:1616513073.149289:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:15.0:1616513073.149291:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.144@tcp1(10.31.140.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513073.149292:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.149294:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82550 00000400:00000200:18.0:1616513073.149297:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee660 00000800:00000200:15.0:1616513073.149299:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513073.149301:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513073.149304:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.19@tcp1(10.31.136.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.149305:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.149308:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513073.149309:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:15.0:1616513073.149310:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.149311:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513073.149315:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513073.149317:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.149320:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513073.149320:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.149322:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513073.149322:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.149327:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513073.149329:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513073.149332:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513073.149333:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:12.0:1616513073.149334:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513073.149335:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.149338:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513073.149341:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513073.149341:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513073.149345:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513073.149346:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774880 00000400:00000200:28.0:1616513073.149348:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.149348:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513073.149349:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513073.149353:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.149358:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241980 00000800:00000200:28.0:1616513073.149361:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.149553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.149562:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.149567:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513073.149570:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513073.149575:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.18@tcp1 : PUT - for me 00000800:00000200:18.0:1616513073.149576:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513073.149578:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.149582:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.18@tcp1 of length 224 into portal 26 MB=0x5ffdbbabbe2c0 00000400:00000200:18.0:1616513073.149584:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.18@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.149588:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.18@tcp1 of length 224/224 into md 0xf97c4f49 [1] + 0 00000400:00000200:18.0:1616513073.149592:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.18@tcp1 of length 224 into portal 12 MB=0x5ffdbbabbe380 00000400:00000200:14.0:1616513073.149592:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba088 00000400:00000200:14.0:1616513073.149594:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.149598:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.18@tcp1 of length 224/224 into md 0xf96450b9 [1] + 29032 00000400:00000200:18.0:1616513073.149602:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.149602:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.149606:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba088 00000100:00000200:18.0:1616513073.149609:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.149609:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.149611:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.149612:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.149615:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.149618:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.149620:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.149620:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694079349440 00010000:00000200:12.0:1616513073.149626:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688694079349440/t0(0) o400->25464559-c4de-56e0-02b9-34a4e2adbf77@10.31.141.18@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513073.149629:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694079349632 00010000:00000200:10.0:1616513073.149635:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d2880 x1688694079349632/t0(0) o400->21f30af5-11d1-d076-7013-c35697dc5156@10.31.141.18@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513073.149637:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694079349440, offset 192 00000400:00000200:12.0:1616513073.149643:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.18@tcp1 00000100:00000200:10.0:1616513073.149646:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694079349632, offset 192 00000400:00000200:12.0:1616513073.149648:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.18@tcp1 routed destination 00000400:00000200:10.0:1616513073.149651:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.18@tcp1 00000400:00000200:12.0:1616513073.149662:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.18@tcp1(10.31.141.18@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.149667:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:10.0:1616513073.149669:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.18@tcp1 routed destination 00000800:00000200:12.0:1616513073.149673:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.149676:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.149677:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.149681:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513073.149683:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.18@tcp1(10.31.141.18@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513073.149685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.149686:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513073.149688:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513073.149693:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513073.149694:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513073.149695:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513073.149697:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513073.149698:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.149700:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513073.149700:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513073.149701:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.149704:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513073.149705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513073.149708:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240110 00000800:00000200:26.0:1616513073.149710:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513073.149715:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.149721:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.149723:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513073.149725:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513073.149730:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.149734:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82bb0 00000800:00000200:16.0:1616513073.149738:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.150474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.150483:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.150488:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513073.150490:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513073.150495:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.168@tcp1 : PUT - for me 00000800:00000200:18.0:1616513073.150496:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513073.150498:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.150503:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.168@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.150503:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.168@tcp1 of length 224 into portal 26 MB=0x5ffdb77f89340 00000400:00000200:14.0:1616513073.150508:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.168@tcp1 of length 224/224 into md 0xf97c4f59 [1] + 0 00000400:00000200:18.0:1616513073.150509:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.168@tcp1 of length 224 into portal 12 MB=0x5ffdb77f893c0 00000400:00000200:14.0:1616513073.150511:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6baf68 00000400:00000200:14.0:1616513073.150513:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.150515:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.168@tcp1 of length 224/224 into md 0xf96450b9 [1] + 29256 00000400:00000200:18.0:1616513073.150519:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.150519:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.150522:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baf68 00000100:00000200:18.0:1616513073.150524:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.150524:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.150526:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.150528:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.150529:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.150531:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.150532:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.150534:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692959253312 00010000:00000200:12.0:1616513073.150539:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688692959253312/t0(0) o400->65026146-757a-e365-a785-c60b574b4039@10.31.141.168@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513073.150539:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692959253440 00010000:00000200:10.0:1616513073.150545:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d0480 x1688692959253440/t0(0) o400->f0f75981-fab5-487c-3a0f-ae73e818d4d4@10.31.141.168@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513073.150550:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692959253312, offset 192 00000400:00000200:12.0:1616513073.150555:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.168@tcp1 00000100:00000200:10.0:1616513073.150555:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692959253440, offset 192 00000400:00000200:12.0:1616513073.150560:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.168@tcp1 routed destination 00000400:00000200:10.0:1616513073.150560:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.168@tcp1 00000400:00000200:10.0:1616513073.150566:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.168@tcp1 routed destination 00000400:00000200:10.0:1616513073.150579:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.168@tcp1(10.31.141.168@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513073.150580:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.168@tcp1(10.31.141.168@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513073.150585:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.150586:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.150590:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513073.150590:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.150592:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513073.150592:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.150594:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513073.150594:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513073.150596:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.150597:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513073.150601:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513073.150601:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513073.150602:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513073.150610:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513073.150610:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513073.150613:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:8.0:1616513073.150613:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.150615:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.150617:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.150617:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513073.150620:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.150622:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.150623:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e837e8 00000400:00000200:26.0:1616513073.150625:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241298 00000800:00000200:8.0:1616513073.150625:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513073.150627:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513073.150628:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.150683:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.150692:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.150698:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.150700:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.150701:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.150703:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.150704:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.151893:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.151945:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.151954:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.151963:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.182@tcp1 : PUT - for me 00000800:00000200:11.2:1616513073.151963:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513073.151973:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.182@tcp1 of length 224 into portal 26 MB=0x5ffd491eedc80 00000800:00000200:8.0:1616513073.151974:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513073.151979:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.182@tcp1 of length 224/224 into md 0xf97c4f79 [1] + 0 00000800:00000200:8.0:1616513073.151979:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:14.0:1616513073.151984:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bbba0 00000400:00000200:8.0:1616513073.151985:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.182@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.151987:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.151990:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.151991:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.182@tcp1 of length 224 into portal 12 MB=0x5ffd491eeddc0 00000400:00000200:14.0:1616513073.151994:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbba0 00000400:00000200:8.0:1616513073.151995:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.182@tcp1 of length 224/224 into md 0x865b9f31 [1] + 62680 00000800:00000200:14.0:1616513073.151997:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.151998:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513073.151999:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513073.152000:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:8.0:1616513073.152001:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.152002:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513073.152006:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.152008:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.152009:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.152011:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.152078:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663330053248 00000100:00000200:16.0:1616513073.152079:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663330053568 00010000:00000200:12.0:1616513073.152089:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688663330053248/t0(0) o400->91f89efb-5cb5-4466-0555-5cb2fd6ce4f7@10.31.136.182@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:16.0:1616513073.152092:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9cc80 x1688663330053568/t0(0) o400->41835b40-dfe9-952b-7628-637d7ddd8a73@10.31.136.182@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513073.152103:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663330053248, offset 192 00000400:00000200:12.0:1616513073.152109:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.182@tcp1 00000100:00000200:16.0:1616513073.152112:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663330053568, offset 192 00000400:00000200:12.0:1616513073.152117:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.182@tcp1 routed destination 00000400:00000200:16.0:1616513073.152118:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.182@tcp1 00000400:00000200:16.0:1616513073.152127:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.182@tcp1 routed destination 00000400:00000200:12.0:1616513073.152133:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.182@tcp1(10.31.136.182@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.152139:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513073.152142:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.182@tcp1(10.31.136.182@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.152148:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.152149:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.152152:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.152154:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.152154:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513073.152157:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.152159:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.152159:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513073.152164:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513073.152164:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.152165:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.152170:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.152182:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.152185:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.152187:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513073.152191:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513073.152193:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.152196:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a422a8 00000800:00000200:16.0:1616513073.152198:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513073.152219:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.152226:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.152229:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513073.152234:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.152235:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.152241:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a90 00000800:00000200:2.0:1616513073.152245:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.160592:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.160646:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.160656:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513073.160659:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513073.160665:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.235@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.160673:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.235@tcp1 of length 224 into portal 26 MB=0x5fff31c681800 00000400:00000200:18.0:1616513073.160681:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.235@tcp1 of length 224/224 into md 0xf97c4f89 [1] + 0 00000400:00000200:18.0:1616513073.160686:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba000 00000400:00000200:18.0:1616513073.160689:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.160691:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.160695:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba000 00000800:00000200:18.0:1616513073.160698:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.160701:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.160703:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.160747:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:12.0:1616513073.160758:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688794502273024 00010000:00000200:12.0:1616513073.160769:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688794502273024/t0(0) o400->7924d718-c5a0-50fe-77f8-ad685b70e9d2@10.31.141.235@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.160784:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688794502273024, offset 192 00000400:00000200:12.0:1616513073.160790:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.235@tcp1 00000400:00000200:12.0:1616513073.160797:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.235@tcp1 routed destination 00000800:00000200:26.0:1616513073.160800:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.160810:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513073.160812:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.235@tcp1(10.31.141.235@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.160819:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.160821:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.235@tcp1 : PUT - for me 00000800:00000200:12.0:1616513073.160824:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.160829:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513073.160830:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.235@tcp1 of length 224 into portal 12 MB=0x5fff31c681880 00000800:00000200:12.0:1616513073.160831:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:26.0:1616513073.160836:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.235@tcp1 of length 224/224 into md 0xf96450b9 [1] + 29480 00000800:00000200:12.0:1616513073.160836:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513073.160841:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.160843:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:12.0:1616513073.160843:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513073.160850:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513073.160853:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513073.160853:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513073.160860:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513073.160862:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513073.160864:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.160864:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:10.0:1616513073.160868:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688794502273152 00000400:00000200:2.0:1616513073.160871:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.160877:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240880 00010000:00000200:10.0:1616513073.160878:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693180 x1688794502273152/t0(0) o400->15b05069-7878-ff58-df9e-106656be6ae4@10.31.141.235@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513073.160880:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513073.160891:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688794502273152, offset 192 00000400:00000200:10.0:1616513073.160896:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.235@tcp1 00000400:00000200:10.0:1616513073.160902:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.235@tcp1 routed destination 00000400:00000200:10.0:1616513073.160915:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.235@tcp1(10.31.141.235@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513073.160927:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513073.160933:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513073.160935:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513073.160938:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513073.160942:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.160988:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.160999:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.161003:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.161005:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.161009:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.161011:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82dd0 00000800:00000200:8.0:1616513073.161014:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.161016:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.174471:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.174529:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.174537:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.174545:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.212@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.174553:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.212@tcp1 of length 224 into portal 26 MB=0x603deaa131640 00000400:00000200:18.0:1616513073.174559:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.212@tcp1 of length 224/224 into md 0xf97c4fa9 [1] + 0 00000400:00000200:18.0:1616513073.174563:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6badd0 00000400:00000200:18.0:1616513073.174566:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.174569:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.174572:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6badd0 00000800:00000200:18.0:1616513073.174575:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.174577:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.174579:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.174581:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.174591:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.174599:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.212@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.174606:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.212@tcp1 of length 224 into portal 12 MB=0x603deaa131780 00000400:00000200:26.0:1616513073.174612:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.212@tcp1 of length 224/224 into md 0xf96450b9 [1] + 29704 00000400:00000200:26.0:1616513073.174617:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.174619:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.174627:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.174630:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.174632:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513073.174634:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693104731264576 00000800:00000200:26.0:1616513073.174635:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:12.0:1616513073.174644:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1693104731264576/t0(0) o400->d4628aa6-2676-9e40-a06b-52c8339d5f52@10.31.141.212@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513073.174646:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693104731264896 00010000:00000200:18.0:1616513073.174656:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691200 x1693104731264896/t0(0) o400->f560b535-6ef9-1b38-19cd-5808b849ce9d@10.31.141.212@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513073.174657:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693104731264576, offset 192 00000400:00000200:12.0:1616513073.174663:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.212@tcp1 00000100:00000200:18.0:1616513073.174668:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693104731264896, offset 192 00000400:00000200:12.0:1616513073.174669:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.212@tcp1 routed destination 00000400:00000200:18.0:1616513073.174673:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.212@tcp1 00000400:00000200:12.0:1616513073.174683:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.212@tcp1(10.31.141.212@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.174689:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.174691:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.212@tcp1 routed destination 00000800:00000200:12.0:1616513073.174695:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.174697:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.174699:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513073.174704:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.212@tcp1(10.31.141.212@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.174704:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.174710:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:12.0:1616513073.174710:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513073.174712:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.174715:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513073.174718:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513073.174719:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513073.174725:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513073.174763:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.174770:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513073.174771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513073.174772:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513073.174773:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.174778:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.174784:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240990 00000800:00000200:2.0:1616513073.174787:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513073.174828:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.174835:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.174838:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513073.174845:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.174849:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc7f8 00000800:00000200:24.0:1616513073.174853:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.174856:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.183062:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.183116:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.183126:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.183135:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.89@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.183144:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.89@tcp1 of length 224 into portal 26 MB=0x5ffd579ae2ac0 00000400:00000200:26.0:1616513073.183150:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.89@tcp1 of length 224/224 into md 0xf97c4fc9 [1] + 0 00000400:00000200:26.0:1616513073.183157:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71d38 00000400:00000200:26.0:1616513073.183160:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.183162:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513073.183166:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71d38 00000800:00000200:26.0:1616513073.183170:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.183170:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.183173:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.183177:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.183180:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.183180:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.183189:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.89@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.183198:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.89@tcp1 of length 224 into portal 12 MB=0x5ffd579ae2b40 00000400:00000200:14.0:1616513073.183204:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.89@tcp1 of length 224/224 into md 0xf96450b9 [1] + 29928 00000400:00000200:14.0:1616513073.183209:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.183212:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.183219:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.183223:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.183224:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.183229:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667218127552 00010000:00000200:12.0:1616513073.183240:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688667218127552/t0(0) o400->01338576-07f1-8e43-40c7-eee3c62b7b17@10.31.136.89@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.183254:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667218127552, offset 192 00000400:00000200:12.0:1616513073.183261:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.89@tcp1 00000400:00000200:12.0:1616513073.183268:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.89@tcp1 routed destination 00000100:00000200:22.0:1616513073.183283:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667218127680 00000400:00000200:12.0:1616513073.183283:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.89@tcp1(10.31.136.89@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.183290:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:22.0:1616513073.183295:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c8480 x1688667218127680/t0(0) o400->766efb8b-26ae-8c9b-4721-c5c0249d1b3d@10.31.136.89@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513073.183295:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.183298:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.183300:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.183305:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513073.183310:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667218127680, offset 192 00000800:00000200:7.2:1616513073.183310:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.183312:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513073.183318:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.89@tcp1 00000400:00000200:22.0:1616513073.183325:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.89@tcp1 routed destination 00000400:00000200:22.0:1616513073.183340:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.89@tcp1(10.31.136.89@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.183346:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.183352:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.183355:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.183357:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.183362:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513073.183365:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.183372:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.183375:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513073.183376:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.183381:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.183387:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a08 00000800:00000200:2.0:1616513073.183390:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.183408:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.183421:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.183427:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.183430:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.183440:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.183444:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b980 00000800:00000200:8.0:1616513073.183447:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.183450:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.183456:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.183458:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.183460:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.183461:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.183464:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.189209:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513073.189211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513073.189275:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513073.189282:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513073.189285:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:16.0:1616513073.189291:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.189291:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513073.189294:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.36@tcp1 : PUT - for me 00000400:00000200:16.0:1616513073.189300:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.36@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.189303:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.36@tcp1 of length 224 into portal 12 MB=0x5f91b3ca4d6c0 00000400:00000200:16.0:1616513073.189309:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.36@tcp1 of length 224 into portal 26 MB=0x5f91b3ca4d640 00000400:00000200:14.0:1616513073.189310:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.36@tcp1 of length 224/224 into md 0xf96450b9 [1] + 30152 00000400:00000200:14.0:1616513073.189315:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.189317:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.36@tcp1 of length 224/224 into md 0xf97c4fe9 [1] + 0 00000400:00000200:16.0:1616513073.189322:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f713a8 00000100:00000200:14.0:1616513073.189324:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513073.189325:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513073.189329:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513073.189330:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513073.189331:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513073.189334:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f713a8 00000800:00000200:14.0:1616513073.189334:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.189335:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513073.189337:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.189340:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.189342:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513073.189394:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681270260422336 00010000:00000200:22.0:1616513073.189406:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53ce300 x1681270260422336/t0(0) o400->bd8a079d-4413-be01-c7c9-f87f8597df86@10.31.132.36@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513073.189412:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681270260422208 00000100:00000200:22.0:1616513073.189420:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681270260422336, offset 192 00010000:00000200:12.0:1616513073.189423:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1681270260422208/t0(0) o400->2be3187b-2051-b1cd-6432-6f24bfb9aaef@10.31.132.36@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513073.189427:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.36@tcp1 00000400:00000200:22.0:1616513073.189448:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.36@tcp1 routed destination 00000100:00000200:12.0:1616513073.189451:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681270260422208, offset 192 00000400:00000200:12.0:1616513073.189457:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.36@tcp1 00000400:00000200:22.0:1616513073.189463:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.36@tcp1(10.31.132.36@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513073.189464:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.36@tcp1 routed destination 00000800:00000200:22.0:1616513073.189470:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.189475:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.189478:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513073.189479:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.36@tcp1(10.31.132.36@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.189480:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.189485:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513073.189486:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513073.189490:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.189492:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.189494:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.189496:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.189501:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.189506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.189508:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513073.189546:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.189553:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.189555:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.189558:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.189562:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513073.189566:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513073.189566:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a880 00000800:00000200:26.0:1616513073.189570:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513073.189573:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.189576:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513073.189582:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.189588:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241100 00000800:00000200:28.0:1616513073.189591:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.189594:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.202268:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.202280:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.202286:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.202294:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.5@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.202303:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.5@tcp1 of length 224 into portal 26 MB=0x5ffe5e63e9280 00000400:00000200:8.0:1616513073.202309:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.5@tcp1 of length 224/224 into md 0xf97c4ff9 [1] + 0 00000400:00000200:8.0:1616513073.202313:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71ed0 00000400:00000200:8.0:1616513073.202316:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.202318:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.202322:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71ed0 00000800:00000200:8.0:1616513073.202324:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.202327:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.202328:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.202330:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.202335:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.202335:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.202341:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.5@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.202346:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.5@tcp1 of length 224 into portal 12 MB=0x5ffe5e63e9380 00000400:00000200:8.0:1616513073.202351:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.5@tcp1 of length 224/224 into md 0x865b9f31 [1] + 62904 00000400:00000200:8.0:1616513073.202354:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.202356:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513073.202361:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.202363:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.202364:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.202382:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737759007360 00000100:00000200:16.0:1616513073.202384:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737759007616 00010000:00000200:12.0:1616513073.202392:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688737759007360/t0(0) o400->edec09e4-376b-8425-0ddc-b8739792abff@10.31.141.5@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:16.0:1616513073.202396:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9bf00 x1688737759007616/t0(0) o400->40cdb472-e267-899b-9c95-962146028423@10.31.141.5@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513073.202405:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688737759007360, offset 192 00000400:00000200:12.0:1616513073.202411:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.5@tcp1 00000400:00000200:12.0:1616513073.202417:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.5@tcp1 routed destination 00000100:00000200:16.0:1616513073.202420:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688737759007616, offset 192 00000400:00000200:16.0:1616513073.202427:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.5@tcp1 00000400:00000200:12.0:1616513073.202431:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.5@tcp1(10.31.141.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513073.202445:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.5@tcp1 routed destination 00000800:00000200:12.0:1616513073.202449:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513073.202460:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.5@tcp1(10.31.141.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.202466:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.202467:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.202469:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.202471:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513073.202472:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.202475:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.202476:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513073.202477:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.202482:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513073.202482:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.202483:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.202487:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.202499:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.202506:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.202509:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513073.202511:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513073.202515:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.202519:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a422a8 00000800:00000200:24.0:1616513073.202523:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513073.202535:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.202541:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.202543:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.202549:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.202554:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241650 00000800:00000200:18.0:1616513073.202557:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.202560:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.217945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.218001:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.218011:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513073.218017:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.218022:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.54@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.218032:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.54@tcp1 of length 224 into portal 26 MB=0x6018a34d99a80 00000400:00000200:2.0:1616513073.218039:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.54@tcp1 of length 224/224 into md 0xf97c5009 [1] + 0 00000400:00000200:2.0:1616513073.218043:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f702a8 00000400:00000200:2.0:1616513073.218046:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.218049:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.218054:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f702a8 00000800:00000200:2.0:1616513073.218057:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.218059:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.218061:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.218116:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690542964054656 00010000:00000200:12.0:1616513073.218126:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1690542964054656/t0(0) o400->698bbd8e-a4c3-7c70-dd42-fd129f82efcc@10.31.142.54@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.218138:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690542964054656, offset 192 00000400:00000200:12.0:1616513073.218145:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.54@tcp1 00000400:00000200:12.0:1616513073.218153:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.54@tcp1 routed destination 00000400:00000200:12.0:1616513073.218167:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.54@tcp1(10.31.142.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.218173:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.218178:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:7.2:1616513073.218180:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.218181:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.218183:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513073.218188:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:12.0:1616513073.218194:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513073.218234:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.218243:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.218252:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.54@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.218261:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.54@tcp1 of length 224 into portal 12 MB=0x6018a34d99d80 00000400:00000200:18.0:1616513073.218267:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.54@tcp1 of length 224/224 into md 0xf96450b9 [1] + 30376 00000400:00000200:18.0:1616513073.218273:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.218275:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513073.218282:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513073.218285:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513073.218287:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:18.0:1616513073.218294:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513073.218295:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513073.218297:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.218298:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.218305:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.218310:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782415c8 00000800:00000200:14.0:1616513073.218314:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.218347:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690542964055424 00010000:00000200:22.0:1616513073.218359:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d6300 x1690542964055424/t0(0) o400->d5ecc5d1-25c5-adb2-8fce-7e260a43d0b4@10.31.142.54@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.218374:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690542964055424, offset 192 00000400:00000200:22.0:1616513073.218380:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.54@tcp1 00000400:00000200:22.0:1616513073.218388:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.54@tcp1 routed destination 00000400:00000200:22.0:1616513073.218403:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.54@tcp1(10.31.142.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.218410:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.218415:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.218420:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.218422:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.218427:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.218473:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.218486:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.218495:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.218497:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.218503:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.218506:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4abb0 00000800:00000200:8.0:1616513073.218510:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.218512:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.237230:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513073.237246:0:48728:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.237298:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513073.237304:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513073.237308:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513073.237314:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513073.237318:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.9@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.237329:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.9@tcp1 : PUT - for me 00000400:00000200:16.0:1616513073.237330:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.9@tcp1 of length 224 into portal 26 MB=0x5f9131be7c4c0 00000400:00000200:16.0:1616513073.237337:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.9@tcp1 of length 224/224 into md 0xf97c5029 [1] + 0 00000400:00000200:2.0:1616513073.237338:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.9@tcp1 of length 224 into portal 12 MB=0x5f9131be7c500 00000400:00000200:16.0:1616513073.237342:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70220 00000400:00000200:16.0:1616513073.237345:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.237347:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.9@tcp1 of length 224/224 into md 0xf96450b9 [1] + 30600 00000100:00000200:16.0:1616513073.237353:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.237353:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.237357:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70220 00000100:00000200:2.0:1616513073.237360:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513073.237361:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.237364:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513073.237365:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.237368:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513073.237368:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.237369:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.237371:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.237374:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.237418:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681235351422144 00010000:00000200:12.0:1616513073.237428:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1681235351422144/t0(0) o400->caffc1b3-e4bd-1b33-fab0-30332e296ab4@10.31.132.9@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.237453:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681235351422144, offset 192 00000100:00000200:22.0:1616513073.237456:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681235351422208 00000400:00000200:12.0:1616513073.237458:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.9@tcp1 00000400:00000200:12.0:1616513073.237464:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.9@tcp1 routed destination 00010000:00000200:22.0:1616513073.237468:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138da00 x1681235351422208/t0(0) o400->cdaf632c-74bc-71f6-aa23-3316482dec1d@10.31.132.9@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513073.237477:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.9@tcp1(10.31.132.9@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513073.237483:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681235351422208, offset 192 00000800:00000200:12.0:1616513073.237483:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:22.0:1616513073.237489:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.9@tcp1 00000800:00000200:12.0:1616513073.237489:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.237491:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.237493:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:22.0:1616513073.237496:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.9@tcp1 routed destination 00000800:00000200:12.0:1616513073.237497:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.237502:0:48728:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.237504:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513073.237511:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.9@tcp1(10.31.132.9@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.237518:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.237523:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.237526:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.237528:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.237533:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.237538:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.237549:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.237555:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513073.237557:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:8.0:1616513073.237557:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.237563:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513073.237564:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:8.0:1616513073.237566:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a2a8 00000400:00000200:26.0:1616513073.237567:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.237570:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513073.237574:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:8.0:1616513073.237574:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513073.237580:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240088 00000800:00000200:26.0:1616513073.237583:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.237586:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.246740:0:48729:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.246801:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.246810:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.246819:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.246@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.246829:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.246@tcp1 of length 224 into portal 26 MB=0x6041932fd0280 00000400:00000200:18.0:1616513073.246835:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.246@tcp1 of length 224/224 into md 0xf97c5039 [1] + 0 00000400:00000200:18.0:1616513073.246840:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71ba0 00000400:00000200:18.0:1616513073.246843:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.246846:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.246850:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71ba0 00000800:00000200:18.0:1616513073.246853:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.246855:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.246856:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.246859:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.246914:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356136399488 00010000:00000200:12.0:1616513073.246925:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1693356136399488/t0(0) o400->44b64bbe-1546-d00c-5424-fb1f2ddaa227@10.31.128.246@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.246939:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693356136399488, offset 192 00000400:00000200:12.0:1616513073.246945:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.246@tcp1 00000400:00000200:12.0:1616513073.246953:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.246@tcp1 routed destination 00000400:00000200:12.0:1616513073.246968:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.246@tcp1(10.31.128.246@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.246974:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.246981:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.246984:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.246986:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.246991:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.246996:0:48729:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.246998:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513073.247049:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.247056:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.247059:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.247065:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.247071:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241ba0 00000800:00000200:26.0:1616513073.247074:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.247077:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.247153:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.247209:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.247218:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.247227:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.246@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.247235:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.246@tcp1 of length 224 into portal 12 MB=0x6041932fd0680 00000400:00000200:24.0:1616513073.247242:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.246@tcp1 of length 224/224 into md 0x865b9f31 [1] + 63128 00000400:00000200:24.0:1616513073.247249:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.247251:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513073.247259:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.247262:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513073.247262:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.247266:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.247324:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356136400512 00010000:00000200:20.0:1616513073.247336:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc1200 x1693356136400512/t0(0) o400->b960c3f3-e69f-8877-e129-5539aaa80801@10.31.128.246@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.247351:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693356136400512, offset 192 00000400:00000200:20.0:1616513073.247357:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.246@tcp1 00000400:00000200:20.0:1616513073.247364:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.246@tcp1 routed destination 00000400:00000200:20.0:1616513073.247379:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.246@tcp1(10.31.128.246@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.247386:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.247391:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.247394:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.247396:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.247401:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.247406:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.247415:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.247420:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.247423:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.247429:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.247432:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23760 00000800:00000200:8.0:1616513073.247441:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.247443:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.262646:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.262700:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.262709:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.262719:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.42@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.262728:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.42@tcp1 of length 224 into portal 26 MB=0x5ffd3d58f6fc0 00000400:00000200:2.0:1616513073.262735:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.42@tcp1 of length 224/224 into md 0xf97c5049 [1] + 0 00000400:00000200:2.0:1616513073.262740:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71a08 00000400:00000200:2.0:1616513073.262742:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.262745:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.262749:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71a08 00000800:00000200:2.0:1616513073.262752:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.262755:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.262755:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.262759:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.262814:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660169682880 00010000:00000200:12.0:1616513073.262825:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688660169682880/t0(0) o400->7f349f95-4116-d8f0-dd53-7a857a3f6654@10.31.129.42@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.262839:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660169682880, offset 192 00000400:00000200:12.0:1616513073.262846:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.42@tcp1 00000400:00000200:12.0:1616513073.262853:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.42@tcp1 routed destination 00000400:00000200:12.0:1616513073.262869:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.42@tcp1(10.31.129.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.262875:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.262881:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.262883:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.262886:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.262890:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.262895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.262897:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513073.262947:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.262954:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.262957:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.262963:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.262969:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241ba0 00000800:00000200:26.0:1616513073.262974:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.262977:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.262983:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513073.262986:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.262991:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.42@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.262999:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.42@tcp1 of length 224 into portal 12 MB=0x5ffd3d58f7300 00000400:00000200:26.0:1616513073.263006:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.42@tcp1 of length 224/224 into md 0xf96450b9 [1] + 30824 00000400:00000200:26.0:1616513073.263011:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.263014:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.263020:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.263022:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.263024:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.263042:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660169683712 00010000:00000200:22.0:1616513073.263054:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c690900 x1688660169683712/t0(0) o400->a73bded5-cbe5-a4f0-0c8b-135500bf9fab@10.31.129.42@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.263068:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660169683712, offset 192 00000400:00000200:22.0:1616513073.263075:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.42@tcp1 00000400:00000200:22.0:1616513073.263082:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.42@tcp1 routed destination 00000400:00000200:22.0:1616513073.263097:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.42@tcp1(10.31.129.42@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.263104:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.263110:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.263113:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.263115:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.263119:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.263127:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.263183:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.263189:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.263192:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.263199:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.263203:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a5d8 00000800:00000200:16.0:1616513073.263207:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.263210:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.264518:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513073.264546:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.264556:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.264564:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.264572:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.47@tcp1 : PUT - for me 00000800:00000200:2.0:1616513073.264573:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513073.264582:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.47@tcp1 of length 224 into portal 26 MB=0x5f909c941a100 00000800:00000200:2.0:1616513073.264582:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:8.0:1616513073.264588:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.47@tcp1 of length 224/224 into md 0xf97c5069 [1] + 0 00000400:00000200:2.0:1616513073.264591:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.47@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.264593:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71650 00000400:00000200:8.0:1616513073.264596:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.264598:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.264600:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.47@tcp1 of length 224 into portal 12 MB=0x5f909c941a180 00000400:00000200:8.0:1616513073.264603:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71650 00000800:00000200:8.0:1616513073.264606:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513073.264607:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.47@tcp1 of length 224/224 into md 0xf96450b9 [1] + 31048 00000800:00000200:28.0:1616513073.264608:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.264608:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.264613:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513073.264613:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.264615:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513073.264621:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.264624:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.264626:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.264629:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.264668:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681195310096640 00010000:00000200:12.0:1616513073.264679:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1681195310096640/t0(0) o400->e6415f22-b002-3c80-6191-458a0f4a5fa3@10.31.131.47@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513073.264691:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681195310096768 00000100:00000200:12.0:1616513073.264693:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681195310096640, offset 192 00000400:00000200:12.0:1616513073.264699:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.47@tcp1 00010000:00000200:22.0:1616513073.264703:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138de80 x1681195310096768/t0(0) o400->8013c9c0-625f-b010-4a48-118b9a62bc0e@10.31.131.47@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513073.264707:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.47@tcp1 routed destination 00000400:00000200:12.0:1616513073.264721:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.47@tcp1(10.31.131.47@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513073.264728:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681195310096768, offset 192 00000800:00000200:12.0:1616513073.264728:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:22.0:1616513073.264734:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.47@tcp1 00000800:00000200:12.0:1616513073.264734:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.264736:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.264738:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:22.0:1616513073.264741:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.47@tcp1 routed destination 00000800:00000200:12.0:1616513073.264743:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.264748:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:22.0:1616513073.264756:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.47@tcp1(10.31.131.47@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:12.0:1616513073.264762:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513073.264763:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.264769:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.264771:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.264774:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.264778:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.264783:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513073.264805:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.264812:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.264815:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.264822:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.264827:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782415c8 00000800:00000200:14.0:1616513073.264831:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.264834:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513073.264838:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.264845:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.264847:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.264854:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.264858:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b078 00000800:00000200:16.0:1616513073.264862:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.264865:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.271878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.271931:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.271940:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.271949:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.69@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.271958:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.69@tcp1 of length 224 into portal 26 MB=0x60182e3e4d100 00000400:00000200:18.0:1616513073.271965:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.69@tcp1 of length 224/224 into md 0xf97c5079 [1] + 0 00000400:00000200:18.0:1616513073.271970:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71430 00000400:00000200:18.0:1616513073.271972:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.271975:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.271979:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71430 00000800:00000200:18.0:1616513073.271982:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.271985:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.271985:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.271989:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.272039:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511541063936 00010000:00000200:12.0:1616513073.272049:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1690511541063936/t0(0) o400->6d7e264f-75e9-6e9f-1dac-6ba7f95134ab@10.31.142.69@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.272063:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511541063936, offset 192 00000400:00000200:12.0:1616513073.272069:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.69@tcp1 00000400:00000200:12.0:1616513073.272076:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.69@tcp1 routed destination 00000400:00000200:12.0:1616513073.272090:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.69@tcp1(10.31.142.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.272096:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.272101:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.272104:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.272106:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.272111:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.272116:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.272117:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513073.272151:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.272158:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.272160:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.272167:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.272173:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241100 00000800:00000200:14.0:1616513073.272176:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.272179:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.272198:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.272205:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.272213:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.272216:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.272225:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.69@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.272233:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.69@tcp1 of length 224 into portal 12 MB=0x60182e3e4db40 00000400:00000200:26.0:1616513073.272239:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.69@tcp1 of length 224/224 into md 0xf96450b9 [1] + 31272 00000400:00000200:26.0:1616513073.272244:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.272247:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.272253:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.272256:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.272258:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.272318:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511541066560 00010000:00000200:22.0:1616513073.272330:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691680 x1690511541066560/t0(0) o400->29d415f0-375d-9e92-f100-d8e6654a3104@10.31.142.69@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.272344:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511541066560, offset 192 00000400:00000200:22.0:1616513073.272351:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.69@tcp1 00000400:00000200:22.0:1616513073.272358:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.69@tcp1 routed destination 00000400:00000200:22.0:1616513073.272373:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.69@tcp1(10.31.142.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.272380:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.272385:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.272388:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.272390:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.272395:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.272400:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.272415:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.272422:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.272424:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.272424:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.272431:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.272450:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bc28 00000800:00000200:28.0:1616513073.272454:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.276889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513073.276907:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.276947:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.276957:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513073.276965:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513073.276966:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.77@tcp1 : PUT - for me 00000400:00000200:16.0:1616513073.276975:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.77@tcp1 of length 224 into portal 26 MB=0x602161b074600 00000800:00000200:14.0:1616513073.276975:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513073.276982:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.77@tcp1 of length 224/224 into md 0xf97c5089 [1] + 0 00000400:00000200:14.0:1616513073.276983:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.77@tcp1 : PUT - for me 00000400:00000200:16.0:1616513073.276987:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70990 00000400:00000200:16.0:1616513073.276989:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513073.276992:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.276992:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.77@tcp1 of length 224 into portal 12 MB=0x602161b0746c0 00000400:00000200:16.0:1616513073.276996:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70990 00000800:00000200:16.0:1616513073.276999:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513073.277001:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.77@tcp1 of length 224/224 into md 0xf96450b9 [1] + 31496 00000800:00000200:16.0:1616513073.277002:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513073.277003:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.277006:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513073.277006:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.277009:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.277015:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.277018:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.277020:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.277023:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.277061:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691143826261504 00010000:00000200:12.0:1616513073.277071:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1691143826261504/t0(0) o400->896d2b4a-ec88-4d65-02cd-f59b9ed06019@10.31.141.77@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.277084:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691143826261504, offset 192 00000100:00000200:22.0:1616513073.277086:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691143826261696 00000400:00000200:12.0:1616513073.277090:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.77@tcp1 00000400:00000200:12.0:1616513073.277096:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.77@tcp1 routed destination 00010000:00000200:22.0:1616513073.277098:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cad00 x1691143826261696/t0(0) o400->0ece1d1f-8c57-6a05-76ab-a58c058d009c@10.31.141.77@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513073.277111:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.77@tcp1(10.31.141.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513073.277112:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691143826261696, offset 192 00000800:00000200:12.0:1616513073.277117:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:22.0:1616513073.277118:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.77@tcp1 00000800:00000200:12.0:1616513073.277122:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.277125:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:22.0:1616513073.277126:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.77@tcp1 routed destination 00000800:00000200:12.0:1616513073.277127:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.277131:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.277136:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.277137:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513073.277141:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.77@tcp1(10.31.141.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.277149:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.277155:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.277157:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.277159:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.277164:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.277169:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.277177:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.277180:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.277182:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.277186:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513073.277189:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:8.0:1616513073.277189:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b760 00000800:00000200:8.0:1616513073.277191:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.277193:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513073.277195:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.277198:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.277205:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.277210:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240990 00000800:00000200:18.0:1616513073.277214:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.277217:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.279179:0:48732:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.279233:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.279242:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.279251:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.82@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.279262:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.82@tcp1 of length 224 into portal 26 MB=0x6059f6d828480 00000400:00000200:14.0:1616513073.279268:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.82@tcp1 of length 224/224 into md 0xf97c50a9 [1] + 0 00000400:00000200:14.0:1616513073.279273:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f703b8 00000400:00000200:14.0:1616513073.279276:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.279279:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.279282:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f703b8 00000800:00000200:14.0:1616513073.279285:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.279286:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.279288:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.279291:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.279356:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695032155473024 00010000:00000200:12.0:1616513073.279366:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1695032155473024/t0(0) o400->a1042dd0-17c3-acd7-6b7e-65b3e93f8cae@10.31.138.82@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.279379:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695032155473024, offset 192 00000400:00000200:12.0:1616513073.279387:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.82@tcp1 00000400:00000200:12.0:1616513073.279394:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.82@tcp1 routed destination 00000400:00000200:12.0:1616513073.279408:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.82@tcp1(10.31.138.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.279414:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.279419:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.279422:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.279424:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.279428:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.279433:0:48732:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.279446:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513073.279484:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.279490:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.279492:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.279498:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.279503:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a90 00000800:00000200:18.0:1616513073.279507:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.279509:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.280487:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.280545:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.280564:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513073.280573:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.82@tcp1 : PUT - for me 00000400:00000200:16.0:1616513073.280581:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.82@tcp1 of length 224 into portal 12 MB=0x6059f6d828940 00000400:00000200:16.0:1616513073.280587:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.82@tcp1 of length 224/224 into md 0x865b9f31 [1] + 63352 00000400:00000200:16.0:1616513073.280592:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513073.280594:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513073.280600:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.280603:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513073.280603:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513073.280607:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.280673:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695032155474240 00010000:00000200:20.0:1616513073.280686:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd4380 x1695032155474240/t0(0) o400->04bf754e-33b1-c7e2-c6b8-3bc79a2ae458@10.31.138.82@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.280700:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695032155474240, offset 192 00000400:00000200:20.0:1616513073.280706:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.82@tcp1 00000400:00000200:20.0:1616513073.280714:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.82@tcp1 routed destination 00000400:00000200:20.0:1616513073.280728:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.82@tcp1(10.31.138.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.280735:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.280740:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.280743:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.280745:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.280750:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.280755:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.280807:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.280813:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.280815:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.280821:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.280824:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22dd0 00000800:00000200:8.0:1616513073.280828:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.280832:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.289992:0:48733:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.290046:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.290055:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.290064:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.102@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.290074:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.102@tcp1 of length 224 into portal 26 MB=0x5fff20920b4c0 00000400:00000200:2.0:1616513073.290080:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.102@tcp1 of length 224/224 into md 0xf97c50c9 [1] + 0 00000400:00000200:2.0:1616513073.290084:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71188 00000400:00000200:2.0:1616513073.290087:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.290090:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.290093:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71188 00000800:00000200:2.0:1616513073.290096:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.290099:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.290099:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.290103:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.290155:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688789883860160 00010000:00000200:12.0:1616513073.290165:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688789883860160/t0(0) o400->85313e54-7165-8752-c064-4eaef731d025@10.31.136.102@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.290177:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688789883860160, offset 192 00000400:00000200:12.0:1616513073.290183:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.102@tcp1 00000400:00000200:12.0:1616513073.290190:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.102@tcp1 routed destination 00000400:00000200:12.0:1616513073.290204:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.102@tcp1(10.31.136.102@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.290210:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.290216:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.290218:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.290220:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.290224:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.290229:0:48733:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.290231:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513073.290290:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.290297:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.290299:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.290305:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.290311:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240110 00000800:00000200:18.0:1616513073.290316:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.290318:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.290405:0:48733:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.290416:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.290424:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513073.290426:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513073.290445:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.102@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.290454:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.102@tcp1 of length 224 into portal 12 MB=0x5fff20920b5c0 00000400:00000200:14.0:1616513073.290460:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.102@tcp1 of length 224/224 into md 0xf96450b9 [1] + 31720 00000400:00000200:14.0:1616513073.290466:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.290468:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.290476:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.290479:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.290480:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.290539:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688789883860416 00010000:00000200:22.0:1616513073.290551:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cf500 x1688789883860416/t0(0) o400->4e8fe3e9-514c-9da2-0094-62ce0a33ef01@10.31.136.102@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.290564:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688789883860416, offset 192 00000400:00000200:22.0:1616513073.290569:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.102@tcp1 00000400:00000200:22.0:1616513073.290577:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.102@tcp1 routed destination 00000400:00000200:22.0:1616513073.290591:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.102@tcp1(10.31.136.102@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.290597:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.290602:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.290604:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.290606:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.290611:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.290618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.290673:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.290680:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.290683:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513073.290690:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.290694:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b5c8 00000800:00000200:24.0:1616513073.290697:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.290700:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.291699:0:48733:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.291754:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.291763:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.291772:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.234@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.291781:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.234@tcp1 of length 224 into portal 26 MB=0x603e105c75a40 00000400:00000200:18.0:1616513073.291788:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.234@tcp1 of length 224/224 into md 0xf97c50e9 [1] + 0 00000400:00000200:18.0:1616513073.291792:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71078 00000400:00000200:18.0:1616513073.291795:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.291798:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.291802:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71078 00000800:00000200:18.0:1616513073.291804:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.291807:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.291809:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.291812:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.291876:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114859739712 00010000:00000200:12.0:1616513073.291886:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1693114859739712/t0(0) o400->1ac05e7c-f825-cfd3-8362-83c25c928f89@10.31.136.234@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.291899:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114859739712, offset 192 00000800:00000200:11.2:1616513073.291904:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:12.0:1616513073.291905:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.234@tcp1 00000400:00000200:12.0:1616513073.291912:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.234@tcp1 routed destination 00000800:00000200:8.0:1616513073.291914:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.291922:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513073.291926:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.234@tcp1(10.31.136.234@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513073.291929:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.234@tcp1 : PUT - for me 00000800:00000200:12.0:1616513073.291936:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513073.291937:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.234@tcp1 of length 224 into portal 12 MB=0x603e105c75ac0 00000800:00000200:12.0:1616513073.291941:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:8.0:1616513073.291941:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.234@tcp1 of length 224/224 into md 0x865b9f31 [1] + 63576 00000800:00000200:12.0:1616513073.291944:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.291946:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513073.291946:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.291948:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513073.291951:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513073.291954:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.291956:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513073.291956:0:48733:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.291957:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513073.291958:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.291960:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.292016:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114859739840 00000800:00000200:2.0:1616513073.292025:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:20.0:1616513073.292028:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b105a00 x1693114859739840/t0(0) o400->886876fa-8196-a63b-2824-662ad07527e2@10.31.136.234@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513073.292032:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.292035:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:20.0:1616513073.292041:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114859739840, offset 192 00000400:00000200:2.0:1616513073.292041:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.292047:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241f58 00000800:00000200:2.0:1616513073.292052:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513073.292054:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.234@tcp1 00000800:00000200:2.0:1616513073.292055:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513073.292061:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.234@tcp1 routed destination 00000400:00000200:20.0:1616513073.292074:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.234@tcp1(10.31.136.234@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.292080:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.292086:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.292088:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.292090:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.292095:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.292100:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.292111:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.292118:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.292121:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513073.292128:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.292131:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22660 00000800:00000200:24.0:1616513073.292135:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.292138:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.301547:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.301559:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.301566:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.301575:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.74@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.301583:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.74@tcp1 of length 224 into portal 26 MB=0x5ffd41b5c8ac0 00000400:00000200:8.0:1616513073.301590:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.74@tcp1 of length 224/224 into md 0xf97c50f9 [1] + 0 00000400:00000200:8.0:1616513073.301594:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f717e8 00000400:00000200:8.0:1616513073.301597:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.301599:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.301603:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f717e8 00000800:00000200:8.0:1616513073.301606:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.301609:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.301610:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.301613:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.301623:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513073.301632:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.74@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.301641:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.74@tcp1 of length 224 into portal 12 MB=0x5ffd41b5c8b40 00000400:00000200:28.0:1616513073.301647:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.74@tcp1 of length 224/224 into md 0x865b9f31 [1] + 63800 00000400:00000200:28.0:1616513073.301653:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.301655:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513073.301664:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.301667:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:12.0:1616513073.301667:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661340752576 00000800:00000200:28.0:1616513073.301669:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.301672:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:12.0:1616513073.301677:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688661340752576/t0(0) o400->2512e803-885f-304e-1203-2855ecbe7258@10.31.133.74@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.301692:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661340752576, offset 192 00000400:00000200:12.0:1616513073.301698:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.74@tcp1 00000400:00000200:12.0:1616513073.301705:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.74@tcp1 routed destination 00000400:00000200:12.0:1616513073.301720:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.74@tcp1(10.31.133.74@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.301726:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:20.0:1616513073.301729:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661340752704 00000800:00000200:12.0:1616513073.301732:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.301735:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.301737:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:20.0:1616513073.301741:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf87080 x1688661340752704/t0(0) o400->70a1e146-5602-50e4-256e-6aea6e086f5b@10.31.133.74@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513073.301741:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513073.301748:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513073.301748:0:48734:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513073.301756:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661340752704, offset 192 00000400:00000200:20.0:1616513073.301762:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.74@tcp1 00000400:00000200:20.0:1616513073.301770:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.74@tcp1 routed destination 00000400:00000200:20.0:1616513073.301787:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.74@tcp1(10.31.133.74@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.301797:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.301803:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.301805:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.301807:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513073.301809:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513073.301812:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513073.301816:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513073.301817:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513073.301819:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.301826:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.301831:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241b18 00000800:00000200:18.0:1616513073.301834:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.301837:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513073.301892:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.301900:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.301902:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.301902:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513073.301911:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.301915:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22c38 00000800:00000200:16.0:1616513073.301919:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:25.2:1616513073.342924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cdb3d1c00] (20)++ 00000800:00000200:25.0:1616513073.342946:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (21)++ 00000800:00000200:25.0:1616513073.342954:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.146@o2ib1 00000400:00000200:25.0:1616513073.342964:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.146@o2ib1 : PUT - for me 00000400:00000200:25.0:1616513073.342973:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.146@o2ib1 of length 224 into portal 12 MB=0x604a72a3b6d00 00000400:00000200:25.0:1616513073.342980:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.146@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 38288 00000400:00000200:25.0:1616513073.342986:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:25.0:1616513073.342989:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:25.0:1616513073.342999:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cdb3d1c00] (22)++ 00000800:00000200:17.0:1616513073.342999:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)++ 00000800:00000200:25.0:1616513073.343002:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cdb3d1c00] (23)-- 00000800:00000200:25.0:1616513073.343007:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (23)-- 00000800:00000200:25.0:1616513073.343010:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)-- 00000800:00000200:17.0:1616513073.343011:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.146@o2ib1 00000400:00000200:17.0:1616513073.343020:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.146@o2ib1 : PUT - for me 00000400:00000200:17.0:1616513073.343029:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.146@o2ib1 of length 224 into portal 26 MB=0x604a72a3b6c80 00000400:00000200:17.0:1616513073.343036:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.146@o2ib1 of length 224/224 into md 0xf97c5119 [1] + 0 00000400:00000200:17.0:1616513073.343041:0:6468:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71dc0 00000400:00000200:17.0:1616513073.343044:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:17.0:1616513073.343047:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:17.0:1616513073.343053:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71dc0 00000800:00000200:17.0:1616513073.343056:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cdb3d1c00] (21)++ 00000800:00000200:17.0:1616513073.343060:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cdb3d1c00] (22)-- 00000800:00000200:17.0:1616513073.343061:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (21)-- 00000100:00000200:29.0:1616513073.343070:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693965874851072 00010000:00000200:29.0:1616513073.343084:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843a2ec800 x1693965874851072/t0(0) o400->d0eeb338-1574-8814-a38c-3edcd7fc9caf@10.242.46.146@o2ib1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:29.0:1616513073.343100:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693965874851072, offset 192 00000400:00000200:29.0:1616513073.343106:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.146@o2ib1 00000400:00000200:29.0:1616513073.343113:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.146@o2ib1 local destination 00000100:00000200:12.0:1616513073.343121:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693965874850944 00000400:00000200:29.0:1616513073.343127:0:170085:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.146@o2ib1(10.242.46.146@o2ib1:10.242.46.146@o2ib1) : PUT try# 0 00010000:00000200:12.0:1616513073.343132:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1693965874850944/t0(0) o400->187808d2-e90b-a5a7-7a66-336ae372a50d@10.242.46.146@o2ib1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:29.0:1616513073.343134:0:170085:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.146@o2ib1 00000800:00000200:29.0:1616513073.343141:0:170085:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8419b46e00] -> 10.242.46.146@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513073.343143:0:170085:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cdb3d1c00] (20)++ 00000800:00000200:29.0:1616513073.343146:0:170085:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cdb3d1c00] (21)++ 00000100:00000200:12.0:1616513073.343147:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693965874850944, offset 192 00000800:00000200:29.0:1616513073.343151:0:170085:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cdb3d1c00] (22)-- 00000400:00000200:12.0:1616513073.343154:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.146@o2ib1 00000800:00000200:25.2:1616513073.343157:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cdb3d1c00] (21)++ 00000400:00000200:12.0:1616513073.343162:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.146@o2ib1 local destination 00000800:00000200:25.0:1616513073.343165:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)++ 00000800:00000200:25.0:1616513073.343170:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cdb3d1c00] (23)-- 00000400:00000200:25.0:1616513073.343172:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:29.0:1616513073.343177:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)-- 00000400:00000200:12.0:1616513073.343177:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.146@o2ib1(10.242.46.146@o2ib1:10.242.46.146@o2ib1) : PUT try# 0 00000400:00000200:25.0:1616513073.343179:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.146@o2ib1: PUT: OK 00000400:00000200:25.0:1616513073.343182:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2990 00000800:00000200:12.0:1616513073.343184:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.146@o2ib1 00000800:00000200:25.0:1616513073.343186:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (21)-- 00000800:00000200:12.0:1616513073.343191:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8419b46e00] -> 10.242.46.146@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.343194:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cdb3d1c00] (20)++ 00000800:00000200:12.0:1616513073.343198:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cdb3d1c00] (21)++ 00000800:00000200:12.0:1616513073.343204:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cdb3d1c00] (22)-- 00000800:00000200:25.2:1616513073.343210:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cdb3d1c00] (21)++ 00000100:00000200:12.0:1616513073.343211:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:25.0:1616513073.343218:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)++ 00000800:00000200:25.0:1616513073.343226:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cdb3d1c00] (23)-- 00000400:00000200:25.0:1616513073.343228:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:17.0:1616513073.343230:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)-- 00000400:00000200:25.0:1616513073.343233:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.146@o2ib1: PUT: OK 00000400:00000200:25.0:1616513073.343237:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240b28 00000800:00000200:25.0:1616513073.343240:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (21)-- 00000800:00000200:7.2:1616513073.343943:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.344005:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.344020:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.344030:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.198@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.344039:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.198@tcp1 of length 224 into portal 26 MB=0x605a073310d80 00000400:00000200:2.0:1616513073.344045:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.198@tcp1 of length 224/224 into md 0xf97c5139 [1] + 0 00000400:00000200:2.0:1616513073.344051:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f707f8 00000400:00000200:2.0:1616513073.344054:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.344057:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.344061:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f707f8 00000800:00000200:2.0:1616513073.344065:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.344066:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.344068:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.344073:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:12.0:1616513073.344074:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036545764736 00000800:00000200:2.0:1616513073.344076:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.344077:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:12.0:1616513073.344080:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1695036545764736/t0(0) o400->412145a5-0ab1-695e-4846-cd1e927fad38@10.31.129.198@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513073.344087:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.198@tcp1 : PUT - for me 00000100:00000200:12.0:1616513073.344092:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036545764736, offset 192 00000400:00000200:14.0:1616513073.344096:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.198@tcp1 of length 224 into portal 12 MB=0x605a073310e40 00000400:00000200:12.0:1616513073.344098:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.198@tcp1 00000400:00000200:14.0:1616513073.344103:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.198@tcp1 of length 224/224 into md 0xf96450b9 [1] + 31944 00000400:00000200:12.0:1616513073.344104:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.198@tcp1 routed destination 00000400:00000200:14.0:1616513073.344109:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.344112:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513073.344118:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.198@tcp1(10.31.129.198@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513073.344123:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.344124:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513073.344127:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.344129:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513073.344129:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.344131:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.344133:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.344137:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513073.344141:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513073.344142:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513073.344187:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036545764928 00000800:00000200:26.0:1616513073.344198:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:22.0:1616513073.344200:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cde80 x1695036545764928/t0(0) o400->1cdb4a60-cf5f-ce45-9322-16a30a2efe1c@10.31.129.198@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:26.0:1616513073.344205:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.344208:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.344209:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513073.344214:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036545764928, offset 192 00000400:00000200:26.0:1616513073.344215:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513073.344220:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.198@tcp1 00000400:00000200:26.0:1616513073.344221:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782407f8 00000400:00000200:22.0:1616513073.344228:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.198@tcp1 routed destination 00000800:00000200:26.0:1616513073.344233:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:22.0:1616513073.344243:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.198@tcp1(10.31.129.198@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.344249:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.344255:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.344258:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.344260:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.344265:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.344272:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.344327:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.344334:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.344337:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513073.344344:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.344348:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bba0 00000800:00000200:24.0:1616513073.344352:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.344355:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.348895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.348951:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.348960:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.348969:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.213@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.348978:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.213@tcp1 of length 224 into portal 26 MB=0x601958b4f99c0 00000400:00000200:18.0:1616513073.348985:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.213@tcp1 of length 224/224 into md 0xf97c5149 [1] + 0 00000400:00000200:18.0:1616513073.348989:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70e58 00000400:00000200:18.0:1616513073.348992:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.348995:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.348999:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70e58 00000800:00000200:18.0:1616513073.349002:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.349004:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.349005:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.349011:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.349014:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.349015:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.349023:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.213@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.349032:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.213@tcp1 of length 224 into portal 12 MB=0x601958b4f9a80 00000400:00000200:14.0:1616513073.349038:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.213@tcp1 of length 224/224 into md 0xf96450b9 [1] + 32168 00000400:00000200:14.0:1616513073.349043:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.349046:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.349052:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.349055:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.349057:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.349063:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690591659268544 00010000:00000200:12.0:1616513073.349074:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1690591659268544/t0(0) o400->8f8bcb46-5e93-ef38-68b3-3fc42df6c1d5@10.31.140.213@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.349087:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690591659268544, offset 192 00000400:00000200:12.0:1616513073.349093:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.213@tcp1 00000400:00000200:12.0:1616513073.349101:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.213@tcp1 routed destination 00000100:00000200:22.0:1616513073.349115:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690591659268736 00000400:00000200:12.0:1616513073.349116:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.213@tcp1(10.31.140.213@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.349122:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:22.0:1616513073.349127:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cc800 x1690591659268736/t0(0) o400->3a2d1f26-303c-317c-7ef3-9eb302efe586@10.31.140.213@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513073.349128:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.349131:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.349133:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.349138:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513073.349141:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690591659268736, offset 192 00000800:00000200:7.2:1616513073.349143:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.349144:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513073.349148:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.213@tcp1 00000400:00000200:22.0:1616513073.349155:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.213@tcp1 routed destination 00000400:00000200:22.0:1616513073.349170:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.213@tcp1(10.31.140.213@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.349177:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.349183:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.349185:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.349187:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.349192:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513073.349196:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513073.349200:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513073.349203:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.349206:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513073.349207:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.349212:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:8.0:1616513073.349212:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513073.349218:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:8.0:1616513073.349218:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.349220:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513073.349222:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513073.349226:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.349230:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bd38 00000800:00000200:8.0:1616513073.349233:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.349236:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513073.354520:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924112, portal 4 00000100:00000200:18.0:1616513073.354526:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924112, offset 0 00000400:00000200:18.0:1616513073.354533:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:18.0:1616513073.354538:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:18.0:1616513073.354551:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513073.354558:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:18.0:1616513073.354564:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513073.354567:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:18.0:1616513073.354569:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:18.0:1616513073.354574:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:27.0:1616513073.354597:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924128, portal 4 00000100:00000200:27.0:1616513073.354603:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924128, offset 0 00000400:00000200:27.0:1616513073.354609:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:27.0:1616513073.354616:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:5.2:1616513073.354622:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:27.0:1616513073.354630:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513073.354639:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:27.0:1616513073.354646:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513073.354649:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:27.0:1616513073.354651:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:27.0:1616513073.354656:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (24)-- 00000800:00000200:24.0:1616513073.354680:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:24.0:1616513073.354687:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (24)-- 00000400:00000200:24.0:1616513073.354690:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513073.354695:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)++ 00000400:00000200:24.0:1616513073.354696:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:24.0:1616513073.354702:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a7da00 x1693067725924112/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513073.354703:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (24)-- 00000400:00000200:28.0:1616513073.354706:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.354706:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513073.354712:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.354714:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc000 00000100:00000200:28.0:1616513073.354719:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc566300 x1693067725924128/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:24.0:1616513073.354721:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513073.354731:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248990 00000800:00000200:28.0:1616513073.354735:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513073.354826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513073.354881:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513073.354891:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.230@o2ib1 00000800:00000200:8.0:1616513073.354891:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513073.354897:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:24.0:1616513073.354900:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:16.0:1616513073.354900:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513073.354905:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513073.354908:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626710 00000400:00000200:8.0:1616513073.354911:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626720 00000400:00000200:16.0:1616513073.354915:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d35c1 [1] + 192 00000400:00000200:16.0:1616513073.354920:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.354922:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d35d1 [1] + 192 00000400:00000200:8.0:1616513073.354926:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513073.354929:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a7da00 x1693067725924112/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513073.354943:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000100:00000200:8.0:1616513073.354945:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc566300 x1693067725924128/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513073.354947:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:16.0:1616513073.354948:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513073.354959:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:18.0:1616513073.354961:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a7da00 x1693067725924112/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513073.354961:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513073.354963:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:18.0:1616513073.354974:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a7da00 x1693067725924112/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513073.354984:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccff0 00000100:00000200:27.0:1616513073.354991:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc566300 x1693067725924128/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513073.355004:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc566300 x1693067725924128/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513073.355015:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248e58 00000800:00000200:11.2:1616513073.357484:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513073.357539:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.357548:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513073.357557:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.44@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.357568:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.44@tcp1 of length 224 into portal 12 MB=0x5ffd52a430f40 00000400:00000200:28.0:1616513073.357574:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.44@tcp1 of length 224/224 into md 0x865b9f31 [1] + 64024 00000400:00000200:28.0:1616513073.357579:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.357582:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513073.357588:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.357591:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.357592:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513073.357596:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.357635:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513073.357653:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665885708096 00010000:00000200:20.0:1616513073.357665:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf82880 x1688665885708096/t0(0) o400->06cdc5ef-d526-8926-57f3-95227d6c380d@10.31.138.44@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.357680:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665885708096, offset 192 00000400:00000200:20.0:1616513073.357686:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.44@tcp1 00000800:00000200:14.0:1616513073.357692:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513073.357694:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.44@tcp1 routed destination 00000800:00000200:14.0:1616513073.357701:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513073.357710:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.44@tcp1(10.31.138.44@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.357716:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.357719:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.44@tcp1 : PUT - for me 00000800:00000200:20.0:1616513073.357722:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.357725:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513073.357727:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513073.357728:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.44@tcp1 of length 224 into portal 26 MB=0x5ffd52a430e80 00000800:00000200:20.0:1616513073.357732:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513073.357735:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.44@tcp1 of length 224/224 into md 0xf97c5159 [1] + 0 00000400:00000200:14.0:1616513073.357739:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f716d8 00000400:00000200:14.0:1616513073.357742:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.357745:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513073.357746:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513073.357749:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f716d8 00000800:00000200:14.0:1616513073.357752:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513073.357754:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:14.0:1616513073.357755:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:14.0:1616513073.357756:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.357757:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513073.357759:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.357764:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.357768:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee223b8 00000800:00000200:2.0:1616513073.357772:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.357814:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665885707904 00010000:00000200:12.0:1616513073.357825:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688665885707904/t0(0) o400->2892f2fb-e743-7755-0176-6d410445ce5d@10.31.138.44@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.357841:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665885707904, offset 192 00000400:00000200:12.0:1616513073.357847:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.44@tcp1 00000400:00000200:12.0:1616513073.357855:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.44@tcp1 routed destination 00000400:00000200:12.0:1616513073.357870:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.44@tcp1(10.31.138.44@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.357877:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.357883:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.357885:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.357887:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.357892:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.357897:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.357899:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513073.357957:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.357964:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.357966:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513073.357967:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.357973:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.357979:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240000 00000800:00000200:16.0:1616513073.357982:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.364604:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.364659:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.364669:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.364678:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.162@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.364688:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.162@tcp1 of length 224 into portal 26 MB=0x5ffd474af34c0 00000400:00000200:26.0:1616513073.364694:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.162@tcp1 of length 224/224 into md 0xf97c5169 [1] + 0 00000400:00000200:26.0:1616513073.364699:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71210 00000400:00000200:26.0:1616513073.364701:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.364704:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513073.364708:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71210 00000800:00000200:26.0:1616513073.364711:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.364713:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.364714:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.364728:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.364731:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.364732:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.364741:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.162@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.364750:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.162@tcp1 of length 224 into portal 12 MB=0x5ffd474af3580 00000400:00000200:14.0:1616513073.364757:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.162@tcp1 of length 224/224 into md 0xf96450b9 [1] + 32392 00000400:00000200:14.0:1616513073.364762:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.364765:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.364771:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.364773:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662839342272 00000800:00000200:14.0:1616513073.364774:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.364776:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:12.0:1616513073.364784:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688662839342272/t0(0) o400->f11b9082-771c-f4f0-89d9-ebea821a717d@10.31.138.162@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.364798:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662839342272, offset 192 00000400:00000200:12.0:1616513073.364804:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.162@tcp1 00000400:00000200:12.0:1616513073.364812:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.162@tcp1 routed destination 00000400:00000200:12.0:1616513073.364827:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.162@tcp1(10.31.138.162@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.364834:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:22.0:1616513073.364835:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662839342464 00000800:00000200:12.0:1616513073.364839:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.364842:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.364844:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:22.0:1616513073.364847:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cb180 x1688662839342464/t0(0) o400->2d83e85e-c3f5-06ba-4105-7a4c3297b32b@10.31.138.162@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513073.364849:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.364853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.364855:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513073.364862:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662839342464, offset 192 00000400:00000200:22.0:1616513073.364868:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.162@tcp1 00000400:00000200:22.0:1616513073.364875:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.162@tcp1 routed destination 00000400:00000200:22.0:1616513073.364891:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.162@tcp1(10.31.138.162@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.364897:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.364903:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.364910:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.364912:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.364917:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513073.364918:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513073.364923:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513073.364925:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.364927:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513073.364928:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513073.364933:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.364939:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240440 00000800:00000200:18.0:1616513073.364942:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513073.364977:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.364983:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.364986:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513073.364993:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.364997:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4acc0 00000800:00000200:24.0:1616513073.365001:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.365004:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.365534:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.365544:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.365551:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.365559:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.173@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.365567:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.173@tcp1 of length 224 into portal 26 MB=0x5ffde6bc0fcc0 00000400:00000200:8.0:1616513073.365573:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.173@tcp1 of length 224/224 into md 0xf97c5189 [1] + 0 00000400:00000200:8.0:1616513073.365577:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71298 00000400:00000200:8.0:1616513073.365579:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.365582:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.365586:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71298 00000800:00000200:8.0:1616513073.365589:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.365591:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.365593:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.365595:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.365599:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705639185600 00000800:00000200:8.0:1616513073.365600:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513073.365603:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:12.0:1616513073.365604:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688705639185600/t0(0) o400->1d199978-7f9e-c9cf-9509-838f6cd60402@10.31.132.173@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513073.365606:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.173@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.365612:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.173@tcp1 of length 224 into portal 12 MB=0x5ffde6bc0fd40 00000400:00000200:8.0:1616513073.365617:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.173@tcp1 of length 224/224 into md 0x865b9f31 [1] + 64248 00000100:00000200:12.0:1616513073.365621:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705639185600, offset 192 00000400:00000200:8.0:1616513073.365621:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.365623:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513073.365629:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513073.365631:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.173@tcp1 00000800:00000200:8.0:1616513073.365631:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.365632:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:12.0:1616513073.365637:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.173@tcp1 routed destination 00000400:00000200:12.0:1616513073.365652:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.173@tcp1(10.31.132.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.365658:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.365663:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.365665:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.365667:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.365670:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513073.365675:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513073.365675:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.365686:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.365693:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:20.0:1616513073.365695:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705639185728 00000400:00000200:2.0:1616513073.365696:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513073.365697:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.365702:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:20.0:1616513073.365707:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b104380 x1688705639185728/t0(0) o400->5f2609d3-718e-63fd-8503-31a99a27cd8b@10.31.132.173@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513073.365708:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:2.0:1616513073.365711:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513073.365722:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705639185728, offset 192 00000400:00000200:20.0:1616513073.365728:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.173@tcp1 00000400:00000200:20.0:1616513073.365735:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.173@tcp1 routed destination 00000400:00000200:20.0:1616513073.365750:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.173@tcp1(10.31.132.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.365757:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.365763:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.365765:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.365770:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.365775:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.365779:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.365790:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.365797:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.365800:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513073.365802:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.365807:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.365811:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22bb0 00000800:00000200:28.0:1616513073.365814:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.368323:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.368378:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.368387:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.368396:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.37@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.368405:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.37@tcp1 of length 224 into portal 26 MB=0x5ffd3d68f7f80 00000400:00000200:26.0:1616513073.368411:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.37@tcp1 of length 224/224 into md 0xf97c5199 [1] + 0 00000400:00000200:26.0:1616513073.368416:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70a18 00000400:00000200:26.0:1616513073.368419:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.368422:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513073.368426:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70a18 00000800:00000200:26.0:1616513073.368429:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.368430:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.368432:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.368453:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.368487:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660186464128 00010000:00000200:12.0:1616513073.368497:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688660186464128/t0(0) o400->97cf0e5f-37b3-2183-fe03-bd8c493a4ae6@10.31.129.37@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.368510:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660186464128, offset 192 00000400:00000200:12.0:1616513073.368516:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.37@tcp1 00000400:00000200:12.0:1616513073.368523:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.37@tcp1 routed destination 00000400:00000200:12.0:1616513073.368537:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.37@tcp1(10.31.129.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.368543:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.368549:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.368551:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.368553:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.368558:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513073.368570:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513073.368574:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.368624:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513073.368624:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513073.368631:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.368633:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.368639:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.368645:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240b28 00000800:00000200:14.0:1616513073.368648:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.368651:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513073.368678:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.368687:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.368687:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513073.368695:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.37@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.368703:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.37@tcp1 of length 224 into portal 12 MB=0x5ffd3d68f8340 00000400:00000200:24.0:1616513073.368709:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.37@tcp1 of length 224/224 into md 0x865b9f31 [1] + 64472 00000400:00000200:24.0:1616513073.368714:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.368716:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513073.368723:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.368725:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.368727:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.368785:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660186465088 00010000:00000200:20.0:1616513073.368798:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc4c80 x1688660186465088/t0(0) o400->941f8f31-2888-0c63-b528-40eb46061b6e@10.31.129.37@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.368812:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660186465088, offset 192 00000400:00000200:20.0:1616513073.368818:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.37@tcp1 00000400:00000200:20.0:1616513073.368825:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.37@tcp1 routed destination 00000400:00000200:20.0:1616513073.368840:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.37@tcp1(10.31.129.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.368846:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.368852:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.368855:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.368857:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.368862:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.368867:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.368919:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.368925:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.368928:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.368934:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.368938:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee234b8 00000800:00000200:16.0:1616513073.368941:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.368944:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.374047:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.374105:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.374114:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.374123:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.57@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.374132:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.57@tcp1 of length 224 into portal 26 MB=0x601953c6e8d40 00000400:00000200:18.0:1616513073.374138:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.57@tcp1 of length 224/224 into md 0xf97c51a9 [1] + 0 00000400:00000200:18.0:1616513073.374143:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70cc0 00000400:00000200:18.0:1616513073.374146:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.374150:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.374154:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70cc0 00000800:00000200:18.0:1616513073.374157:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.374158:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.374160:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.374162:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.374218:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590335896896 00010000:00000200:12.0:1616513073.374229:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1690590335896896/t0(0) o400->2f2fa441-a979-bf7a-761c-e6321abf82aa@10.31.141.57@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.374243:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690590335896896, offset 192 00000400:00000200:12.0:1616513073.374249:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.57@tcp1 00000400:00000200:12.0:1616513073.374256:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.57@tcp1 routed destination 00000400:00000200:12.0:1616513073.374271:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.57@tcp1(10.31.141.57@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.374277:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.374283:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.374287:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.374290:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.374295:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.374299:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.374301:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513073.374352:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.374359:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.374362:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.374368:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.374374:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241f58 00000800:00000200:14.0:1616513073.374377:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.374380:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.374675:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.374687:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.374694:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.374702:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.57@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.374709:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.57@tcp1 of length 224 into portal 12 MB=0x601953c6e9940 00000400:00000200:8.0:1616513073.374714:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.57@tcp1 of length 224/224 into md 0x865b9f31 [1] + 64696 00000400:00000200:8.0:1616513073.374719:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.374721:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513073.374726:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.374728:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.374730:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.374732:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.374790:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590335899968 00010000:00000200:20.0:1616513073.374802:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b101680 x1690590335899968/t0(0) o400->6f2e69c8-2a15-454d-1d98-3d91b0ec28a4@10.31.141.57@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.374817:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690590335899968, offset 192 00000400:00000200:20.0:1616513073.374823:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.57@tcp1 00000400:00000200:20.0:1616513073.374830:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.57@tcp1 routed destination 00000400:00000200:20.0:1616513073.374845:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.57@tcp1(10.31.141.57@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.374851:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.374857:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.374860:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.374862:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.374867:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.374871:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.374925:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.374932:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.374934:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.374941:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.374945:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22550 00000800:00000200:16.0:1616513073.374951:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.374954:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.376408:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.376461:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.376470:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.376479:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.178@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.376489:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.178@tcp1 of length 224 into portal 26 MB=0x5ffe6505dc680 00000400:00000200:26.0:1616513073.376496:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.178@tcp1 of length 224/224 into md 0xf97c51b9 [1] + 0 00000400:00000200:26.0:1616513073.376502:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782416d8 00000400:00000200:26.0:1616513073.376505:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.376508:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513073.376512:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782416d8 00000800:00000200:18.0:1616513073.376514:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.376516:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.376518:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513073.376521:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.376524:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.376524:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.376532:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.178@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.376541:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.178@tcp1 of length 224 into portal 12 MB=0x5ffe6505dc700 00000400:00000200:18.0:1616513073.376547:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.178@tcp1 of length 224/224 into md 0xf96450b9 [1] + 32616 00000400:00000200:18.0:1616513073.376551:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.376553:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513073.376560:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.376563:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.376564:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.376574:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739539437184 00010000:00000200:12.0:1616513073.376584:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688739539437184/t0(0) o400->a04fd33a-9954-c3a2-e695-e2f27f766d7e@10.31.141.178@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.376597:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688739539437184, offset 192 00000400:00000200:12.0:1616513073.376603:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.178@tcp1 00000400:00000200:12.0:1616513073.376609:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.178@tcp1 routed destination 00000400:00000200:12.0:1616513073.376623:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.178@tcp1(10.31.141.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513073.376624:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739539437312 00000800:00000200:12.0:1616513073.376629:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.376634:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513073.376635:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d0900 x1688739539437312/t0(0) o400->5a911e3b-f9ab-8514-2eb3-2b9a8d8a6f53@10.31.141.178@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513073.376637:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.376639:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.376643:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513073.376648:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688739539437312, offset 192 00000800:00000200:7.2:1616513073.376649:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.376650:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513073.376654:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.178@tcp1 00000400:00000200:22.0:1616513073.376661:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.178@tcp1 routed destination 00000400:00000200:22.0:1616513073.376676:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.178@tcp1(10.31.141.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513073.376676:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.376683:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513073.376684:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.376686:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513073.376688:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513073.376688:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.376690:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.376692:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513073.376693:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513073.376697:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513073.376699:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a90 00000800:00000200:2.0:1616513073.376702:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.376704:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.376713:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.376722:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.376724:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.376729:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.376732:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4add0 00000800:00000200:8.0:1616513073.376735:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.376737:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.389671:0:48744:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.389726:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.389735:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.389743:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.389752:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d0322a01c0 00000400:00000200:18.0:1616513073.389758:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0xf96450b9 [1] + 32840 00000400:00000200:18.0:1616513073.389762:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.389765:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513073.389771:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.389774:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.389775:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.389778:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.389833:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042589958592 00010000:00000200:22.0:1616513073.389918:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d6780 x1693042589958592/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:699/0 lens 440/368 e 0 to 0 dl 1616513079 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513073.389932:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042589958592, offset 192 00000400:00000200:22.0:1616513073.389938:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:22.0:1616513073.389945:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:22.0:1616513073.389959:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.389964:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.389970:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.389973:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.389975:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.389979:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.389984:0:48744:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.390037:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.390044:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.390046:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.390049:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.390052:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.390056:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a3b8 00000800:00000200:26.0:1616513073.390059:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.426162:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.426220:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.426230:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513073.426239:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.148@tcp1 : PUT - for me 00000400:00000200:16.0:1616513073.426248:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.148@tcp1 of length 224 into portal 12 MB=0x601ac1d2b14c0 00000400:00000200:16.0:1616513073.426255:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.148@tcp1 of length 224/224 into md 0x865b9f31 [1] + 64920 00000400:00000200:16.0:1616513073.426260:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513073.426263:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513073.426269:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.426273:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513073.426275:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.426279:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.426292:0:48748:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513073.426334:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690688595629248 00010000:00000200:20.0:1616513073.426348:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5580 x1690688595629248/t0(0) o400->e4e8a52e-fd82-26c5-6550-30a9ea490642@10.31.140.148@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513073.426350:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513073.426363:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690688595629248, offset 192 00000400:00000200:20.0:1616513073.426371:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.148@tcp1 00000400:00000200:20.0:1616513073.426379:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.148@tcp1 routed destination 00000800:00000200:14.0:1616513073.426394:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513073.426395:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.148@tcp1(10.31.140.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.426402:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513073.426404:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.148@tcp1 : PUT - for me 00000800:00000200:18.0:1616513073.426405:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513073.426408:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.426411:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.426413:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513073.426413:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.148@tcp1 of length 224 into portal 26 MB=0x601ac1d2b1440 00000800:00000200:20.0:1616513073.426419:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513073.426420:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.148@tcp1 of length 224/224 into md 0xf97c51c9 [1] + 0 00000400:00000200:14.0:1616513073.426424:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782404c8 00000800:00000200:11.2:1616513073.426424:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513073.426427:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.426430:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513073.426434:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.426444:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.426446:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.426447:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.426448:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782404c8 00000800:00000200:14.0:1616513073.426453:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513073.426454:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513073.426456:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.426458:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513073.426458:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22000 00000800:00000200:8.0:1616513073.426461:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.426510:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690688595629120 00010000:00000200:12.0:1616513073.426521:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1690688595629120/t0(0) o400->58107f31-1d0b-a66e-c689-a92fdae27730@10.31.140.148@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.426534:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690688595629120, offset 192 00000400:00000200:12.0:1616513073.426540:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.148@tcp1 00000400:00000200:12.0:1616513073.426547:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.148@tcp1 routed destination 00000400:00000200:12.0:1616513073.426561:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.148@tcp1(10.31.140.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.426567:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.426573:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.426575:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.426577:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.426582:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.426586:0:48748:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.426589:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513073.426603:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.426610:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.426613:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.426619:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.426625:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241100 00000800:00000200:2.0:1616513073.426629:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.426632:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.433951:0:48749:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.434009:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.434019:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.434028:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.67@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.434039:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.67@tcp1 of length 224 into portal 26 MB=0x5ffed349c9040 00000400:00000200:18.0:1616513073.434045:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.67@tcp1 of length 224/224 into md 0xf97c51e9 [1] + 0 00000400:00000200:18.0:1616513073.434052:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241430 00000400:00000200:18.0:1616513073.434055:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.434058:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.434062:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241430 00000800:00000200:14.0:1616513073.434063:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.434065:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.434068:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513073.434070:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.434073:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.434073:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.434082:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.67@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.434091:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.67@tcp1 of length 224 into portal 12 MB=0x5ffed349c9100 00000400:00000200:14.0:1616513073.434098:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.67@tcp1 of length 224/224 into md 0xf96450b9 [1] + 33280 00000400:00000200:14.0:1616513073.434103:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.434106:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.434113:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.434116:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.434118:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.434127:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688769138561088 00010000:00000200:12.0:1616513073.434137:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688769138561088/t0(0) o400->3af62b3a-be3d-f86f-48c0-5e7df3a70a50@10.31.142.67@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.434151:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688769138561088, offset 192 00000400:00000200:12.0:1616513073.434157:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.67@tcp1 00000400:00000200:12.0:1616513073.434165:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.67@tcp1 routed destination 00000100:00000200:22.0:1616513073.434179:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688769138561280 00000400:00000200:12.0:1616513073.434180:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.67@tcp1(10.31.142.67@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.434187:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:22.0:1616513073.434191:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cf080 x1688769138561280/t0(0) o400->26ea18ef-93a0-f19e-de6b-66b1ebf1f47f@10.31.142.67@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513073.434193:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.434195:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.434197:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.434202:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513073.434206:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688769138561280, offset 192 00000100:00000200:12.0:1616513073.434209:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.434210:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513073.434212:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.67@tcp1 00000400:00000200:22.0:1616513073.434219:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.67@tcp1 routed destination 00000400:00000200:22.0:1616513073.434234:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.67@tcp1(10.31.142.67@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.434241:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.434247:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.434250:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.434254:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.434258:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.434262:0:48749:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513073.434265:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.434271:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.434274:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513073.434281:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.434286:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241ba0 00000800:00000200:28.0:1616513073.434292:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.434295:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513073.434330:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.434337:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.434340:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513073.434341:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.434347:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.434351:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a990 00000800:00000200:26.0:1616513073.434355:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.448389:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.448446:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.448456:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.448458:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513073.448465:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.156@tcp1 : PUT - for me 00000800:00000200:8.0:1616513073.448467:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.448475:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.156@tcp1 of length 224 into portal 26 MB=0x5ffdc8bd10280 00000400:00000200:8.0:1616513073.448477:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.156@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.448482:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.156@tcp1 of length 224/224 into md 0xf97c51f9 [1] + 0 00000400:00000200:8.0:1616513073.448485:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.156@tcp1 of length 224 into portal 12 MB=0x5ffdc8bd10300 00000400:00000200:24.0:1616513073.448487:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241cb0 00000400:00000200:24.0:1616513073.448489:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.448491:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.156@tcp1 of length 224/224 into md 0x865b9f31 [1] + 65144 00000100:00000200:24.0:1616513073.448492:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513073.448497:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241cb0 00000400:00000200:8.0:1616513073.448497:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513073.448500:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:8.0:1616513073.448500:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513073.448502:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513073.448506:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.448506:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513073.448509:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.448509:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.448511:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.448559:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697587171968 00010000:00000200:12.0:1616513073.448570:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688697587171968/t0(0) o400->e5c33d50-9ea0-ccaa-df0f-93aaac00778e@10.31.141.156@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513073.448582:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697587172096 00000100:00000200:12.0:1616513073.448584:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697587171968, offset 192 00000400:00000200:12.0:1616513073.448590:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.156@tcp1 00010000:00000200:20.0:1616513073.448595:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b100000 x1688697587172096/t0(0) o400->ce5b5a93-c9e4-087b-44ee-78864047f552@10.31.141.156@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513073.448597:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.156@tcp1 routed destination 00000100:00000200:20.0:1616513073.448609:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697587172096, offset 192 00000400:00000200:12.0:1616513073.448612:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.156@tcp1(10.31.141.156@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513073.448615:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.156@tcp1 00000800:00000200:12.0:1616513073.448619:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513073.448623:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.156@tcp1 routed destination 00000800:00000200:12.0:1616513073.448630:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.448633:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.448635:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513073.448638:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.156@tcp1(10.31.141.156@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.448640:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513073.448644:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513073.448645:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.448647:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513073.448650:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.448653:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.448655:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.448659:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.448666:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513073.448698:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.448705:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.448708:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513073.448710:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513073.448715:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.448720:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241100 00000800:00000200:14.0:1616513073.448722:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513073.448724:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513073.448729:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.448732:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.448739:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.448743:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22440 00000800:00000200:14.0:1616513073.448746:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.448749:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.455071:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.455126:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.455135:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.455135:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.455141:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.455145:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.23@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.455148:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.23@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.455154:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.23@tcp1 of length 224 into portal 26 MB=0x5ffdce33e9cc0 00000400:00000200:8.0:1616513073.455155:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.23@tcp1 of length 224 into portal 12 MB=0x5ffdce33e9d40 00000400:00000200:8.0:1616513073.455160:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.23@tcp1 of length 224/224 into md 0x865b9f31 [1] + 65368 00000400:00000200:24.0:1616513073.455161:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.23@tcp1 of length 224/224 into md 0xf97c5219 [1] + 0 00000400:00000200:8.0:1616513073.455164:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513073.455165:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240aa0 00000100:00000200:8.0:1616513073.455166:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513073.455168:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.455171:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513073.455171:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513073.455173:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513073.455174:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.455175:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240aa0 00000800:00000200:8.0:1616513073.455177:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.455178:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.455181:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.455183:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.455235:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699053972800 00010000:00000200:20.0:1616513073.455257:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b107080 x1688699053972800/t0(0) o400->7070d657-4afd-fd2d-8f01-3441333e8ffb@10.31.138.23@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513073.455264:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699053972672 00000100:00000200:20.0:1616513073.455271:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699053972800, offset 192 00010000:00000200:12.0:1616513073.455274:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688699053972672/t0(0) o400->a21e121a-714e-e743-d9c3-c5ee59f27cac@10.31.138.23@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513073.455276:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.23@tcp1 00000400:00000200:20.0:1616513073.455283:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.23@tcp1 routed destination 00000100:00000200:12.0:1616513073.455287:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699053972672, offset 192 00000400:00000200:12.0:1616513073.455294:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.23@tcp1 00000400:00000200:20.0:1616513073.455297:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.23@tcp1(10.31.138.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.455303:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513073.455305:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.23@tcp1 routed destination 00000800:00000200:20.0:1616513073.455312:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.455316:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.455318:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513073.455319:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.23@tcp1(10.31.138.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.455323:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.455325:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513073.455328:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.455331:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.455333:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.455335:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.455340:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513073.455346:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513073.455347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.455366:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.455373:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.455376:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.455383:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513073.455383:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513073.455388:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a90 00000800:00000200:18.0:1616513073.455391:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513073.455391:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.455393:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.455393:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513073.455394:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513073.455400:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.455404:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23d38 00000800:00000200:16.0:1616513073.455408:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.468024:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.468080:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.468089:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.468098:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.174@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.468109:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.174@tcp1 of length 224 into portal 26 MB=0x5ffde559f43c0 00000400:00000200:14.0:1616513073.468115:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.174@tcp1 of length 224/224 into md 0xf97c5229 [1] + 0 00000400:00000200:14.0:1616513073.468120:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782402a8 00000400:00000200:14.0:1616513073.468123:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.468126:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.468130:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782402a8 00000800:00000200:14.0:1616513073.468133:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.468134:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.468136:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.468138:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.468195:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705267876800 00010000:00000200:12.0:1616513073.468206:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688705267876800/t0(0) o400->0d6fc176-9794-bc66-d7d9-e100accbbfad@10.31.136.174@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.468220:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705267876800, offset 192 00000400:00000200:12.0:1616513073.468226:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.174@tcp1 00000400:00000200:12.0:1616513073.468234:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.174@tcp1 routed destination 00000400:00000200:12.0:1616513073.468249:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.174@tcp1(10.31.136.174@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513073.468251:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.468255:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.468261:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.468263:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.468265:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.468270:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513073.468277:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.468277:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513073.468306:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.468315:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.468323:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.174@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.468332:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.174@tcp1 of length 224 into portal 12 MB=0x5ffde559f4540 00000400:00000200:18.0:1616513073.468339:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.174@tcp1 of length 224/224 into md 0xf96450b9 [1] + 33504 00000800:00000200:24.0:1616513073.468343:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513073.468344:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.468347:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513073.468350:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.468352:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513073.468354:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.468357:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:24.0:1616513073.468359:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513073.468359:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.468362:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513073.468365:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241f58 00000800:00000200:24.0:1616513073.468368:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.468371:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513073.468429:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705267877184 00010000:00000200:22.0:1616513073.468455:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d4380 x1688705267877184/t0(0) o400->97f46e42-79e0-5bf3-3f8e-965c096de168@10.31.136.174@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.468470:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705267877184, offset 192 00000400:00000200:22.0:1616513073.468476:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.174@tcp1 00000400:00000200:22.0:1616513073.468484:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.174@tcp1 routed destination 00000400:00000200:22.0:1616513073.468498:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.174@tcp1(10.31.136.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.468505:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.468511:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.468514:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.468516:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.468520:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.468525:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.468535:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.468540:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.468543:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513073.468545:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.468549:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.468553:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b188 00000800:00000200:2.0:1616513073.468557:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.497884:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.497942:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.497952:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.497961:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.215@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.497971:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.215@tcp1 of length 224 into portal 12 MB=0x5ffe46fe7f3c0 00000400:00000200:26.0:1616513073.497977:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.215@tcp1 of length 224/224 into md 0xf96450b9 [1] + 33728 00000400:00000200:26.0:1616513073.497982:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.497985:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.497993:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.497996:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.497996:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.498000:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.498018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.498031:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.498039:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.498048:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.215@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.498056:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.215@tcp1 of length 224 into portal 26 MB=0x5ffe46fe7f0c0 00000100:00000200:22.0:1616513073.498059:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731478651840 00000400:00000200:8.0:1616513073.498062:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.215@tcp1 of length 224/224 into md 0xf97c5249 [1] + 0 00000400:00000200:8.0:1616513073.498067:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240a18 00000400:00000200:8.0:1616513073.498070:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:22.0:1616513073.498072:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695e80 x1688731478651840/t0(0) o400->536322f1-abab-46d9-5e31-68a55cef62d5@10.31.141.215@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513073.498072:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.498076:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240a18 00000800:00000200:8.0:1616513073.498079:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.498082:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513073.498084:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:22.0:1616513073.498086:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688731478651840, offset 192 00000800:00000200:8.0:1616513073.498088:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513073.498092:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.215@tcp1 00000400:00000200:22.0:1616513073.498100:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.215@tcp1 routed destination 00000400:00000200:22.0:1616513073.498115:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.215@tcp1(10.31.141.215@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.498122:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.498128:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.498130:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.498132:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.498138:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513073.498141:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731478651072 00000800:00000200:11.2:1616513073.498143:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:12.0:1616513073.498152:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688731478651072/t0(0) o400->282e6f85-35f1-7de6-6bed-cc537fbd65e3@10.31.141.215@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.498165:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688731478651072, offset 192 00000400:00000200:12.0:1616513073.498171:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.215@tcp1 00000400:00000200:12.0:1616513073.498178:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.215@tcp1 routed destination 00000400:00000200:12.0:1616513073.498192:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.215@tcp1(10.31.141.215@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513073.498197:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.498202:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513073.498203:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.498206:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513073.498207:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.498210:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.498212:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513073.498213:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.498217:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a330 00000800:00000200:12.0:1616513073.498217:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513073.498221:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513073.498221:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513073.498223:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513073.498224:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513073.498278:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.498285:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.498288:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.498290:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513073.498294:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.498300:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:14.0:1616513073.498303:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.510159:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.510217:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.510227:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.510229:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513073.510236:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.121@tcp1 : PUT - for me 00000400:00000200:16.0:1616513073.510245:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.121@tcp1 of length 224 into portal 26 MB=0x6012da8771040 00000400:00000200:16.0:1616513073.510252:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.121@tcp1 of length 224/224 into md 0xf97c5259 [1] + 0 00000400:00000200:16.0:1616513073.510257:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240dd0 00000400:00000200:16.0:1616513073.510259:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513073.510262:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513073.510269:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240dd0 00000800:00000200:16.0:1616513073.510272:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.510275:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.510277:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.510332:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690145471795264 00010000:00000200:12.0:1616513073.510343:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1690145471795264/t0(0) o400->1a9a8190-d2a8-64c1-d4f9-d6857b868920@10.31.140.121@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.510358:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690145471795264, offset 192 00000400:00000200:12.0:1616513073.510364:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.121@tcp1 00000400:00000200:12.0:1616513073.510372:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.121@tcp1 routed destination 00000400:00000200:12.0:1616513073.510387:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.121@tcp1(10.31.140.121@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.510393:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.510399:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.510401:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.510404:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.510409:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.510414:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.510415:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513073.510467:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.510474:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.510477:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513073.510483:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.510489:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240dd0 00000800:00000200:24.0:1616513073.510492:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.510497:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.510925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.510934:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.510941:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.510944:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513073.510950:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.121@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.510964:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.121@tcp1 of length 224 into portal 12 MB=0x6012da8771740 00000400:00000200:8.0:1616513073.510970:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.121@tcp1 of length 224/224 into md 0x865b9f31 [1] + 65592 00000400:00000200:8.0:1616513073.510975:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.510977:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513073.510983:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.510986:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.510988:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.511046:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690145471797056 00010000:00000200:20.0:1616513073.511058:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3a80 x1690145471797056/t0(0) o400->6753570c-c4b2-d8e3-40db-fba70f8bfba1@10.31.140.121@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.511071:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690145471797056, offset 192 00000400:00000200:20.0:1616513073.511077:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.121@tcp1 00000400:00000200:20.0:1616513073.511084:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.121@tcp1 routed destination 00000400:00000200:20.0:1616513073.511098:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.121@tcp1(10.31.140.121@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.511104:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.511110:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.511112:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.511114:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.511119:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.511126:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.511184:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.511192:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.511194:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.511201:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.511205:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22220 00000800:00000200:18.0:1616513073.511209:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.511213:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.515613:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.515670:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.515679:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.515688:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.23@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.515697:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.23@tcp1 of length 224 into portal 26 MB=0x5ffd3c2fa9100 00000400:00000200:24.0:1616513073.515704:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.23@tcp1 of length 224/224 into md 0xf97c5279 [1] + 0 00000400:00000200:24.0:1616513073.515709:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241210 00000400:00000200:24.0:1616513073.515712:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.515714:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513073.515718:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241210 00000800:00000200:24.0:1616513073.515721:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.515723:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.515724:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.515727:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.515780:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659857936640 00010000:00000200:12.0:1616513073.515790:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688659857936640/t0(0) o400->5fdf63cb-68b9-ff94-ca26-aa670b140420@10.31.136.23@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.515803:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688659857936640, offset 192 00000400:00000200:12.0:1616513073.515809:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.23@tcp1 00000400:00000200:12.0:1616513073.515816:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.23@tcp1 routed destination 00000400:00000200:12.0:1616513073.515829:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.23@tcp1(10.31.136.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.515835:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.515840:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.515843:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.515846:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.515851:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513073.515857:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.515858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.515923:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.515930:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.515933:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.515934:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513073.515939:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.515945:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240a18 00000800:00000200:16.0:1616513073.515948:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.516286:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513073.516297:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.516305:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513073.516307:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.516314:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.23@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.516322:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.23@tcp1 of length 224 into portal 12 MB=0x5ffd3c2fa92c0 00000400:00000200:28.0:1616513073.516329:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.23@tcp1 of length 224/224 into md 0x865b9f31 [1] + 65816 00000400:00000200:28.0:1616513073.516334:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.516336:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513073.516343:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.516346:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.516348:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.516409:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659857937088 00010000:00000200:20.0:1616513073.516421:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf83a80 x1688659857937088/t0(0) o400->c5a503fe-eb71-4140-05a4-3987026c82d9@10.31.136.23@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.516449:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688659857937088, offset 192 00000400:00000200:20.0:1616513073.516455:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.23@tcp1 00000400:00000200:20.0:1616513073.516463:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.23@tcp1 routed destination 00000400:00000200:20.0:1616513073.516477:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.23@tcp1(10.31.136.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.516484:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.516489:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.516492:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.516494:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.516499:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.516504:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.516557:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.516564:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.516567:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.516574:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.516578:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23b18 00000800:00000200:2.0:1616513073.516582:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.516585:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.541825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.541838:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.541845:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.541853:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.56@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.541860:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.56@tcp1 of length 224 into portal 26 MB=0x5ffdcccdd0340 00000400:00000200:8.0:1616513073.541866:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.56@tcp1 of length 224/224 into md 0xf97c5289 [1] + 0 00000400:00000200:8.0:1616513073.541870:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240ee0 00000400:00000200:8.0:1616513073.541872:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.541875:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.541879:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ee0 00000800:00000200:8.0:1616513073.541881:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.541884:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.541885:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.541887:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.541892:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.541893:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513073.541897:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.56@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.541902:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.56@tcp1 of length 224 into portal 12 MB=0x5ffdcccdd0400 00000400:00000200:8.0:1616513073.541906:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.56@tcp1 of length 224/224 into md 0x865b9f31 [1] + 66040 00000400:00000200:8.0:1616513073.541909:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.541911:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513073.541916:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.541917:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.541919:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.541939:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698678477632 00010000:00000200:12.0:1616513073.541949:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688698678477632/t0(0) o400->b92a324d-08ef-d084-0103-19682218c73e@10.31.141.56@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.541962:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698678477632, offset 192 00000400:00000200:12.0:1616513073.541968:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.56@tcp1 00000400:00000200:12.0:1616513073.541974:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.56@tcp1 routed destination 00000100:00000200:20.0:1616513073.541982:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698678477824 00000400:00000200:12.0:1616513073.541988:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.56@tcp1(10.31.141.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:20.0:1616513073.541994:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d2400 x1688698678477824/t0(0) o400->a14b4588-1f05-bb6d-8f2e-1ddd8ca801e1@10.31.141.56@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513073.541994:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.541999:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.542002:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.542004:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.542008:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513073.542014:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.542014:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513073.542019:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698678477824, offset 192 00000400:00000200:20.0:1616513073.542026:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.56@tcp1 00000400:00000200:20.0:1616513073.542033:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.56@tcp1 routed destination 00000400:00000200:20.0:1616513073.542048:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.56@tcp1(10.31.141.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.542054:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.542060:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.542063:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.542065:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513073.542068:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513073.542070:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513073.542075:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513073.542077:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513073.542078:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513073.542079:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.542085:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.542090:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782402a8 00000800:00000200:28.0:1616513073.542094:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513073.542133:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.542141:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.542144:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.542150:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.542154:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23210 00000800:00000200:18.0:1616513073.542159:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.542161:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.542167:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513073.542170:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.542172:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513073.542173:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.542176:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513073.546515:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924144, portal 4 00000100:00000200:2.0:1616513073.546521:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924144, offset 0 00000400:00000200:2.0:1616513073.546527:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:2.0:1616513073.546533:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:2.0:1616513073.546546:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513073.546553:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:2.0:1616513073.546559:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513073.546563:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:2.0:1616513073.546565:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:2.0:1616513073.546570:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513073.546619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.546637:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513073.546644:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513073.546647:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513073.546654:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513073.546662:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a7b600 x1693067725924144/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513073.546675:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed6d8 00000800:00000200:1.0:1616513073.546681:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513073.546683:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513073.546797:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513073.546806:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.546811:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:25.0:1616513073.546814:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513073.546820:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513073.546828:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626730 00000400:00000200:1.0:1616513073.546835:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff965 [1] + 192 00000400:00000200:1.0:1616513073.546841:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513073.546846:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a7b600 x1693067725924144/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513073.546861:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.546864:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513073.546866:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:2.0:1616513073.546870:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a7b600 x1693067725924144/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513073.546886:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a7b600 x1693067725924144/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513073.546897:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed760 00000800:00000200:11.2:1616513073.551254:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.551265:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.551271:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.551280:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.208@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.551288:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.208@tcp1 of length 224 into portal 26 MB=0x60182c2b00d00 00000400:00000200:8.0:1616513073.551294:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.208@tcp1 of length 224/224 into md 0xf97c5299 [1] + 0 00000400:00000200:8.0:1616513073.551298:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241320 00000400:00000200:8.0:1616513073.551300:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.551302:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.551306:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241320 00000800:00000200:8.0:1616513073.551308:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.551310:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.551311:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.551314:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.551369:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510983957760 00010000:00000200:12.0:1616513073.551380:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1690510983957760/t0(0) o400->82e2c496-b6dc-8f3a-8351-f8e7773f8735@10.31.140.208@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.551394:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510983957760, offset 192 00000400:00000200:12.0:1616513073.551400:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.208@tcp1 00000400:00000200:12.0:1616513073.551408:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.208@tcp1 routed destination 00000400:00000200:12.0:1616513073.551423:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.208@tcp1(10.31.140.208@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.551429:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.551453:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.551456:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.551458:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.551463:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.551468:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.551471:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513073.551522:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.551529:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.551532:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513073.551538:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.551544:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240aa0 00000800:00000200:16.0:1616513073.551547:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.551550:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.551844:0:48761:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.551900:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.551910:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.551918:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.208@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.551927:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.208@tcp1 of length 224 into portal 12 MB=0x60182c2b01140 00000400:00000200:14.0:1616513073.551934:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.208@tcp1 of length 224/224 into md 0xf96450b9 [1] + 33952 00000400:00000200:14.0:1616513073.551940:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.551942:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.551950:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.551953:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.551955:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.551958:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.552017:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510983958848 00010000:00000200:22.0:1616513073.552029:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cbf00 x1690510983958848/t0(0) o400->e810b75a-4c89-d8b6-2238-41f0a54b4c40@10.31.140.208@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.552044:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510983958848, offset 192 00000400:00000200:22.0:1616513073.552052:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.208@tcp1 00000400:00000200:22.0:1616513073.552059:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.208@tcp1 routed destination 00000400:00000200:22.0:1616513073.552074:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.208@tcp1(10.31.140.208@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.552081:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.552087:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.552089:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.552091:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.552096:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.552100:0:48761:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.552156:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.552163:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.552166:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.552168:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.552174:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.552178:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b540 00000800:00000200:26.0:1616513073.552182:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.552735:0:48761:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.552752:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.552762:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513073.552763:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513073.552780:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.122@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.552790:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.122@tcp1 of length 224 into portal 26 MB=0x5ffd4c576ed00 00000400:00000200:18.0:1616513073.552796:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.122@tcp1 of length 224/224 into md 0xf97c52b9 [1] + 0 00000400:00000200:18.0:1616513073.552800:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240220 00000400:00000200:18.0:1616513073.552803:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.552805:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513073.552809:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240220 00000800:00000200:18.0:1616513073.552812:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.552815:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.552816:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.552873:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664194608384 00010000:00000200:12.0:1616513073.552884:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688664194608384/t0(0) o400->53b212cb-be48-5c47-ca4e-ce74e1fa93d2@10.31.132.122@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.552900:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664194608384, offset 192 00000400:00000200:12.0:1616513073.552906:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.122@tcp1 00000400:00000200:12.0:1616513073.552914:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.122@tcp1 routed destination 00000400:00000200:12.0:1616513073.552929:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.122@tcp1(10.31.132.122@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.552936:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.552941:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.552944:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.552946:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.552950:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513073.552957:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.552992:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.553001:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.553004:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.553006:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.553011:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.553014:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241cb0 00000800:00000200:8.0:1616513073.553016:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.553018:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.553515:0:48761:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.553569:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.553578:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.553591:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.122@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.553600:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.122@tcp1 of length 224 into portal 12 MB=0x5ffd4c576ee40 00000400:00000200:2.0:1616513073.553607:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.122@tcp1 of length 224/224 into md 0xf96450b9 [1] + 34176 00000400:00000200:2.0:1616513073.553612:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.553614:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513073.553621:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.553624:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.553626:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.553628:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.553683:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664194608704 00010000:00000200:22.0:1616513073.553695:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138e300 x1688664194608704/t0(0) o400->8225a013-3bb4-9062-3c44-b802c5701c41@10.31.132.122@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.553710:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664194608704, offset 192 00000400:00000200:22.0:1616513073.553716:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.122@tcp1 00000400:00000200:22.0:1616513073.553724:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.122@tcp1 routed destination 00000400:00000200:22.0:1616513073.553739:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.122@tcp1(10.31.132.122@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.553745:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.553751:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.553754:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.553756:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.553761:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.553766:0:48761:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.553776:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.553780:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.553782:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513073.553784:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513073.553788:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.553791:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a198 00000800:00000200:14.0:1616513073.553794:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.559358:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.559412:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.559421:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.559429:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.50@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.559456:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.50@tcp1 of length 224 into portal 26 MB=0x5ffd470604b80 00000400:00000200:2.0:1616513073.559462:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.50@tcp1 of length 224/224 into md 0xf97c52c9 [1] + 0 00000800:00000200:26.0:1616513073.559466:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.559466:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782406e8 00000400:00000200:2.0:1616513073.559468:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.559471:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.559475:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782406e8 00000800:00000200:2.0:1616513073.559478:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.559481:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.559482:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.559488:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513073.559500:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.559509:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513073.559517:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.50@tcp1 : PUT - for me 00000400:00000200:18.0:1616513073.559525:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.50@tcp1 of length 224 into portal 12 MB=0x5ffd470604d80 00000400:00000200:18.0:1616513073.559531:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.50@tcp1 of length 224/224 into md 0xf96450b9 [1] + 34400 00000400:00000200:18.0:1616513073.559536:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.559538:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:12.0:1616513073.559539:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662767061888 00000800:00000200:18.0:1616513073.559546:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.559549:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00010000:00000200:12.0:1616513073.559549:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688662767061888/t0(0) o400->64c2ac63-e432-4285-062a-0252b9f1e24d@10.31.136.50@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513073.559551:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.559554:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.559562:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662767061888, offset 192 00000400:00000200:12.0:1616513073.559568:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.50@tcp1 00000400:00000200:12.0:1616513073.559575:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.50@tcp1 routed destination 00000400:00000200:12.0:1616513073.559588:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.50@tcp1(10.31.136.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.559595:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.559600:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.559602:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.559604:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513073.559609:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662767062400 00000800:00000200:12.0:1616513073.559609:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513073.559615:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.559617:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513073.559621:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d6c00 x1688662767062400/t0(0) o400->1f89d7ca-bb20-c3d1-ebba-3f5a93a07007@10.31.136.50@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.559635:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662767062400, offset 192 00000400:00000200:22.0:1616513073.559642:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.50@tcp1 00000400:00000200:22.0:1616513073.559650:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.50@tcp1 routed destination 00000400:00000200:22.0:1616513073.559664:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.50@tcp1(10.31.136.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.559671:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513073.559674:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513073.559677:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.559680:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513073.559681:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.559683:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513073.559683:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513073.559688:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513073.559689:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513073.559693:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513073.559695:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782406e8 00000800:00000200:16.0:1616513073.559699:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.559702:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513073.559703:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.559709:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.559711:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.559712:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.559718:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.559722:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b7e8 00000800:00000200:26.0:1616513073.559726:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.580824:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.580880:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.580890:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.580899:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.146@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.580909:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.146@tcp1 of length 224 into portal 26 MB=0x5ffd713c7a280 00000400:00000200:14.0:1616513073.580916:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.146@tcp1 of length 224/224 into md 0xf97c52d9 [1] + 0 00000400:00000200:14.0:1616513073.580920:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241e48 00000400:00000200:14.0:1616513073.580923:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.580926:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.580930:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241e48 00000800:00000200:18.0:1616513073.580934:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.580935:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.580938:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.580940:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.580993:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674098455168 00010000:00000200:12.0:1616513073.581003:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688674098455168/t0(0) o400->57694c40-5c60-0b87-bd14-8ddff6a20c1e@10.31.131.146@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.581016:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688674098455168, offset 192 00000400:00000200:12.0:1616513073.581022:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.146@tcp1 00000400:00000200:12.0:1616513073.581028:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.146@tcp1 routed destination 00000400:00000200:12.0:1616513073.581044:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.146@tcp1(10.31.131.146@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.581050:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.581055:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.581058:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.581060:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.581065:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513073.581078:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.581083:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.581101:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.581106:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.581108:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.581114:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.581118:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240220 00000800:00000200:8.0:1616513073.581121:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.581124:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.581124:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513073.581126:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.581154:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.581165:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513073.581169:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.581171:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.581173:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.581175:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513073.581193:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.581202:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.581211:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.146@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.581218:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.146@tcp1 of length 224 into portal 12 MB=0x5ffd713c7a740 00000400:00000200:2.0:1616513073.581227:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.146@tcp1 of length 224/224 into md 0xf96450b9 [1] + 34624 00000400:00000200:2.0:1616513073.581231:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.581234:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513073.581240:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.581243:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.581245:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.581248:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.581306:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674098456384 00010000:00000200:22.0:1616513073.581318:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138e780 x1688674098456384/t0(0) o400->ee69d52d-cf2e-fc5c-f932-f392dc584489@10.31.131.146@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.581333:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688674098456384, offset 192 00000400:00000200:22.0:1616513073.581339:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.146@tcp1 00000400:00000200:22.0:1616513073.581346:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.146@tcp1 routed destination 00000400:00000200:22.0:1616513073.581360:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.146@tcp1(10.31.131.146@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.581367:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.581373:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.581376:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.581378:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.581383:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.581388:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.581398:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.581403:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.581406:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513073.581408:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513073.581412:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.581416:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b3a8 00000800:00000200:18.0:1616513073.581419:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513073.594627:0:6480:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924160, portal 4 00000100:00000200:24.0:1616513073.594633:0:6480:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924160, offset 0 00000400:00000200:24.0:1616513073.594639:0:6480:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:24.0:1616513073.594645:0:6480:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:24.0:1616513073.594658:0:6480:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513073.594665:0:6480:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:24.0:1616513073.594671:0:6480:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513073.594674:0:6480:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:24.0:1616513073.594676:0:6480:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:24.0:1616513073.594682:0:6480:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513073.594730:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513073.594786:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513073.594793:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513073.594796:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.594802:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513073.594808:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc7e00480 x1693067725924160/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513073.594821:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:26.0:1616513073.594824:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513073.594827:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513073.594907:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:11.2:1616513073.594972:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513073.594973:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513073.594982:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513073.594991:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:8.0:1616513073.594994:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513073.594999:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626740 00000800:00000200:8.0:1616513073.595001:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513073.595004:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513073.595005:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c5969 [1] + 192 00000400:00000200:8.0:1616513073.595009:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.38@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.595010:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513073.595011:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.595015:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:14.0:1616513073.595015:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc7e00480 x1693067725924160/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513073.595017:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.38@tcp1 of length 224 into portal 26 MB=0x5ffd52791b880 00000400:00000200:24.0:1616513073.595020:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.38@tcp1 : PUT - for me 00000800:00000200:18.0:1616513073.595028:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513073.595031:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000400:00000200:8.0:1616513073.595032:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.38@tcp1 of length 224/224 into md 0xf97c52e9 [1] + 0 00000400:00000200:24.0:1616513073.595033:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.38@tcp1 of length 224 into portal 12 MB=0x5ffd52791b900 00000800:00000200:14.0:1616513073.595033:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513073.595035:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000400:00000200:8.0:1616513073.595036:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241c28 00000400:00000200:8.0:1616513073.595038:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513073.595039:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.38@tcp1 of length 224/224 into md 0x865b9f31 [1] + 66264 00000100:00000200:8.0:1616513073.595041:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513073.595044:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.595045:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241c28 00000100:00000200:24.0:1616513073.595046:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513073.595048:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.595051:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.595052:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.595053:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.595056:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.595057:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513073.595066:0:6480:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc7e00480 x1693067725924160/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513073.595073:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665840531712 00000100:00000200:24.0:1616513073.595078:0:6480:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc7e00480 x1693067725924160/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00010000:00000200:20.0:1616513073.595085:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc0900 x1688665840531712/t0(0) o400->bf7372e2-8c3e-3888-a400-12e651929b40@10.31.136.38@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513073.595088:0:6480:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000100:00000200:20.0:1616513073.595099:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665840531712, offset 192 00000400:00000200:20.0:1616513073.595106:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.38@tcp1 00000100:00000200:12.0:1616513073.595108:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665840531584 00000400:00000200:20.0:1616513073.595113:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.38@tcp1 routed destination 00010000:00000200:12.0:1616513073.595119:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688665840531584/t0(0) o400->09c29f0e-6cff-0dd0-0182-a2428a88023c@10.31.136.38@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513073.595128:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.38@tcp1(10.31.136.38@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:12.0:1616513073.595133:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665840531584, offset 192 00000800:00000200:20.0:1616513073.595135:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513073.595139:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.38@tcp1 00000800:00000200:20.0:1616513073.595141:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.595143:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.595146:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513073.595147:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.38@tcp1 routed destination 00000800:00000200:20.0:1616513073.595150:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.595155:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513073.595162:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.38@tcp1(10.31.136.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513073.595167:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.595168:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513073.595174:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513073.595174:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513073.595177:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513073.595177:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.595178:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.595179:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513073.595183:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:12.0:1616513073.595184:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513073.595187:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23540 00000100:00000200:12.0:1616513073.595190:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513073.595190:0:48765:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513073.595191:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513073.595202:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.595206:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.595208:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.595211:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.595213:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.595217:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241320 00000800:00000200:26.0:1616513073.595220:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.596955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.596966:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.596973:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.596981:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.161@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.596988:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.161@tcp1 of length 224 into portal 26 MB=0x60216e008c340 00000400:00000200:8.0:1616513073.596994:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.161@tcp1 of length 224/224 into md 0xf97c52f9 [1] + 0 00000400:00000200:8.0:1616513073.596997:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782417e8 00000400:00000200:8.0:1616513073.597000:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.597002:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.597005:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782417e8 00000800:00000200:8.0:1616513073.597008:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.597010:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.597012:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.597013:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.597018:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513073.597019:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513073.597024:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.161@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.597029:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.161@tcp1 of length 224 into portal 12 MB=0x60216e008c400 00000400:00000200:8.0:1616513073.597034:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.161@tcp1 of length 224/224 into md 0x865b9f31 [1] + 66488 00000400:00000200:8.0:1616513073.597037:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.597039:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513073.597043:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.597045:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.597046:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.597068:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691147131470656 00010000:00000200:12.0:1616513073.597079:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1691147131470656/t0(0) o400->52019eb0-93a9-dff7-58cb-50fa97bdac0b@10.31.140.161@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.597093:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691147131470656, offset 192 00000400:00000200:12.0:1616513073.597099:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.161@tcp1 00000400:00000200:12.0:1616513073.597107:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.161@tcp1 routed destination 00000100:00000200:20.0:1616513073.597116:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691147131470848 00000400:00000200:12.0:1616513073.597121:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.161@tcp1(10.31.140.161@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:20.0:1616513073.597128:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4380 x1691147131470848/t0(0) o400->7890681e-8c96-740d-a5de-919fbcdc2cb9@10.31.140.161@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513073.597128:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.597134:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.597138:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.597140:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513073.597143:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691147131470848, offset 192 00000800:00000200:12.0:1616513073.597145:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513073.597149:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.161@tcp1 00000800:00000200:11.2:1616513073.597150:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.597152:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513073.597156:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.161@tcp1 routed destination 00000800:00000200:28.0:1616513073.597162:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.597170:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513073.597171:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.161@tcp1(10.31.140.161@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513073.597172:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513073.597174:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.597179:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513073.597179:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513073.597185:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ee0 00000800:00000200:20.0:1616513073.597185:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513073.597188:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:20.0:1616513073.597188:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.597190:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.597195:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.597201:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.597256:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.597263:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513073.597266:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513073.597272:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513073.597276:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23ba0 00000800:00000200:18.0:1616513073.597280:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513073.597283:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513073.610598:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924176, portal 4 00000100:00000200:20.0:1616513073.610603:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924176, offset 0 00000400:00000200:20.0:1616513073.610610:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:20.0:1616513073.610616:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:20.0:1616513073.610629:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.610636:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:20.0:1616513073.610642:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.610646:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:20.0:1616513073.610648:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:20.0:1616513073.610653:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513073.610719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513073.610741:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513073.610747:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513073.610750:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513073.610757:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513073.610764:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affd2400 x1693067725924176/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513073.610776:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee227f8 00000800:00000200:27.0:1616513073.610780:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513073.610783:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513073.610873:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513073.610882:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513073.610888:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.231@o2ib1 00000800:00000200:11.0:1616513073.610891:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513073.610896:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513073.610904:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626750 00000400:00000200:27.0:1616513073.610911:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371655fd [1] + 192 00000400:00000200:27.0:1616513073.610916:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513073.610920:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affd2400 x1693067725924176/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513073.610935:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513073.610937:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513073.610939:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:20.0:1616513073.610945:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affd2400 x1693067725924176/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513073.610958:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affd2400 x1693067725924176/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513073.610968:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee238f8 00000100:00000200:5.0:1616513073.629354:0:6491:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924192, portal 4 00000100:00000200:5.0:1616513073.629359:0:6491:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924192, offset 0 00000400:00000200:5.0:1616513073.629366:0:6491:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:5.0:1616513073.629372:0:6491:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:5.0:1616513073.629385:0:6491:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:5.0:1616513073.629392:0:6491:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513073.629398:0:6491:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513073.629401:0:6491:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:5.0:1616513073.629403:0:6491:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513073.629408:0:6491:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513073.629454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.629472:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513073.629478:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513073.629480:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513073.629487:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513073.629494:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1393180 x1693067725924192/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513073.629506:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55ef68 00000800:00000200:1.0:1616513073.629510:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513073.629513:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513073.629627:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513073.629636:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.629643:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513073.629645:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513073.629650:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513073.629659:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626760 00000400:00000200:1.0:1616513073.629665:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff975 [1] + 192 00000400:00000200:1.0:1616513073.629670:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513073.629674:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1393180 x1693067725924192/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513073.629688:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.629691:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513073.629692:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:5.0:1616513073.629694:0:6491:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1393180 x1693067725924192/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:5.0:1616513073.629707:0:6491:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1393180 x1693067725924192/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513073.629717:0:6491:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55e7f8 00000800:00000200:7.2:1616513073.630647:0:48769:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.630704:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.630714:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.630723:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.197@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.630734:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.197@tcp1 of length 224 into portal 26 MB=0x604a4872136c0 00000400:00000200:2.0:1616513073.630740:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.197@tcp1 of length 224/224 into md 0xf97c5319 [1] + 0 00000400:00000200:2.0:1616513073.630745:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240550 00000400:00000200:2.0:1616513073.630748:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.630751:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.630755:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240550 00000800:00000200:26.0:1616513073.630758:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.630759:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.630762:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.630763:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.630780:0:48769:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:12.0:1616513073.630824:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954548512448 00010000:00000200:12.0:1616513073.630835:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1693954548512448/t0(0) o400->e1ef48ff-3b22-544a-9924-432904be2d55@10.31.140.197@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513073.630836:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513073.630846:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:12.0:1616513073.630849:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954548512448, offset 192 00000400:00000200:18.0:1616513073.630854:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.197@tcp1 : PUT - for me 00000400:00000200:12.0:1616513073.630856:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.197@tcp1 00000400:00000200:18.0:1616513073.630864:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.197@tcp1 of length 224 into portal 12 MB=0x604a487213800 00000400:00000200:12.0:1616513073.630864:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.197@tcp1 routed destination 00000400:00000200:18.0:1616513073.630870:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.197@tcp1 of length 224/224 into md 0xf96450b9 [1] + 34848 00000400:00000200:18.0:1616513073.630876:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513073.630878:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513073.630879:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.197@tcp1(10.31.140.197@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.630886:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513073.630887:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513073.630890:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.630890:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513073.630892:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513073.630894:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513073.630894:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.630897:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.630901:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513073.630908:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.630908:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.630919:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.630924:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.630926:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.630932:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.630937:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241210 00000800:00000200:8.0:1616513073.630940:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.630942:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513073.630951:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954548512768 00010000:00000200:22.0:1616513073.630962:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d1680 x1693954548512768/t0(0) o400->18969db2-c6cb-bc68-11a4-bd047e7f51e1@10.31.140.197@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.630976:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954548512768, offset 192 00000400:00000200:22.0:1616513073.630982:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.197@tcp1 00000400:00000200:22.0:1616513073.630989:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.197@tcp1 routed destination 00000400:00000200:22.0:1616513073.631002:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.197@tcp1(10.31.140.197@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.631008:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.631014:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.631017:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.631019:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.631023:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.631027:0:48769:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.631038:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.631043:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.631046:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.631047:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.631053:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.631057:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ab28 00000800:00000200:26.0:1616513073.631061:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:11.0:1616513073.642522:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924208, portal 4 00000100:00000200:11.0:1616513073.642526:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924208, offset 0 00000400:00000200:11.0:1616513073.642532:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:11.0:1616513073.642537:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:11.0:1616513073.642547:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513073.642552:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:11.0:1616513073.642557:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513073.642560:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:11.0:1616513073.642562:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:11.0:1616513073.642566:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513073.642612:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.642631:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513073.642636:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513073.642638:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513073.642645:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513073.642652:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be41f80 x1693067725924208/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513073.642664:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a6e8 00000800:00000200:1.0:1616513073.642668:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513073.642670:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513073.642781:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513073.642790:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.642796:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513073.642804:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513073.642812:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626770 00000400:00000200:1.0:1616513073.642818:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff995 [1] + 192 00000400:00000200:1.0:1616513073.642823:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513073.642827:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be41f80 x1693067725924208/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513073.642841:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513073.642841:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513073.642843:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000100:00000200:11.0:1616513073.642845:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be41f80 x1693067725924208/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513073.642845:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:11.0:1616513073.642855:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be41f80 x1693067725924208/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513073.642863:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a2a8 00000800:00000200:11.2:1616513073.660333:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.660390:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.660399:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.660419:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.130@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.660428:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.130@tcp1 of length 224 into portal 26 MB=0x5ffd8a0e14ec0 00000400:00000200:24.0:1616513073.660449:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.130@tcp1 of length 224/224 into md 0xf97c5339 [1] + 0 00000400:00000200:24.0:1616513073.660454:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240770 00000400:00000200:24.0:1616513073.660457:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:28.0:1616513073.660459:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513073.660459:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513073.660464:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240770 00000800:00000200:24.0:1616513073.660467:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513073.660469:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513073.660470:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.660471:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.660474:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.660481:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.130@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.660490:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.130@tcp1 of length 224 into portal 12 MB=0x5ffd8a0e14f40 00000400:00000200:28.0:1616513073.660496:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.130@tcp1 of length 224/224 into md 0x865b9f31 [1] + 66712 00000400:00000200:28.0:1616513073.660502:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.660504:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513073.660511:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.660515:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.660517:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.660529:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688680760692416 00010000:00000200:12.0:1616513073.660540:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688680760692416/t0(0) o400->d50e971a-8720-8290-8b01-b43edd2f62c5@10.31.143.130@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.660554:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688680760692416, offset 192 00000400:00000200:12.0:1616513073.660561:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.130@tcp1 00000400:00000200:12.0:1616513073.660568:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.130@tcp1 routed destination 00000100:00000200:20.0:1616513073.660577:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688680760692544 00000400:00000200:12.0:1616513073.660583:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.130@tcp1(10.31.143.130@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:20.0:1616513073.660590:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf83f00 x1688680760692544/t0(0) o400->ca744503-5b8e-c18d-7b79-2984c42e5fbb@10.31.143.130@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513073.660590:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.660596:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.660599:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.660601:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513073.660604:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688680760692544, offset 192 00000800:00000200:12.0:1616513073.660606:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513073.660610:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.130@tcp1 00000800:00000200:11.2:1616513073.660610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.660612:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513073.660618:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.130@tcp1 routed destination 00000800:00000200:16.0:1616513073.660623:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513073.660630:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.660632:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513073.660633:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.130@tcp1(10.31.143.130@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513073.660633:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513073.660639:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513073.660640:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513073.660645:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513073.660645:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240770 00000800:00000200:16.0:1616513073.660649:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:20.0:1616513073.660650:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.660652:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.660657:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.660663:0:48772:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.660725:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.660732:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.660735:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.660742:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.660745:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee238f8 00000800:00000200:14.0:1616513073.660749:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.660752:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.664122:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513073.664148:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513073.664176:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.664186:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.664195:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.60@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.664205:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.60@tcp1 of length 224 into portal 26 MB=0x6037bcd0b2ec0 00000800:00000200:24.0:1616513073.664206:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513073.664211:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.60@tcp1 of length 224/224 into md 0xf97c5349 [1] + 0 00000800:00000200:24.0:1616513073.664215:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513073.664216:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774aa0 00000400:00000200:2.0:1616513073.664219:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.664222:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513073.664224:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.60@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.664226:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774aa0 00000800:00000200:2.0:1616513073.664229:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.664232:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.664233:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:24.0:1616513073.664234:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.60@tcp1 of length 224 into portal 12 MB=0x6037bcd0b2f40 00000800:00000200:2.0:1616513073.664237:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513073.664240:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.60@tcp1 of length 224/224 into md 0x865b9f31 [1] + 66936 00000400:00000200:24.0:1616513073.664245:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.664250:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513073.664256:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.664259:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.664261:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.664264:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.664293:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692680116186816 00010000:00000200:12.0:1616513073.664304:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1692680116186816/t0(0) o400->2d6e1b29-637e-3f91-eb96-0bc3891d3725@10.31.140.60@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.664318:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692680116186816, offset 192 00000100:00000200:20.0:1616513073.664323:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692680116186944 00000400:00000200:12.0:1616513073.664325:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.60@tcp1 00000400:00000200:12.0:1616513073.664332:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.60@tcp1 routed destination 00010000:00000200:20.0:1616513073.664335:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc5a00 x1692680116186944/t0(0) o400->84bae4ee-0338-1641-9a20-ee4cf017e286@10.31.140.60@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513073.664347:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.60@tcp1(10.31.140.60@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:20.0:1616513073.664350:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692680116186944, offset 192 00000800:00000200:12.0:1616513073.664353:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513073.664356:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.60@tcp1 00000800:00000200:12.0:1616513073.664359:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.664362:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:20.0:1616513073.664364:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.60@tcp1 routed destination 00000800:00000200:12.0:1616513073.664364:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.664369:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.664373:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.664375:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513073.664379:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.60@tcp1(10.31.140.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513073.664383:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513073.664386:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513073.664388:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513073.664391:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:8.0:1616513073.664391:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513073.664394:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.664394:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513073.664396:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513073.664397:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513073.664401:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513073.664402:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240550 00000800:00000200:8.0:1616513073.664405:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.664406:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.664459:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.664466:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.664469:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.664475:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.664479:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23a08 00000800:00000200:14.0:1616513073.664483:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.664486:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.674576:0:6482:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924224, portal 4 00000100:00000200:16.0:1616513073.674581:0:6482:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924224, offset 0 00000400:00000200:16.0:1616513073.674588:0:6482:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:16.0:1616513073.674593:0:6482:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000100:00000200:28.0:1616513073.674600:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924240, portal 4 00000100:00000200:28.0:1616513073.674606:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924240, offset 0 00000400:00000200:16.0:1616513073.674606:0:6482:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513073.674612:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:16.0:1616513073.674613:0:6482:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:28.0:1616513073.674618:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:16.0:1616513073.674626:0:6482:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.674629:0:6482:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000400:00000200:28.0:1616513073.674631:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.674631:0:6482:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:16.0:1616513073.674636:0:6482:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:28.0:1616513073.674638:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:28.0:1616513073.674644:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513073.674647:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:28.0:1616513073.674650:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:28.0:1616513073.674654:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.2:1616513073.674682:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513073.674701:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:27.0:1616513073.674712:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:27.0:1616513073.674714:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513073.674721:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513073.674727:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc7e00900 x1693067725924224/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:11.2:1616513073.674729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:27.0:1616513073.674739:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a426e8 00000800:00000200:27.0:1616513073.674744:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513073.674746:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513073.674750:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513073.674751:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.0:1616513073.674753:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513073.674756:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513073.674761:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdc57f500 x1693067725924240/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513073.674771:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd210 00000800:00000200:27.0:1616513073.674773:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:24.0:1616513073.674784:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.674793:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513073.674795:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513073.674803:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.64@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.674811:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.64@tcp1 of length 224 into portal 26 MB=0x60218611c79c0 00000400:00000200:24.0:1616513073.674818:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.64@tcp1 of length 224/224 into md 0xf97c5359 [1] + 0 00000400:00000200:24.0:1616513073.674822:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77747f8 00000400:00000200:24.0:1616513073.674825:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.674828:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513073.674832:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77747f8 00000800:00000200:24.0:1616513073.674836:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.674838:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.674840:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.674851:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691153591990720 00010000:00000200:12.0:1616513073.674861:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1691153591990720/t0(0) o400->32d9e64d-a6e7-b37f-fa4d-80df1e80fa71@10.31.140.64@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:27.2:1616513073.674866:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513073.674874:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:12.0:1616513073.674875:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691153591990720, offset 192 00000800:00000200:27.0:1616513073.674880:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000400:00000200:12.0:1616513073.674882:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.64@tcp1 00000800:00000200:11.0:1616513073.674882:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513073.674888:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:11.0:1616513073.674889:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:12.0:1616513073.674890:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.64@tcp1 routed destination 00000800:00000200:31.0:1616513073.674891:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513073.674896:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626780 00000400:00000200:11.0:1616513073.674898:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513073.674903:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716561d [1] + 192 00000400:00000200:12.0:1616513073.674906:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.64@tcp1(10.31.140.64@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:11.0:1616513073.674907:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626790 00000800:00000200:7.2:1616513073.674907:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:27.0:1616513073.674908:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513073.674912:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc7e00900 x1693067725924224/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513073.674913:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.674919:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.674922:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.674924:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:27.0:1616513073.674925:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:11.0:1616513073.674927:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716562d [1] + 192 00000800:00000200:27.0:1616513073.674928:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:12.0:1616513073.674928:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:27.0:1616513073.674930:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:11.2:1616513073.674933:0:6475:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.674935:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:11.0:1616513073.674940:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513073.674945:0:6482:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc7e00900 x1693067725924224/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513073.674953:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:16.0:1616513073.674958:0:6482:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc7e00900 x1693067725924224/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513073.674958:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.674960:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.674961:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513073.674963:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513073.674965:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513073.674968:0:6482:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a436d8 00000400:00000200:12.0:1616513073.674969:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782417e8 00000800:00000200:26.0:1616513073.674974:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.674974:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:11.0:1616513073.674974:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdc57f500 x1693067725924240/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513073.674983:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.64@tcp1 : PUT - for me 00000800:00000200:11.0:1616513073.674989:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:26.0:1616513073.674992:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.64@tcp1 of length 224 into portal 12 MB=0x60218611c7a40 00000800:00000200:11.0:1616513073.674992:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:11.0:1616513073.674994:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:28.0:1616513073.674997:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdc57f500 x1693067725924240/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513073.674998:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.64@tcp1 of length 224/224 into md 0xf96450b9 [1] + 35072 00000400:00000200:26.0:1616513073.675003:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.675006:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:28.0:1616513073.675010:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdc57f500 x1693067725924240/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513073.675013:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.675016:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.675018:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513073.675020:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd8f8 00000800:00000200:26.0:1616513073.675022:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.675079:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691153591990848 00010000:00000200:22.0:1616513073.675091:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c694c80 x1691153591990848/t0(0) o400->8b9210f6-25f2-1dc1-9ed1-72a40c7b8c75@10.31.140.64@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.675117:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691153591990848, offset 192 00000400:00000200:22.0:1616513073.675123:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.64@tcp1 00000400:00000200:22.0:1616513073.675130:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.64@tcp1 routed destination 00000400:00000200:22.0:1616513073.675145:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.64@tcp1(10.31.140.64@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.675152:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.675158:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.675161:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.675163:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.675168:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.675173:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.675227:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.675234:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.675237:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513073.675238:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513073.675243:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.675249:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a088 00000800:00000200:14.0:1616513073.675253:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513073.690606:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924256, portal 4 00000100:00000200:2.0:1616513073.690611:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924256, offset 0 00000400:00000200:2.0:1616513073.690618:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:2.0:1616513073.690623:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:2.0:1616513073.690636:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513073.690643:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:2.0:1616513073.690649:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513073.690652:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513073.690654:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513073.690659:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513073.690705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513073.690720:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513073.690725:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513073.690727:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513073.690730:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513073.690732:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513073.690737:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d2400 x1693067725924256/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513073.690749:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eeccc0 00000800:00000200:2.0:1616513073.690752:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513073.690882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513073.690935:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513073.690945:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:14.0:1616513073.690950:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513073.690954:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513073.690961:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6267a0 00000400:00000200:22.0:1616513073.690968:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c59f9 [1] + 192 00000400:00000200:22.0:1616513073.690973:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513073.690978:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d2400 x1693067725924256/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513073.690993:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513073.690997:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513073.690999:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:2.0:1616513073.690999:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d2400 x1693067725924256/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513073.691010:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d2400 x1693067725924256/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513073.691019:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed760 00000800:00000200:11.2:1616513073.692317:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513073.692372:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.692381:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513073.692390:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.9@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.692399:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.9@tcp1 of length 224 into portal 26 MB=0x6001bd6998480 00000400:00000200:28.0:1616513073.692405:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.9@tcp1 of length 224/224 into md 0xf97c5379 [1] + 0 00000400:00000200:28.0:1616513073.692410:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775100 00000400:00000200:28.0:1616513073.692415:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.692418:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513073.692422:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775100 00000800:00000200:28.0:1616513073.692425:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.692425:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.692428:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513073.692430:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513073.692486:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969424766080 00010000:00000200:12.0:1616513073.692497:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688969424766080/t0(0) o400->a9192ddf-56aa-7680-e2ad-49adf518e12b@10.31.133.9@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.692521:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688969424766080, offset 192 00000400:00000200:12.0:1616513073.692527:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.9@tcp1 00000400:00000200:12.0:1616513073.692534:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.9@tcp1 routed destination 00000400:00000200:12.0:1616513073.692547:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.9@tcp1(10.31.133.9@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.692553:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.692559:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.692561:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.692563:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.692567:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.692572:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.692573:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513073.692583:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.692589:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.692592:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513073.692594:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513073.692598:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.692602:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782417e8 00000800:00000200:8.0:1616513073.692606:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.692704:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.692715:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.692723:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513073.692725:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513073.692732:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.9@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.692741:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.9@tcp1 of length 224 into portal 12 MB=0x6001bd6998580 00000400:00000200:24.0:1616513073.692747:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.9@tcp1 of length 224/224 into md 0x865b9f31 [1] + 67160 00000400:00000200:24.0:1616513073.692752:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.692754:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513073.692761:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.692764:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.692765:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.692822:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969424766336 00010000:00000200:20.0:1616513073.692833:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc4380 x1688969424766336/t0(0) o400->df2f0c42-c4d9-b462-9c31-ddbab08f1595@10.31.133.9@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.692847:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688969424766336, offset 192 00000400:00000200:20.0:1616513073.692852:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.9@tcp1 00000400:00000200:20.0:1616513073.692859:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.9@tcp1 routed destination 00000400:00000200:20.0:1616513073.692873:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.9@tcp1(10.31.133.9@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.692879:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.692884:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.692887:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.692889:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.692893:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.692900:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.692955:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.692962:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.692965:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.692971:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.692975:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee236d8 00000800:00000200:26.0:1616513073.692979:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.692982:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.696278:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.696330:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.696339:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.696348:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.175@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.696358:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.175@tcp1 of length 224 into portal 26 MB=0x5ffd3eaf3c1c0 00000400:00000200:14.0:1616513073.696364:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.175@tcp1 of length 224/224 into md 0xf97c5389 [1] + 0 00000400:00000200:14.0:1616513073.696369:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774ee0 00000400:00000200:14.0:1616513073.696372:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.696374:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.696378:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774ee0 00000800:00000200:14.0:1616513073.696382:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.696384:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.696384:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.696388:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.696458:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660528579008 00010000:00000200:12.0:1616513073.696469:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688660528579008/t0(0) o400->df086ab3-d569-d27f-20fe-b1b4c69f5421@10.31.129.175@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.696483:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660528579008, offset 192 00000400:00000200:12.0:1616513073.696490:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.175@tcp1 00000400:00000200:12.0:1616513073.696497:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.175@tcp1 routed destination 00000400:00000200:12.0:1616513073.696512:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.175@tcp1(10.31.129.175@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.696519:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.696524:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.696527:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.696529:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.696534:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.696540:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.696541:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513073.696554:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.696558:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.696560:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.696561:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513073.696567:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513073.696570:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240220 00000800:00000200:12.0:1616513073.696573:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.696583:0:48776:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.696636:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.696646:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.696655:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.175@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.696663:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.175@tcp1 of length 224 into portal 12 MB=0x5ffd3eaf3c700 00000400:00000200:26.0:1616513073.696670:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.175@tcp1 of length 224/224 into md 0xf96450b9 [1] + 35296 00000400:00000200:26.0:1616513073.696675:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.696678:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.696684:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.696688:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.696689:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.696694:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.696706:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660528580352 00010000:00000200:22.0:1616513073.696718:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c690d80 x1688660528580352/t0(0) o400->dcc05e09-ba7d-7475-94e0-cf095182f0b1@10.31.129.175@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.696732:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660528580352, offset 192 00000400:00000200:22.0:1616513073.696739:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.175@tcp1 00000400:00000200:22.0:1616513073.696746:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.175@tcp1 routed destination 00000400:00000200:22.0:1616513073.696761:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.175@tcp1(10.31.129.175@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.696770:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.696775:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.696778:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.696780:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.696784:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.696788:0:48776:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.696800:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.696804:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513073.696806:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513073.696809:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513073.696811:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513073.696814:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a6e8 00000800:00000200:22.0:1616513073.696817:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.700239:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513073.700254:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.700263:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513073.700271:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.70@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.700281:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.70@tcp1 of length 224 into portal 26 MB=0x5ffec33b05e40 00000400:00000200:28.0:1616513073.700287:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.70@tcp1 of length 224/224 into md 0xf97c5399 [1] + 0 00000400:00000200:28.0:1616513073.700291:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77753a8 00000400:00000200:28.0:1616513073.700294:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.700297:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513073.700300:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77753a8 00000800:00000200:28.0:1616513073.700303:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.700306:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.700307:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513073.700314:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513073.700317:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.700318:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.700326:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.70@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.700335:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.70@tcp1 of length 224 into portal 12 MB=0x5ffec33b05ec0 00000400:00000200:24.0:1616513073.700342:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.70@tcp1 of length 224/224 into md 0x865b9f31 [1] + 67384 00000400:00000200:24.0:1616513073.700353:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.700355:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513073.700362:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.700363:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688764828114496 00000800:00000200:24.0:1616513073.700365:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.700367:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:12.0:1616513073.700374:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688764828114496/t0(0) o400->434263dc-2032-f05f-aa10-0d2ced4332fb@10.31.142.70@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513073.700388:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688764828114496, offset 192 00000400:00000200:12.0:1616513073.700394:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.70@tcp1 00000400:00000200:12.0:1616513073.700402:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.70@tcp1 routed destination 00000400:00000200:12.0:1616513073.700417:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.70@tcp1(10.31.142.70@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.700423:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:20.0:1616513073.700424:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688764828114624 00000800:00000200:12.0:1616513073.700429:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.700431:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.700445:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:20.0:1616513073.700448:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc7980 x1688764828114624/t0(0) o400->d122d2bc-e5b0-6982-a08d-8c69e440d822@10.31.142.70@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513073.700450:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.700455:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.700457:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513073.700470:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.700475:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:20.0:1616513073.700477:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688764828114624, offset 192 00000400:00000200:12.0:1616513073.700477:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.700478:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513073.700483:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.70@tcp1 00000400:00000200:12.0:1616513073.700483:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513073.700487:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240220 00000400:00000200:20.0:1616513073.700491:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.70@tcp1 routed destination 00000800:00000200:12.0:1616513073.700496:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:20.0:1616513073.700506:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.70@tcp1(10.31.142.70@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.700512:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.700518:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.700520:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.700523:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.700527:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.700535:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.700590:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.700597:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.700600:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513073.700603:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.700607:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.700611:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22b28 00000800:00000200:2.0:1616513073.700614:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.710454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.710523:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.710532:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.710541:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.132@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.710551:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.132@tcp1 of length 224 into portal 26 MB=0x5ffd4b5d89240 00000400:00000200:14.0:1616513073.710557:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.132@tcp1 of length 224/224 into md 0xf97c53b9 [1] + 0 00000400:00000200:14.0:1616513073.710562:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774330 00000400:00000200:14.0:1616513073.710565:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.710568:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.710572:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774330 00000800:00000200:14.0:1616513073.710575:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.710578:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.710579:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.710582:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.710587:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.710588:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513073.710594:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.132@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.710599:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.132@tcp1 of length 224 into portal 12 MB=0x5ffd4b5d89340 00000400:00000200:14.0:1616513073.710605:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.132@tcp1 of length 224/224 into md 0xf96450b9 [1] + 35520 00000400:00000200:14.0:1616513073.710611:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.710613:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.710618:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.710620:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.710621:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513073.710636:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663932572224 00000100:00000200:22.0:1616513073.710640:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663932572480 00010000:00000200:12.0:1616513073.710647:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688663932572224/t0(0) o400->ac5da34e-b51b-71f0-977e-f40ea78874e5@10.31.136.132@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:22.0:1616513073.710652:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c8900 x1688663932572480/t0(0) o400->99f974cb-bd01-8f75-e591-23fe9e8d15be@10.31.136.132@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513073.710662:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663932572224, offset 192 00000100:00000200:22.0:1616513073.710666:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663932572480, offset 192 00000400:00000200:12.0:1616513073.710668:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.132@tcp1 00000400:00000200:22.0:1616513073.710672:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.132@tcp1 00000400:00000200:12.0:1616513073.710676:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.132@tcp1 routed destination 00000400:00000200:22.0:1616513073.710680:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.132@tcp1 routed destination 00000400:00000200:12.0:1616513073.710691:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.132@tcp1(10.31.136.132@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513073.710695:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.132@tcp1(10.31.136.132@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.710698:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.710701:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513073.710704:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.710707:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513073.710708:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.710710:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513073.710710:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.710712:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.710715:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513073.710716:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.710721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513073.710728:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513073.710729:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513073.710738:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.710746:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.710749:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513073.710750:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.710755:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.710759:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a6e8 00000800:00000200:2.0:1616513073.710762:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513073.710782:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.710789:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.710791:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513073.710797:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.710803:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240220 00000800:00000200:28.0:1616513073.710806:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.710809:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.715981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.715992:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.715999:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.716008:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.128@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.716017:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.128@tcp1 of length 224 into portal 26 MB=0x5f95f13d53ec0 00000400:00000200:8.0:1616513073.716024:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.128@tcp1 of length 224/224 into md 0xf97c53e9 [1] + 0 00000400:00000200:8.0:1616513073.716029:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774110 00000400:00000200:8.0:1616513073.716031:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.716033:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.716039:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774110 00000800:00000200:8.0:1616513073.716042:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.716044:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513073.716044:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.716051:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.716054:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.716055:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513073.716063:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.128@tcp1 : PUT - for me 00000400:00000200:12.0:1616513073.716072:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.128@tcp1 of length 224 into portal 12 MB=0x5f95f13d53f40 00000400:00000200:12.0:1616513073.716078:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.128@tcp1 of length 224/224 into md 0x865b9f31 [1] + 67608 00000400:00000200:12.0:1616513073.716084:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513073.716086:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513073.716093:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.716096:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.716098:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513073.716103:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681561633504960 00010000:00000200:16.0:1616513073.716114:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1681561633504960/t0(0) o400->8e41ec44-29bc-a4f8-cee3-4940eadfd1cf@10.31.143.128@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.716128:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681561633504960, offset 192 00000400:00000200:16.0:1616513073.716135:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.128@tcp1 00000400:00000200:16.0:1616513073.716143:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.128@tcp1 routed destination 00000100:00000200:20.0:1616513073.716157:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681561633505088 00000400:00000200:16.0:1616513073.716158:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.128@tcp1(10.31.143.128@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.716164:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:20.0:1616513073.716170:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57bf00 x1681561633505088/t0(0) o400->ea4f45db-ae47-1407-b27c-638ff9a77d77@10.31.143.128@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513073.716170:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.716173:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.716175:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.716180:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513073.716184:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681561633505088, offset 192 00000800:00000200:11.2:1616513073.716185:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513073.716187:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513073.716190:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.128@tcp1 00000400:00000200:20.0:1616513073.716198:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.128@tcp1 routed destination 00000800:00000200:24.0:1616513073.716204:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.716211:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513073.716213:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.128@tcp1(10.31.143.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513073.716214:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.716214:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513073.716220:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513073.716221:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513073.716225:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513073.716226:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a436d8 00000800:00000200:20.0:1616513073.716228:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513073.716230:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:20.0:1616513073.716230:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.716235:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.716241:0:48778:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.716296:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.716303:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.716306:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.716313:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.716317:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee224c8 00000800:00000200:14.0:1616513073.716320:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.716323:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.716683:0:48778:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.716742:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.716751:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513073.716770:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.8@tcp1 : PUT - for me 00000400:00000200:22.0:1616513073.716780:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.8@tcp1 of length 224 into portal 26 MB=0x5ffd4a198afc0 00000400:00000200:22.0:1616513073.716786:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.8@tcp1 of length 224/224 into md 0xf97c53f9 [1] + 0 00000400:00000200:22.0:1616513073.716790:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774088 00000400:00000200:22.0:1616513073.716795:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513073.716798:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513073.716802:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774088 00000800:00000200:2.0:1616513073.716804:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513073.716805:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.716807:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513073.716809:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.716817:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663592841152 00010000:00000200:16.0:1616513073.716823:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688663592841152/t0(0) o400->8830ccf4-e4ab-e0f6-f5dd-967052ffa43d@10.31.138.8@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.716845:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663592841152, offset 192 00000400:00000200:16.0:1616513073.716850:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.8@tcp1 00000400:00000200:16.0:1616513073.716856:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.8@tcp1 routed destination 00000400:00000200:16.0:1616513073.716869:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.8@tcp1(10.31.138.8@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.716875:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.716880:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.716882:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.716884:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.716887:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513073.716891:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.716893:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.716902:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.716905:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.716907:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513073.716911:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513073.716913:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513073.716917:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43078 00000800:00000200:12.0:1616513073.716920:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.717069:0:48778:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.717078:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.717084:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513073.717087:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513073.717091:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.8@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.717099:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.8@tcp1 of length 224 into portal 12 MB=0x5ffd4a198b100 00000400:00000200:14.0:1616513073.717104:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.8@tcp1 of length 224/224 into md 0xf96450b9 [1] + 35744 00000400:00000200:14.0:1616513073.717109:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.717111:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.717117:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.717119:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.717121:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513073.717135:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663592841472 00010000:00000200:22.0:1616513073.717145:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cda00 x1688663592841472/t0(0) o400->767b926d-a84f-e17d-7f17-528eef9ebc66@10.31.138.8@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.717157:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663592841472, offset 192 00000400:00000200:22.0:1616513073.717163:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.8@tcp1 00000400:00000200:22.0:1616513073.717169:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.8@tcp1 routed destination 00000400:00000200:22.0:1616513073.717182:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.8@tcp1(10.31.138.8@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.717188:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.717193:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.717195:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.717197:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.717200:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.717205:0:48778:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.717215:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.717220:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.717222:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513073.717224:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513073.717229:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.717232:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a440 00000800:00000200:2.0:1616513073.717236:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.721028:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.721039:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.721046:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.721054:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.5@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.721064:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.5@tcp1 of length 224 into portal 26 MB=0x5ffd59e34d880 00000400:00000200:8.0:1616513073.721070:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.5@tcp1 of length 224/224 into md 0xf97c5409 [1] + 0 00000400:00000200:8.0:1616513073.721074:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774b28 00000400:00000200:8.0:1616513073.721076:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.721079:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.721083:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774b28 00000800:00000200:8.0:1616513073.721085:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.721088:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.721089:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.721092:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513073.721147:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667830933632 00010000:00000200:16.0:1616513073.721158:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688667830933632/t0(0) o400->e457ea0b-f4ef-a816-d54e-c2d0e6771747@10.31.130.5@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.721172:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667830933632, offset 192 00000400:00000200:16.0:1616513073.721179:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.5@tcp1 00000400:00000200:16.0:1616513073.721187:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.5@tcp1 routed destination 00000400:00000200:16.0:1616513073.721201:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.5@tcp1(10.31.130.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.721208:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.721214:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.721217:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.721219:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.721224:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.721228:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513073.721230:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513073.721281:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.721287:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.721300:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513073.721306:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.721312:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a426e8 00000800:00000200:28.0:1616513073.721315:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.721319:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.721325:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.721327:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.721332:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.5@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.721340:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.5@tcp1 of length 224 into portal 12 MB=0x5ffd59e34d900 00000400:00000200:28.0:1616513073.721345:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.5@tcp1 of length 224/224 into md 0x865b9f31 [1] + 67832 00000400:00000200:28.0:1616513073.721350:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.721352:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513073.721357:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.721360:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.721361:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.721418:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667830933760 00010000:00000200:20.0:1616513073.721430:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf83180 x1688667830933760/t0(0) o400->1f279942-b786-ebcf-ab75-bc35f72f25ea@10.31.130.5@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.721454:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667830933760, offset 192 00000400:00000200:20.0:1616513073.721459:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.5@tcp1 00000400:00000200:20.0:1616513073.721466:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.5@tcp1 routed destination 00000400:00000200:20.0:1616513073.721480:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.5@tcp1(10.31.130.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.721487:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.721493:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.721495:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.721497:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.721502:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.721507:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.721560:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.721566:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.721569:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.721575:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.721580:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23ed0 00000800:00000200:26.0:1616513073.721584:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.721587:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.759762:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.759819:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.759839:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513073.759848:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.42@tcp1 : PUT - for me 00000400:00000200:22.0:1616513073.759857:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.42@tcp1 of length 224 into portal 26 MB=0x5ffdc1244bcc0 00000400:00000200:22.0:1616513073.759863:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.42@tcp1 of length 224/224 into md 0xf97c5419 [1] + 0 00000400:00000200:22.0:1616513073.759868:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775b18 00000400:00000200:22.0:1616513073.759871:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513073.759873:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513073.759877:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775b18 00000800:00000200:2.0:1616513073.759877:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.759880:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.759883:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513073.759885:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513073.759887:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.759888:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.759897:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.42@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.759906:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.42@tcp1 of length 224 into portal 12 MB=0x5ffdc1244bfc0 00000400:00000200:2.0:1616513073.759913:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.42@tcp1 of length 224/224 into md 0xf96450b9 [1] + 35968 00000400:00000200:2.0:1616513073.759918:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.759921:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513073.759928:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.759931:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.759933:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.759944:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695547935936 00000100:00000200:22.0:1616513073.759949:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695547936704 00010000:00000200:16.0:1616513073.759957:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688695547935936/t0(0) o400->702a1d24-1e68-91c5-21ee-2a9eab32ea08@10.31.140.42@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:22.0:1616513073.759959:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138ec00 x1688695547936704/t0(0) o400->997bf0b0-693f-3dc3-bb75-4eb3e6ef64a7@10.31.140.42@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513073.759971:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695547936704, offset 192 00000100:00000200:16.0:1616513073.759972:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695547935936, offset 192 00000400:00000200:22.0:1616513073.759977:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.42@tcp1 00000400:00000200:16.0:1616513073.759978:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.42@tcp1 00000400:00000200:22.0:1616513073.759983:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.42@tcp1 routed destination 00000400:00000200:16.0:1616513073.759986:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.42@tcp1 routed destination 00000400:00000200:22.0:1616513073.760002:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.42@tcp1(10.31.140.42@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513073.760003:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.42@tcp1(10.31.140.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513073.760008:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.760009:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513073.760014:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.760015:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513073.760016:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513073.760018:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.760018:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.760020:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.760022:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513073.760025:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513073.760029:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513073.760030:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513073.760033:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513073.760081:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.760088:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.760091:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513073.760092:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513073.760097:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.760102:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a423b8 00000800:00000200:24.0:1616513073.760103:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513073.760106:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513073.760110:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.760113:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.760114:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513073.760120:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.760124:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a220 00000800:00000200:24.0:1616513073.760128:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.770591:0:6484:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924272, portal 4 00000100:00000200:20.0:1616513073.770596:0:6484:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924272, offset 0 00000400:00000200:20.0:1616513073.770602:0:6484:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:20.0:1616513073.770607:0:6484:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:20.0:1616513073.770620:0:6484:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.770626:0:6484:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:20.0:1616513073.770631:0:6484:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.770634:0:6484:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:20.0:1616513073.770636:0:6484:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:20.0:1616513073.770641:0:6484:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513073.770687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513073.770747:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513073.770754:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513073.770757:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.770764:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513073.770770:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affd3f00 x1693067725924272/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513073.770782:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23870 00000800:00000200:2.0:1616513073.770786:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513073.770789:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513073.770817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513073.770873:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513073.770882:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513073.770891:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513073.770899:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6267b0 00000400:00000200:22.0:1616513073.770905:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c5ad9 [1] + 192 00000400:00000200:22.0:1616513073.770911:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513073.770916:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affd3f00 x1693067725924272/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513073.770926:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513073.770930:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513073.770933:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513073.770935:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:20.0:1616513073.770938:0:6484:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affd3f00 x1693067725924272/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513073.770950:0:6484:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affd3f00 x1693067725924272/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513073.770959:0:6484:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23430 00000800:00000200:7.2:1616513073.774811:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.774867:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.774876:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.774884:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.138@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.774895:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.138@tcp1 of length 224 into portal 26 MB=0x5ffd767439600 00000400:00000200:2.0:1616513073.774901:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.138@tcp1 of length 224/224 into md 0xf97c5439 [1] + 0 00000400:00000200:2.0:1616513073.774905:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77756d8 00000400:00000200:2.0:1616513073.774908:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.774911:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.774915:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77756d8 00000800:00000200:2.0:1616513073.774917:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.774920:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.774920:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.774924:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.774960:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513073.774981:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675499087360 00010000:00000200:16.0:1616513073.774992:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688675499087360/t0(0) o400->b4f6af33-02f6-423c-b903-8f786c0588cc@10.31.129.138@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.775006:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688675499087360, offset 192 00000400:00000200:16.0:1616513073.775013:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.138@tcp1 00000800:00000200:14.0:1616513073.775014:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513073.775020:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.138@tcp1 routed destination 00000800:00000200:14.0:1616513073.775023:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.775032:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.138@tcp1 : PUT - for me 00000400:00000200:16.0:1616513073.775036:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.138@tcp1(10.31.129.138@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513073.775040:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.138@tcp1 of length 224 into portal 12 MB=0x5ffd767439980 00000800:00000200:16.0:1616513073.775042:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.775048:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513073.775049:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.138@tcp1 of length 224/224 into md 0xf96450b9 [1] + 36192 00000800:00000200:16.0:1616513073.775051:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.775053:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513073.775054:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.775057:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513073.775058:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.775063:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513073.775064:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513073.775065:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513073.775065:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.775069:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.775071:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513073.775087:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675499088256 00010000:00000200:26.0:1616513073.775099:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53ccc80 x1688675499088256/t0(0) o400->c374d4ac-5166-8050-3d4c-115501838d30@10.31.129.138@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513073.775113:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688675499088256, offset 192 00000800:00000200:12.0:1616513073.775116:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513073.775119:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.138@tcp1 00000800:00000200:12.0:1616513073.775122:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.775125:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.775126:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.138@tcp1 routed destination 00000400:00000200:12.0:1616513073.775131:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513073.775136:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42908 00000800:00000200:12.0:1616513073.775139:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513073.775141:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.138@tcp1(10.31.129.138@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513073.775142:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513073.775147:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513073.775153:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513073.775156:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.775158:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.775162:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.775167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.775178:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.775182:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.775184:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.775186:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.775190:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.775193:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:26.0:1616513073.775196:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:11.0:1616513073.786626:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924288, portal 4 00000100:00000200:11.0:1616513073.786631:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924288, offset 0 00000400:00000200:11.0:1616513073.786637:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:11.0:1616513073.786642:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:11.0:1616513073.786653:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513073.786658:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:11.0:1616513073.786664:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513073.786666:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:11.0:1616513073.786668:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:11.0:1616513073.786673:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:7.2:1616513073.786708:0:48785:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:27.2:1616513073.786719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513073.786740:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513073.786745:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513073.786748:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513073.786754:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513073.786760:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600900 x1693067725924288/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513073.786764:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:27.0:1616513073.786773:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3b7e8 00000800:00000200:22.0:1616513073.786774:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:27.0:1616513073.786777:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513073.786780:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:22.0:1616513073.786783:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.90@tcp1 : PUT - for me 00000400:00000200:22.0:1616513073.786792:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.90@tcp1 of length 224 into portal 26 MB=0x6012e1984c7c0 00000400:00000200:22.0:1616513073.786799:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.90@tcp1 of length 224/224 into md 0xf97c5459 [1] + 0 00000400:00000200:22.0:1616513073.786808:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774770 00000400:00000200:22.0:1616513073.786811:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513073.786814:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513073.786817:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513073.786818:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774770 00000800:00000200:22.0:1616513073.786822:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.786825:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513073.786826:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.786884:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690147368519616 00000800:00000200:27.2:1616513073.786891:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00010000:00000200:16.0:1616513073.786896:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1690147368519616/t0(0) o400->d5c509e1-ce00-68fc-f695-e0d6aba9c177@10.31.140.90@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:27.0:1616513073.786900:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513073.786906:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:11.0:1616513073.786908:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:16.0:1616513073.786910:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690147368519616, offset 192 00000400:00000200:27.0:1616513073.786914:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513073.786917:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.90@tcp1 00000400:00000200:27.0:1616513073.786922:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6267c0 00000400:00000200:16.0:1616513073.786925:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.90@tcp1 routed destination 00000400:00000200:27.0:1616513073.786928:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716563d [1] + 192 00000400:00000200:27.0:1616513073.786934:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513073.786939:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600900 x1693067725924288/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513073.786940:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.90@tcp1(10.31.140.90@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.786947:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:27.0:1616513073.786952:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:16.0:1616513073.786954:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513073.786955:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513073.786957:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:11.0:1616513073.786958:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600900 x1693067725924288/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513073.786959:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.786961:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.786966:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.786972:0:6511:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513073.786973:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:11.0:1616513073.786977:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600900 x1693067725924288/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513073.786982:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:11.0:1616513073.786985:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a2a8 00000800:00000200:8.0:1616513073.786987:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.786989:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.786995:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.787000:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43210 00000800:00000200:8.0:1616513073.787003:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.787005:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.787475:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.787533:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.787550:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513073.787559:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.90@tcp1 : PUT - for me 00000400:00000200:12.0:1616513073.787568:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.90@tcp1 of length 224 into portal 12 MB=0x6012e1984cb00 00000400:00000200:12.0:1616513073.787574:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.90@tcp1 of length 224/224 into md 0x865b9f31 [1] + 68056 00000400:00000200:12.0:1616513073.787580:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513073.787585:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513073.787592:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.787595:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513073.787596:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513073.787601:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.787658:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690147368520448 00010000:00000200:20.0:1616513073.787670:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57e780 x1690147368520448/t0(0) o400->1b66155a-fd43-daef-0848-317550d3090e@10.31.140.90@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.787685:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690147368520448, offset 192 00000400:00000200:20.0:1616513073.787691:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.90@tcp1 00000400:00000200:20.0:1616513073.787698:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.90@tcp1 routed destination 00000400:00000200:20.0:1616513073.787713:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.90@tcp1(10.31.140.90@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.787720:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.787726:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.787730:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.787732:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.787738:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.787744:0:48785:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.787799:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.787806:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.787809:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.787815:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.787818:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23430 00000800:00000200:2.0:1616513073.787822:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.787825:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.813969:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.813982:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.813994:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.814002:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.174@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.814010:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.174@tcp1 of length 224 into portal 26 MB=0x60085ccef4200 00000400:00000200:8.0:1616513073.814016:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.174@tcp1 of length 224/224 into md 0xf97c5469 [1] + 0 00000400:00000200:8.0:1616513073.814020:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774198 00000400:00000200:8.0:1616513073.814022:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.814025:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.814029:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774198 00000800:00000200:8.0:1616513073.814031:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.814034:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.814035:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.814038:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513073.814092:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689424529146368 00010000:00000200:16.0:1616513073.814102:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1689424529146368/t0(0) o400->d1642000-c431-5b58-a2bb-c612f4c373c4@10.31.132.174@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.814116:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689424529146368, offset 192 00000400:00000200:16.0:1616513073.814122:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.174@tcp1 00000400:00000200:16.0:1616513073.814129:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.174@tcp1 routed destination 00000400:00000200:16.0:1616513073.814143:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.174@tcp1(10.31.132.174@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.814149:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.814155:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.814158:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.814160:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.814164:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.814169:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513073.814171:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513073.814222:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.814229:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.814232:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513073.814238:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.814244:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42440 00000800:00000200:28.0:1616513073.814247:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.814250:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.814580:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.814593:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.814602:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.814602:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513073.814611:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.174@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.814620:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.174@tcp1 of length 224 into portal 12 MB=0x60085ccef42c0 00000400:00000200:24.0:1616513073.814626:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.174@tcp1 of length 224/224 into md 0x865b9f31 [1] + 68280 00000400:00000200:24.0:1616513073.814632:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.814634:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513073.814642:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.814645:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.814646:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.814704:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689424529146560 00010000:00000200:20.0:1616513073.814716:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc5e80 x1689424529146560/t0(0) o400->5fb29952-d8f6-62af-3029-e25262e444ea@10.31.132.174@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.814729:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689424529146560, offset 192 00000400:00000200:20.0:1616513073.814735:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.174@tcp1 00000400:00000200:20.0:1616513073.814742:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.174@tcp1 routed destination 00000400:00000200:20.0:1616513073.814756:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.174@tcp1(10.31.132.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.814762:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.814767:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.814770:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.814773:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.814777:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.814785:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.814842:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.814849:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513073.814852:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.814859:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.814862:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee233a8 00000800:00000200:14.0:1616513073.814866:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.814869:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513073.818574:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924304, portal 4 00000100:00000200:24.0:1616513073.818579:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924304, offset 0 00000400:00000200:24.0:1616513073.818586:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:24.0:1616513073.818591:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:24.0:1616513073.818604:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513073.818613:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:24.0:1616513073.818618:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000100:00000200:15.0:1616513073.818619:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924320, portal 4 00000800:00000200:24.0:1616513073.818621:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:24.0:1616513073.818624:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000100:00000200:15.0:1616513073.818626:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924320, offset 0 00000800:00000200:24.0:1616513073.818628:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000400:00000200:15.0:1616513073.818633:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:15.0:1616513073.818642:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:15.0:1616513073.818655:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.818662:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:15.0:1616513073.818668:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.818671:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:15.0:1616513073.818673:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:25.2:1616513073.818676:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:15.0:1616513073.818679:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513073.818726:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:2.0:1616513073.818733:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513073.818740:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513073.818743:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.818750:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513073.818756:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affd6c00 x1693067725924304/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513073.818768:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2908 00000800:00000200:2.0:1616513073.818773:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513073.818776:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:28.0:1616513073.818786:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513073.818793:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:25.2:1616513073.818793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000400:00000200:28.0:1616513073.818796:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513073.818802:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000400:00000200:28.0:1616513073.818803:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513073.818809:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bfb180 x1693067725924320/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513073.818812:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:26.0:1616513073.818820:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513073.818821:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774198 00000800:00000200:28.0:1616513073.818825:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:26.0:1616513073.818827:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6267d0 00000800:00000200:28.0:1616513073.818828:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000400:00000200:26.0:1616513073.818834:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c5b29 [1] + 192 00000400:00000200:26.0:1616513073.818838:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.818843:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affd6c00 x1693067725924304/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513073.818856:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513073.818860:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513073.818861:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:26.0:1616513073.818865:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:24.0:1616513073.818865:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affd6c00 x1693067725924304/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:24.0:1616513073.818879:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affd6c00 x1693067725924304/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513073.818889:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000800:00000200:5.2:1616513073.818919:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513073.818927:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513073.818933:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:24.0:1616513073.818937:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:8.0:1616513073.818939:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513073.818944:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6267e0 00000400:00000200:8.0:1616513073.818950:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d39c1 [1] + 192 00000400:00000200:8.0:1616513073.818953:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.818958:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bfb180 x1693067725924320/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513073.818970:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513073.818972:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513073.818973:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:15.0:1616513073.818980:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bfb180 x1693067725924320/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513073.818993:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bfb180 x1693067725924320/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513073.819003:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775f58 00000800:00000200:11.2:1616513073.821790:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513073.821844:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.821853:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513073.821862:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.144@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.821871:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.144@tcp1 of length 224 into portal 12 MB=0x5ffd79dae3e40 00000400:00000200:28.0:1616513073.821878:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.144@tcp1 of length 224/224 into md 0x865b9f31 [1] + 68504 00000400:00000200:28.0:1616513073.821883:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.821888:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513073.821894:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.821897:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513073.821898:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513073.821905:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513073.821908:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.821908:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513073.821917:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.144@tcp1 : PUT - for me 00000400:00000200:12.0:1616513073.821926:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.144@tcp1 of length 224 into portal 26 MB=0x5ffd79dae3dc0 00000400:00000200:12.0:1616513073.821933:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.144@tcp1 of length 224/224 into md 0xf97c5479 [1] + 0 00000400:00000200:12.0:1616513073.821938:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774ff0 00000400:00000200:12.0:1616513073.821941:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513073.821944:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513073.821948:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774ff0 00000800:00000200:12.0:1616513073.821951:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.821954:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.821956:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.821958:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676412046912 00010000:00000200:20.0:1616513073.821970:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf84800 x1688676412046912/t0(0) o400->360eb5e2-f87d-1622-5b32-6a31506e6c4c@10.31.143.144@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.821985:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688676412046912, offset 192 00000400:00000200:20.0:1616513073.821991:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.144@tcp1 00000400:00000200:20.0:1616513073.821999:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.144@tcp1 routed destination 00000400:00000200:20.0:1616513073.822014:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.144@tcp1(10.31.143.144@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513073.822014:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676412046784 00000800:00000200:20.0:1616513073.822021:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:16.0:1616513073.822025:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688676412046784/t0(0) o400->fe3df388-a833-fe65-c2ab-b17ce360503e@10.31.143.144@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513073.822026:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.822029:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.822031:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.822036:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513073.822039:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688676412046784, offset 192 00000800:00000200:11.2:1616513073.822041:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513073.822046:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.144@tcp1 00000800:00000200:8.0:1616513073.822050:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513073.822053:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.144@tcp1 routed destination 00000800:00000200:8.0:1616513073.822055:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.822057:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513073.822059:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513073.822063:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.822067:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee233a8 00000400:00000200:16.0:1616513073.822068:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.144@tcp1(10.31.143.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513073.822070:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513073.822075:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513073.822081:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.822083:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.822085:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513073.822090:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.822097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513073.822098:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513073.822153:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.822159:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.822162:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.822169:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.822174:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42110 00000800:00000200:2.0:1616513073.822178:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.822181:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.850615:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924336, portal 4 00000100:00000200:15.0:1616513073.850620:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924336, offset 0 00000400:00000200:15.0:1616513073.850626:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:15.0:1616513073.850632:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:15.0:1616513073.850644:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.850650:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:15.0:1616513073.850656:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.850658:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:15.0:1616513073.850661:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:15.0:1616513073.850665:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513073.850711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.850728:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513073.850736:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513073.850738:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513073.850744:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513073.850750:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd604c80 x1693067725924336/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513073.850762:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774770 00000800:00000200:1.0:1616513073.850766:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513073.850768:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513073.850883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513073.850891:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.850898:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513073.850905:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513073.850913:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6267f0 00000400:00000200:1.0:1616513073.850918:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff9a5 [1] + 192 00000400:00000200:1.0:1616513073.850923:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513073.850927:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd604c80 x1693067725924336/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513073.850939:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:5.0:1616513073.850942:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513073.850942:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000100:00000200:15.0:1616513073.850946:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd604c80 x1693067725924336/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513073.850946:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:15.0:1616513073.850958:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd604c80 x1693067725924336/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513073.850967:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775f58 00000800:00000200:7.2:1616513073.855620:0:48792:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.855677:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.855687:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.855696:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.20@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.855706:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.20@tcp1 of length 224 into portal 26 MB=0x5ffd3dc6af980 00000400:00000200:14.0:1616513073.855713:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.20@tcp1 of length 224/224 into md 0xf97c5489 [1] + 0 00000400:00000200:14.0:1616513073.855727:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775540 00000400:00000200:14.0:1616513073.855730:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.855733:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.855738:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775540 00000800:00000200:14.0:1616513073.855741:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.855744:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.855746:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.855749:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.855760:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513073.855769:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.20@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.855778:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.20@tcp1 of length 224 into portal 12 MB=0x5ffd3dc6afa00 00000400:00000200:26.0:1616513073.855785:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.20@tcp1 of length 224/224 into md 0xf96450b9 [1] + 36416 00000400:00000200:26.0:1616513073.855790:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.855793:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513073.855802:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:16.0:1616513073.855803:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660284733824 00000800:00000200:26.0:1616513073.855805:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.855806:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513073.855812:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:16.0:1616513073.855815:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688660284733824/t0(0) o400->687c5edb-df0f-87ef-6bd4-8f4d2381186e@10.31.138.20@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513073.855823:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660284733952 00000100:00000200:16.0:1616513073.855829:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660284733824, offset 192 00010000:00000200:30.0:1616513073.855834:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c690480 x1688660284733952/t0(0) o400->725c47c9-41a2-df06-427a-b9ce286a9897@10.31.138.20@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513073.855836:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.20@tcp1 00000400:00000200:16.0:1616513073.855844:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.20@tcp1 routed destination 00000100:00000200:30.0:1616513073.855847:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660284733952, offset 192 00000400:00000200:30.0:1616513073.855852:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.20@tcp1 00000400:00000200:30.0:1616513073.855859:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.20@tcp1 routed destination 00000400:00000200:16.0:1616513073.855859:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.20@tcp1(10.31.138.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.855866:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.855872:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513073.855873:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.20@tcp1(10.31.138.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.855875:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.855877:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513073.855879:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513073.855882:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513073.855884:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513073.855886:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513073.855888:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513073.855889:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.855890:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513073.855892:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.855898:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.855948:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.855955:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.855958:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513073.855963:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513073.855964:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513073.855970:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:12.0:1616513073.855970:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42f68 00000400:00000200:22.0:1616513073.855973:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513073.855973:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513073.855974:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513073.855976:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513073.855980:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513073.855984:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eebb0 00000800:00000200:22.0:1616513073.855987:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:23.0:1616513073.870548:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:23.0:1616513073.870558:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513073.885296:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513073.885354:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.885365:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.885366:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.885373:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513073.885374:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.226@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.885384:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.226@tcp1 of length 224 into portal 26 MB=0x5fff211594340 00000400:00000200:28.0:1616513073.885391:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.226@tcp1 of length 224/224 into md 0xf97c54b9 [1] + 0 00000400:00000200:28.0:1616513073.885396:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775870 00000400:00000200:28.0:1616513073.885399:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.885401:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513073.885407:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775870 00000800:00000200:28.0:1616513073.885410:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.885413:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513073.885415:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.885417:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.885423:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513073.885426:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.885429:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.226@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.885450:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.226@tcp1 of length 224 into portal 12 MB=0x5fff2115943c0 00000400:00000200:28.0:1616513073.885455:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.226@tcp1 of length 224/224 into md 0x865b9f31 [1] + 68728 00000400:00000200:28.0:1616513073.885459:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.885462:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513073.885467:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.885469:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.885471:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513073.885471:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688790021784384 00010000:00000200:16.0:1616513073.885483:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688790021784384/t0(0) o400->4c0e5693-ef2c-2642-c313-726d84b06f29@10.31.136.226@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.885498:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688790021784384, offset 192 00000400:00000200:16.0:1616513073.885504:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.226@tcp1 00000400:00000200:16.0:1616513073.885512:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.226@tcp1 routed destination 00000400:00000200:16.0:1616513073.885528:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.226@tcp1(10.31.136.226@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:20.0:1616513073.885534:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688790021784512 00000800:00000200:16.0:1616513073.885535:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.885541:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.885543:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.885545:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:20.0:1616513073.885546:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf86780 x1688790021784512/t0(0) o400->478c2eb9-719c-7216-6c90-db0ef1b32dc1@10.31.136.226@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513073.885551:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.885555:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513073.885558:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513073.885563:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.885568:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.885570:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:20.0:1616513073.885571:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688790021784512, offset 192 00000800:00000200:12.0:1616513073.885572:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513073.885576:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513073.885578:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.226@tcp1 00000400:00000200:8.0:1616513073.885582:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000400:00000200:20.0:1616513073.885585:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.226@tcp1 routed destination 00000800:00000200:8.0:1616513073.885591:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:20.0:1616513073.885600:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.226@tcp1(10.31.136.226@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.885607:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.885613:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.885616:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.885618:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.885623:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.885669:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.885727:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.885734:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.885736:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.885743:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.885747:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23430 00000800:00000200:2.0:1616513073.885751:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.885754:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.899742:0:48796:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.899798:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.899808:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.899817:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.52@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.899826:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.52@tcp1 of length 224 into portal 12 MB=0x5ffd4960a3400 00000400:00000200:14.0:1616513073.899832:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.52@tcp1 of length 224/224 into md 0xf96450b9 [1] + 36640 00000400:00000200:14.0:1616513073.899842:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.899845:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513073.899852:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.899852:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.899855:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.899857:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513073.899873:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663398953984 00010000:00000200:30.0:1616513073.899885:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53ce780 x1688663398953984/t0(0) o400->ffeb2481-c5e2-da5a-610c-8e653e554dde@10.31.138.52@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513073.899900:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663398953984, offset 192 00000400:00000200:30.0:1616513073.899917:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.52@tcp1 00000400:00000200:30.0:1616513073.899923:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.52@tcp1 routed destination 00000400:00000200:30.0:1616513073.899937:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.52@tcp1(10.31.138.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513073.899942:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513073.899947:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513073.899950:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513073.899951:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513073.899956:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.900000:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.900056:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.900062:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.900065:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513073.900072:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.900076:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efed0 00000800:00000200:24.0:1616513073.900080:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.900083:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.900800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.900812:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.900822:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.900831:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.118@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.900840:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.118@tcp1 of length 224 into portal 26 MB=0x5ffd4d80b4fc0 00000400:00000200:2.0:1616513073.900847:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.118@tcp1 of length 224/224 into md 0xf97c54c9 [1] + 0 00000400:00000200:2.0:1616513073.900852:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77758f8 00000400:00000200:2.0:1616513073.900855:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.900857:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.900862:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77758f8 00000800:00000200:2.0:1616513073.900865:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.900866:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.900868:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.900870:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.900927:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664506322880 00010000:00000200:16.0:1616513073.900938:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688664506322880/t0(0) o400->918a6a4a-23ec-9ae2-4f25-82dd461fe578@10.31.136.118@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.900952:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664506322880, offset 192 00000400:00000200:16.0:1616513073.900961:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.118@tcp1 00000400:00000200:16.0:1616513073.900969:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.118@tcp1 routed destination 00000400:00000200:16.0:1616513073.900984:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.118@tcp1(10.31.136.118@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.900991:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513073.900996:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.900999:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.901001:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513073.901006:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.901012:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513073.901013:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513073.901076:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.901083:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513073.901086:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513073.901092:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513073.901098:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43b18 00000800:00000200:22.0:1616513073.901102:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513073.901105:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.901314:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.901369:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.901379:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.901387:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513073.901388:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.118@tcp1 : PUT - for me 00000400:00000200:12.0:1616513073.901397:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.118@tcp1 of length 224 into portal 12 MB=0x5ffd4d80b5100 00000400:00000200:12.0:1616513073.901403:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.118@tcp1 of length 224/224 into md 0x865b9f31 [1] + 68952 00000400:00000200:12.0:1616513073.901408:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513073.901411:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513073.901417:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.901422:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.901424:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.901479:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664506323200 00010000:00000200:20.0:1616513073.901491:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc578900 x1688664506323200/t0(0) o400->98c44682-9a09-c00a-346f-8cc12502839c@10.31.136.118@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.901504:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664506323200, offset 192 00000400:00000200:20.0:1616513073.901510:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.118@tcp1 00000400:00000200:20.0:1616513073.901517:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.118@tcp1 routed destination 00000400:00000200:20.0:1616513073.901531:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.118@tcp1(10.31.136.118@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.901536:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.901542:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.901544:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.901546:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.901551:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.901556:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.901609:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.901616:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.901619:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513073.901625:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513073.901629:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23870 00000800:00000200:24.0:1616513073.901633:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.901636:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.901665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513073.901677:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.901685:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513073.901687:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513073.901694:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.52@tcp1 : PUT - for me 00000400:00000200:26.0:1616513073.901702:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.52@tcp1 of length 224 into portal 26 MB=0x5ffd4960a3300 00000400:00000200:26.0:1616513073.901709:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.52@tcp1 of length 224/224 into md 0xf97c5509 [1] + 0 00000400:00000200:26.0:1616513073.901714:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774908 00000400:00000200:26.0:1616513073.901716:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513073.901719:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513073.901724:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774908 00000800:00000200:26.0:1616513073.901727:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.901729:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.901731:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.901737:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663398953728 00010000:00000200:16.0:1616513073.901743:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688663398953728/t0(0) o400->ba5c95c1-6a46-454b-5cbb-813a46f96b0e@10.31.138.52@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.901755:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663398953728, offset 192 00000400:00000200:16.0:1616513073.901760:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.52@tcp1 00000400:00000200:16.0:1616513073.901766:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.52@tcp1 routed destination 00000400:00000200:16.0:1616513073.901779:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.52@tcp1(10.31.138.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.901785:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513073.901789:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.901792:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.901794:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513073.901797:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513073.901801:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513073.901801:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.901810:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.901814:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513073.901816:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513073.901820:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513073.901821:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513073.901825:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42bb0 00000800:00000200:22.0:1616513073.901827:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:11.0:1616513073.914575:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924352, portal 4 00000100:00000200:11.0:1616513073.914579:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924352, offset 0 00000400:00000200:11.0:1616513073.914585:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:11.0:1616513073.914590:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:11.0:1616513073.914601:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513073.914606:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:11.0:1616513073.914612:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513073.914614:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:11.0:1616513073.914616:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:11.0:1616513073.914620:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:11.0:1616513073.914630:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924368, portal 4 00000100:00000200:11.0:1616513073.914632:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924368, offset 0 00000400:00000200:11.0:1616513073.914636:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:11.0:1616513073.914640:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:11.0:1616513073.914649:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513073.914654:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:11.0:1616513073.914658:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513073.914660:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:11.0:1616513073.914662:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:11.0:1616513073.914665:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:1.2:1616513073.914667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.914686:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513073.914692:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513073.914694:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513073.914702:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513073.914708:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded3a80 x1693067725924352/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:25.2:1616513073.914711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000400:00000200:1.0:1616513073.914720:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3b3a8 00000800:00000200:1.0:1616513073.914725:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513073.914727:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:2.0:1616513073.914767:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513073.914775:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513073.914778:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.914784:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513073.914791:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bfa880 x1693067725924368/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.2:1616513073.914800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:2.0:1616513073.914803:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3b210 00000800:00000200:2.0:1616513073.914808:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:1.0:1616513073.914809:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:2.0:1616513073.914811:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:1.0:1616513073.914817:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513073.914819:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:25.2:1616513073.914822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000400:00000200:1.0:1616513073.914828:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513073.914837:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626800 00000400:00000200:1.0:1616513073.914845:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff9b5 [1] + 192 00000400:00000200:1.0:1616513073.914850:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513073.914855:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded3a80 x1693067725924352/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513073.914868:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513073.914871:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513073.914873:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:14.0:1616513073.914876:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000100:00000200:11.0:1616513073.914877:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded3a80 x1693067725924352/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513073.914885:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000100:00000200:11.0:1616513073.914890:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded3a80 x1693067725924352/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513073.914894:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:11.0:1616513073.914900:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a2a8 00000400:00000200:14.0:1616513073.914902:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626810 00000400:00000200:14.0:1616513073.914909:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c5ba9 [1] + 192 00000400:00000200:14.0:1616513073.914915:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.914920:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bfa880 x1693067725924368/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513073.914929:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513073.914936:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513073.914938:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513073.914940:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:11.0:1616513073.914941:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bfa880 x1693067725924368/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513073.914954:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bfa880 x1693067725924368/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513073.914963:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bc28 00000800:00000200:11.2:1616513073.921702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.921725:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.921734:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.921743:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.56@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.921753:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.56@tcp1 of length 224 into portal 26 MB=0x5ffdd40c63400 00000400:00000200:8.0:1616513073.921759:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.56@tcp1 of length 224/224 into md 0xf97c5519 [1] + 0 00000400:00000200:8.0:1616513073.921764:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241d38 00000400:00000200:8.0:1616513073.921767:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.921770:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513073.921777:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241d38 00000800:00000200:12.0:1616513073.921780:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.921780:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.921783:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.921787:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513073.921790:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513073.921790:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513073.921799:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.56@tcp1 : PUT - for me 00000400:00000200:12.0:1616513073.921808:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.56@tcp1 of length 224 into portal 12 MB=0x5ffdd40c63780 00000400:00000200:12.0:1616513073.921814:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.56@tcp1 of length 224/224 into md 0x865b9f31 [1] + 69176 00000400:00000200:12.0:1616513073.921820:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513073.921822:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513073.921829:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.921832:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.921834:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513073.921852:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700623139840 00010000:00000200:16.0:1616513073.921863:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688700623139840/t0(0) o400->a492b999-d0e8-f6e8-c6bc-aafb7821c5b4@10.31.142.56@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.921877:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700623139840, offset 192 00000400:00000200:16.0:1616513073.921884:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.56@tcp1 00000400:00000200:16.0:1616513073.921892:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.56@tcp1 routed destination 00000100:00000200:20.0:1616513073.921893:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700623140736 00010000:00000200:20.0:1616513073.921905:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57b180 x1688700623140736/t0(0) o400->3312979b-a1cb-7a83-61a0-8d49f63b4080@10.31.142.56@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513073.921907:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.56@tcp1(10.31.142.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.921914:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:20.0:1616513073.921919:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700623140736, offset 192 00000800:00000200:16.0:1616513073.921921:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.921924:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:20.0:1616513073.921926:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.56@tcp1 00000800:00000200:16.0:1616513073.921926:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.921931:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513073.921933:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.56@tcp1 routed destination 00000800:00000200:11.2:1616513073.921937:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513073.921939:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513073.921948:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.56@tcp1(10.31.142.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.921957:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.921962:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.921965:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.921967:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.921972:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.921979:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513073.921989:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.921996:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.921999:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.922000:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.922006:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.922012:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42660 00000800:00000200:28.0:1616513073.922017:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513073.922033:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.922040:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.922042:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.922048:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.922052:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23ed0 00000800:00000200:2.0:1616513073.922065:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.922068:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.943667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.943680:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.943688:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513073.943696:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.60@tcp1 : PUT - for me 00000400:00000200:22.0:1616513073.943705:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.60@tcp1 of length 224 into portal 26 MB=0x601845daf8640 00000400:00000200:22.0:1616513073.943711:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.60@tcp1 of length 224/224 into md 0xf97c5539 [1] + 0 00000400:00000200:22.0:1616513073.943716:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240f68 00000400:00000200:22.0:1616513073.943719:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513073.943722:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513073.943725:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240f68 00000800:00000200:22.0:1616513073.943728:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.943731:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.943732:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513073.943736:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.943787:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517879359040 00010000:00000200:16.0:1616513073.943797:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1690517879359040/t0(0) o400->1bad8a4f-587a-dfbe-6ff6-c3bf56504386@10.31.141.60@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.943810:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690517879359040, offset 192 00000400:00000200:16.0:1616513073.943816:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.60@tcp1 00000400:00000200:16.0:1616513073.943823:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.60@tcp1 routed destination 00000400:00000200:16.0:1616513073.943842:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.60@tcp1(10.31.141.60@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.943848:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.943854:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.943856:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.943858:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.943863:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513073.943869:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.943871:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.943925:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.943933:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.943935:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513073.943942:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513073.943947:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ff0 00000800:00000200:12.0:1616513073.943950:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.943953:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.944199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.944254:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.944263:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.944273:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.60@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.944281:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.60@tcp1 of length 224 into portal 12 MB=0x601845daf9240 00000400:00000200:8.0:1616513073.944288:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.60@tcp1 of length 224/224 into md 0x865b9f31 [1] + 69400 00000400:00000200:8.0:1616513073.944293:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.944295:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513073.944302:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.944305:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.944307:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.944310:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.944372:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517879362112 00010000:00000200:20.0:1616513073.944385:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0480 x1690517879362112/t0(0) o400->0ce8e909-f2f5-e4e3-9fbf-da642260110e@10.31.141.60@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.944399:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690517879362112, offset 192 00000400:00000200:20.0:1616513073.944406:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.60@tcp1 00000400:00000200:20.0:1616513073.944413:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.60@tcp1 routed destination 00000400:00000200:20.0:1616513073.944428:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.60@tcp1(10.31.141.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.944448:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.944454:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.944457:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.944459:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.944464:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.944469:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.944523:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.944530:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.944533:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.944539:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.944543:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee224c8 00000800:00000200:2.0:1616513073.944546:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.944549:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.945878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513073.945931:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.945940:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513073.945949:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.52@tcp1 : PUT - for me 00000400:00000200:12.0:1616513073.945958:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.52@tcp1 of length 224 into portal 12 MB=0x5f90848446d40 00000400:00000200:12.0:1616513073.945965:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.52@tcp1 of length 224/224 into md 0x865b9f31 [1] + 69624 00000400:00000200:12.0:1616513073.945970:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513073.945973:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513073.945979:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.945982:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513073.945984:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.945987:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.946039:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681188851051840 00000800:00000200:7.2:1616513073.946043:0:48800:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:20.0:1616513073.946051:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57cc80 x1681188851051840/t0(0) o400->4f072141-083e-9cbc-5fa9-a8aaf90e25dd@10.31.131.52@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.946064:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681188851051840, offset 192 00000400:00000200:20.0:1616513073.946069:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.52@tcp1 00000400:00000200:20.0:1616513073.946076:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.52@tcp1 routed destination 00000400:00000200:20.0:1616513073.946090:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.52@tcp1(10.31.131.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.946096:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.946101:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.946104:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513073.946105:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.946106:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.946110:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513073.946115:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.946115:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513073.946115:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513073.946124:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.52@tcp1 : PUT - for me 00000800:00000200:8.0:1616513073.946126:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513073.946133:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.52@tcp1 of length 224 into portal 26 MB=0x5f90848446cc0 00000800:00000200:8.0:1616513073.946133:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.946135:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513073.946139:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.52@tcp1 of length 224/224 into md 0xf97c5559 [1] + 0 00000400:00000200:8.0:1616513073.946142:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513073.946144:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240bb0 00000400:00000200:8.0:1616513073.946146:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22b28 00000400:00000200:14.0:1616513073.946147:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.946149:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513073.946150:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513073.946153:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240bb0 00000800:00000200:8.0:1616513073.946153:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513073.946156:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.946159:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.946161:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.946221:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681188851051712 00010000:00000200:16.0:1616513073.946232:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1681188851051712/t0(0) o400->1c2ef9ca-137e-5fb1-721d-69e588985574@10.31.131.52@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.946247:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681188851051712, offset 192 00000400:00000200:16.0:1616513073.946253:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.52@tcp1 00000400:00000200:16.0:1616513073.946261:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.52@tcp1 routed destination 00000400:00000200:16.0:1616513073.946276:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.52@tcp1(10.31.131.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.946282:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513073.946288:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.946291:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.946293:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513073.946298:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.946306:0:48800:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513073.946307:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513073.946360:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.946367:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.946369:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.946376:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.946382:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a437e8 00000800:00000200:2.0:1616513073.946385:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.946388:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.953329:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.953384:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.953393:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.953402:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.20@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.953412:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.20@tcp1 of length 224 into portal 26 MB=0x5ffe641cddf00 00000400:00000200:24.0:1616513073.953419:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.20@tcp1 of length 224/224 into md 0xf97c5569 [1] + 0 00000400:00000200:24.0:1616513073.953424:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240908 00000400:00000200:24.0:1616513073.953426:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.953431:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513073.953449:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240908 00000800:00000200:12.0:1616513073.953450:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.953452:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.953455:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513073.953457:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.953460:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.953460:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513073.953468:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.20@tcp1 : PUT - for me 00000400:00000200:12.0:1616513073.953479:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.20@tcp1 of length 224 into portal 12 MB=0x5ffe641cde280 00000400:00000200:12.0:1616513073.953485:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.20@tcp1 of length 224/224 into md 0x865b9f31 [1] + 69848 00000400:00000200:12.0:1616513073.953490:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513073.953493:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513073.953499:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.953503:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.953504:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513073.953521:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739295125248 00010000:00000200:16.0:1616513073.953533:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688739295125248/t0(0) o400->a0a1c4ce-0379-2198-9194-c4e96c4dea3d@10.31.142.20@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.953547:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688739295125248, offset 192 00000400:00000200:16.0:1616513073.953553:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.20@tcp1 00000400:00000200:16.0:1616513073.953561:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.20@tcp1 routed destination 00000100:00000200:20.0:1616513073.953571:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739295126144 00000400:00000200:16.0:1616513073.953575:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.20@tcp1(10.31.142.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.953582:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:20.0:1616513073.953583:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc579200 x1688739295126144/t0(0) o400->70e0eb3d-5110-1706-c249-dccd632f56ab@10.31.142.20@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513073.953587:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.953590:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.953592:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513073.953596:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688739295126144, offset 192 00000800:00000200:16.0:1616513073.953597:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513073.953602:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.20@tcp1 00000800:00000200:11.2:1616513073.953602:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513073.953604:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513073.953609:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.20@tcp1 routed destination 00000400:00000200:20.0:1616513073.953623:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.20@tcp1(10.31.142.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.953629:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513073.953634:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.953637:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513073.953639:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513073.953643:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.953649:0:48801:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513073.953655:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.953662:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.953665:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513073.953666:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.953671:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.953677:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42880 00000800:00000200:28.0:1616513073.953680:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513073.953704:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.953710:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513073.953713:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513073.953718:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513073.953722:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee236d8 00000800:00000200:22.0:1616513073.953725:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513073.953728:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.953964:0:48801:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.954017:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.954027:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.954036:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.245.87.15@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.954045:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.245.87.15@tcp1 of length 520 into portal 26 MB=0x5e2b7cb9935f0 00000400:00000200:2.0:1616513073.954052:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.245.87.15@tcp1 of length 520/520 into md 0xf97c5579 [1] + 0 00000400:00000200:2.0:1616513073.954056:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241870 00000400:00000200:2.0:1616513073.954059:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.954062:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.954066:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241870 00000800:00000200:26.0:1616513073.954070:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.954070:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.954072:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.954074:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.954080:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1656653906261488 00010000:02000400:16.0:1616513073.954094:0:245737:0:(ldlm_lib.c:1150:target_handle_connect()) MGS: Received new LWP connection from 10.245.87.15@tcp1, removing former export from same NID 00010000:00080000:16.0:1616513073.954099:0:245737:0:(ldlm_lib.c:1226:target_handle_connect()) MGS: connection from fb85350a-ecab-8943-32ab-86d318bf89c1@10.245.87.15@tcp1 t0 exp ffff8d5dc46c8400 cur 1616513073 last 1616513073 00000400:00000200:16.0:1616513073.954112:0:245737:0:(peer.c:1109:LNetPrimaryNID()) NID 10.245.87.15@tcp1 primary NID 10.245.87.15@tcp1 rc 0 00000100:00080000:16.0:1616513073.954117:0:245737:0:(import.c:86:import_set_state_nolock()) ffff8d746a2b8800 : changing import state from FULL to RECOVER 00000100:00080000:16.0:1616513073.954123:0:245737:0:(import.c:86:import_set_state_nolock()) ffff8d746a2b8800 : changing import state from RECOVER to FULL 00000100:02000000:16.0:1616513073.954127:0:245737:0:(import.c:1595:ptlrpc_import_recovery_state_machine()) MGS: Connection restored to (at 10.245.87.15@tcp1) 00010000:00000200:16.0:1616513073.954133:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1656653906261488/t0(0) o250->fb85350a-ecab-8943-32ab-86d318bf89c1@10.245.87.15@tcp1:0/0 lens 520/416 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.954145:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 384 bytes to portal 25, xid 1656653906261488, offset 0 00000400:00000200:16.0:1616513073.954150:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.245.87.15@tcp1 00000400:00000200:16.0:1616513073.954155:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.245.87.15@tcp1 routed destination 00000400:00000200:16.0:1616513073.954168:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.245.87.15@tcp1(10.245.87.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.954174:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 384 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.954179:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.954184:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.954185:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.954188:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513073.954193:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.954193:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.954204:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.954211:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.954214:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513073.954215:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513073.954221:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.954226:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43650 00000800:00000200:8.0:1616513073.954230:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.955004:0:48801:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.955059:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.955068:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513073.955077:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.245.87.15@tcp1 : PUT - for me 00000400:00000200:22.0:1616513073.955086:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.245.87.15@tcp1 of length 328 into portal 26 MB=0x5e2b7cb993770 00000400:00000200:22.0:1616513073.955092:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.245.87.15@tcp1 of length 328/328 into md 0xf97c55a9 [1] + 0 00000400:00000200:22.0:1616513073.955097:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782413a8 00000400:00000200:22.0:1616513073.955100:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513073.955102:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513073.955107:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782413a8 00000800:00000200:22.0:1616513073.955110:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.955112:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513073.955113:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513073.955116:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.955118:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1656653906261872 00010000:00000200:16.0:1616513073.955142:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1656653906261872/t0(0) o101->fb85350a-ecab-8943-32ab-86d318bf89c1@10.245.87.15@tcp1:30/0 lens 328/344 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.955152:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 312 bytes to portal 25, xid 1656653906261872, offset 192 00000400:00000200:16.0:1616513073.955157:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.245.87.15@tcp1 00000400:00000200:16.0:1616513073.955162:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.245.87.15@tcp1 routed destination 00000400:00000200:16.0:1616513073.955175:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.245.87.15@tcp1(10.245.87.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.955180:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 312 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513073.955184:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.955187:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.955188:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513073.955192:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513073.955196:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513073.955197:0:48801:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513073.955287:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.955294:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.955304:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.955310:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.955316:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43e48 00000800:00000200:26.0:1616513073.955319:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.955322:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.955822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.955880:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.955895:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.955904:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.245.87.15@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.955913:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.245.87.15@tcp1 of length 288 into portal 26 MB=0x5e2b7cb993780 00000400:00000200:24.0:1616513073.955919:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.245.87.15@tcp1 of length 288/288 into md 0xf97c55c9 [1] + 0 00000400:00000200:24.0:1616513073.955924:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241760 00000400:00000200:24.0:1616513073.955927:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.955929:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513073.955934:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241760 00000800:00000200:24.0:1616513073.955937:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.955939:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.955940:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.955943:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513073.955943:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1656653906261888 00010000:00000200:16.0:1616513073.955978:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1656653906261888/t0(0) o501->fb85350a-ecab-8943-32ab-86d318bf89c1@10.245.87.15@tcp1:30/0 lens 288/272 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513073.955988:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 240 bytes to portal 25, xid 1656653906261888, offset 192 00000400:00000200:16.0:1616513073.955992:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.245.87.15@tcp1 00000400:00000200:16.0:1616513073.955996:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.245.87.15@tcp1 routed destination 00000400:00000200:16.0:1616513073.956008:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.245.87.15@tcp1(10.245.87.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.956012:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 240 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513073.956016:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.956018:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513073.956019:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513073.956022:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513073.956026:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.956027:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.956080:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.956086:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.956089:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513073.956095:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513073.956100:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a90 00000800:00000200:12.0:1616513073.956104:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.956106:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.956709:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.956772:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.956794:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.956802:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.245.87.15@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.956810:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.245.87.15@tcp1 of length 272 into portal 26 MB=0x5e2b7cb993790 00000400:00000200:14.0:1616513073.956816:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.245.87.15@tcp1 of length 272/272 into md 0xf97c55e9 [1] + 0 00000400:00000200:14.0:1616513073.956820:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241ed0 00000400:00000200:14.0:1616513073.956823:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.956825:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:11.2:1616513073.956826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513073.956830:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241ed0 00000800:00000200:22.0:1616513073.956833:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.956833:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513073.956835:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513073.956837:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513073.956837:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513073.956839:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1656653906261904 00000800:00000200:28.0:1616513073.956845:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513073.956847:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.956852:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513073.956854:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.166@tcp1 : PUT - for me 00000800:00000200:12.0:1616513073.956855:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513073.956858:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.166@tcp1 : PUT - for me 00010000:00000200:16.0:1616513073.956863:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1656653906261904/t0(0) o503->fb85350a-ecab-8943-32ab-86d318bf89c1@10.245.87.15@tcp1:30/0 lens 272/8416 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513073.956864:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.166@tcp1 of length 224 into portal 26 MB=0x5ffde05ce81c0 00000400:00000200:24.0:1616513073.956865:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.166@tcp1 of length 224 into portal 12 MB=0x5ffde05ce8240 00000400:00000200:28.0:1616513073.956870:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.166@tcp1 of length 224/224 into md 0xf97c55f9 [1] + 0 00000400:00000200:24.0:1616513073.956871:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.166@tcp1 of length 224/224 into md 0x865b9f31 [1] + 70072 00000400:00000200:24.0:1616513073.956874:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513073.956874:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 8384 bytes to portal 25, xid 1656653906261904, offset 192 00000400:00000200:28.0:1616513073.956875:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241540 00000100:00000200:24.0:1616513073.956876:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513073.956878:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.956881:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513073.956881:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.956883:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513073.956883:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.245.87.15@tcp1 00000800:00000200:24.0:1616513073.956884:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513073.956886:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241540 00000400:00000200:16.0:1616513073.956887:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.245.87.15@tcp1 routed destination 00000800:00000200:28.0:1616513073.956891:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.956894:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.956895:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:16.0:1616513073.956898:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.245.87.15@tcp1(10.245.87.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513073.956903:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 8384 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513073.956909:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513073.956911:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513073.956912:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.956913:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703928795584 00000800:00000200:16.0:1616513073.956915:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.956920:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513073.956921:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:15.0:1616513073.956926:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688703928795584/t0(0) o400->6f9c5ab3-ad73-9fc6-2e1b-b41612523527@10.31.132.166@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.956941:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703928795584, offset 192 00000100:00000200:20.0:1616513073.956943:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703928795712 00000400:00000200:15.0:1616513073.956949:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.166@tcp1 00010000:00000200:20.0:1616513073.956955:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc6780 x1688703928795712/t0(0) o400->abf4a986-ad0f-c73e-b326-48f2b7b57cf1@10.31.132.166@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513073.956957:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.166@tcp1 routed destination 00000100:00000200:20.0:1616513073.956968:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703928795712, offset 192 00000800:00000200:26.0:1616513073.956974:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513073.956974:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.166@tcp1 00000400:00000200:15.0:1616513073.956974:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.166@tcp1(10.31.132.166@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513073.956980:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:15.0:1616513073.956981:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513073.956982:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513073.956983:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.166@tcp1 routed destination 00000800:00000200:15.0:1616513073.956988:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:7.2:1616513073.956988:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.956990:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513073.956993:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513073.956998:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.166@tcp1(10.31.132.166@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.956998:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513073.956999:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513073.957003:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.957004:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513073.957007:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.242.46.216@o2ib1 00000100:00000200:15.0:1616513073.957007:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513073.957008:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513073.957010:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513073.957010:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.957013:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.957013:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513073.957015:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513073.957015:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:22.0:1616513073.957017:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513073.957020:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:8.0:1616513073.957021:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:7.2:1616513073.957022:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513073.957023:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:8.0:1616513073.957023:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513073.957027:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:12.0:1616513073.957028:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513073.957028:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:8.0:1616513073.957030:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513073.957031:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.957033:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.957035:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775f58 00000800:00000200:26.0:1616513073.957036:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:12.0:1616513073.957036:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23a08 00000800:00000200:2.0:1616513073.957037:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513073.957038:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513073.957039:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.957041:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513073.957045:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.957052:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ba0 00000800:00000200:2.0:1616513073.957056:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.958305:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513073.958358:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.958367:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513073.958377:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.245.87.15@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.958385:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.245.87.15@tcp1 of length 328 into portal 26 MB=0x5e2b7cb9937a0 00000400:00000200:28.0:1616513073.958392:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.245.87.15@tcp1 of length 328/328 into md 0xf97c5609 [1] + 0 00000400:00000200:28.0:1616513073.958397:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240330 00000400:00000200:28.0:1616513073.958399:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.958402:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513073.958407:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240330 00000800:00000200:28.0:1616513073.958410:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.958413:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.958414:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513073.958417:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513073.958470:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1656653906261920 00010000:00000200:15.0:1616513073.958501:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1656653906261920/t0(0) o101->fb85350a-ecab-8943-32ab-86d318bf89c1@10.245.87.15@tcp1:30/0 lens 328/344 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.958515:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 312 bytes to portal 25, xid 1656653906261920, offset 192 00000400:00000200:15.0:1616513073.958521:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.245.87.15@tcp1 00000400:00000200:15.0:1616513073.958529:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.245.87.15@tcp1 routed destination 00000400:00000200:15.0:1616513073.958544:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.245.87.15@tcp1(10.245.87.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.958551:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 312 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.958557:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.958560:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.958562:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.958568:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.958572:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.958575:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513073.958623:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513073.958632:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513073.958634:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513073.958641:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513073.958647:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775b18 00000800:00000200:22.0:1616513073.958651:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513073.958653:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.959182:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.959236:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.959245:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.959254:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.245.87.15@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.959262:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.245.87.15@tcp1 of length 304 into portal 26 MB=0x5e2b7cb9937b0 00000400:00000200:14.0:1616513073.959269:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.245.87.15@tcp1 of length 304/304 into md 0xf97c5639 [1] + 0 00000400:00000200:14.0:1616513073.959274:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241078 00000400:00000200:14.0:1616513073.959276:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.959280:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513073.959285:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241078 00000800:00000200:14.0:1616513073.959288:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.959288:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.959291:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513073.959292:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.959299:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1656653906261936 00010000:00000200:15.0:1616513073.959324:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1656653906261936/t0(0) o501->fb85350a-ecab-8943-32ab-86d318bf89c1@10.245.87.15@tcp1:30/0 lens 304/272 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc -2/0 00000100:00000200:15.0:1616513073.959337:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 240 bytes to portal 25, xid 1656653906261936, offset 192 00000400:00000200:15.0:1616513073.959342:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.245.87.15@tcp1 00000400:00000200:15.0:1616513073.959347:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.245.87.15@tcp1 routed destination 00000400:00000200:15.0:1616513073.959361:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.245.87.15@tcp1(10.245.87.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.959366:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 240 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513073.959373:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.959375:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513073.959377:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513073.959381:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513073.959386:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513073.959387:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.959449:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.959466:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.959470:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513073.959476:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513073.959482:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774088 00000800:00000200:12.0:1616513073.959485:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.959488:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.966869:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513073.966925:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.966934:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513073.966943:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.239@tcp1 : PUT - for me 00000400:00000200:22.0:1616513073.966952:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.239@tcp1 of length 224 into portal 26 MB=0x5ffd43f04ae40 00000400:00000200:22.0:1616513073.966958:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.239@tcp1 of length 224/224 into md 0xf97c5649 [1] + 0 00000400:00000200:22.0:1616513073.966963:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782414b8 00000400:00000200:22.0:1616513073.966966:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513073.966969:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513073.966973:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782414b8 00000800:00000200:22.0:1616513073.966976:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.966978:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513073.966979:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513073.966982:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.967037:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661938974272 00010000:00000200:15.0:1616513073.967048:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688661938974272/t0(0) o400->0d25ae35-695f-608d-d2f6-c39bef81ea34@10.31.132.239@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.967062:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661938974272, offset 192 00000400:00000200:15.0:1616513073.967069:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.239@tcp1 00000400:00000200:15.0:1616513073.967077:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.239@tcp1 routed destination 00000400:00000200:15.0:1616513073.967092:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.239@tcp1(10.31.132.239@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.967098:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.967104:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.967107:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.967109:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.967114:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.967119:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.967121:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513073.967171:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.967178:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513073.967181:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513073.967187:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513073.967193:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774330 00000800:00000200:2.0:1616513073.967197:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.967199:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.967277:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.967341:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.967361:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.967370:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.239@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.967378:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.239@tcp1 of length 224 into portal 12 MB=0x5ffd43f04b000 00000400:00000200:24.0:1616513073.967384:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.239@tcp1 of length 224/224 into md 0x865b9f31 [1] + 70296 00000400:00000200:24.0:1616513073.967389:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.967392:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513073.967397:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.967400:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.967401:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.967404:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.967461:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661938974720 00010000:00000200:20.0:1616513073.967472:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc1680 x1688661938974720/t0(0) o400->f8828f70-e209-4c13-88b2-24d7ab6d0d93@10.31.132.239@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.967486:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661938974720, offset 192 00000400:00000200:20.0:1616513073.967492:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.239@tcp1 00000400:00000200:20.0:1616513073.967499:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.239@tcp1 routed destination 00000400:00000200:20.0:1616513073.967514:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.239@tcp1(10.31.132.239@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.967521:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.967526:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.967529:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.967531:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.967535:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.967540:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513073.967593:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.967599:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.967602:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513073.967605:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513073.967608:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513073.967612:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee238f8 00000800:00000200:12.0:1616513073.967617:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.971026:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513073.971079:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.971088:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.971096:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.116@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.971105:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.116@tcp1 of length 224 into portal 26 MB=0x5ffde0bbcfc40 00000400:00000200:24.0:1616513073.971111:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.116@tcp1 of length 224/224 into md 0xf97c5659 [1] + 0 00000400:00000200:24.0:1616513073.971116:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240088 00000400:00000200:24.0:1616513073.971118:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.971121:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513073.971126:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240088 00000800:00000200:24.0:1616513073.971128:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513073.971131:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513073.971132:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.971135:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513073.971187:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704028310592 00010000:00000200:15.0:1616513073.971198:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688704028310592/t0(0) o400->b9094fbc-0f19-afcb-d2f4-64cc633f3c54@10.31.142.116@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.971211:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704028310592, offset 192 00000400:00000200:15.0:1616513073.971217:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.116@tcp1 00000400:00000200:15.0:1616513073.971225:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.116@tcp1 routed destination 00000400:00000200:15.0:1616513073.971239:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.116@tcp1(10.31.142.116@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.971245:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.971251:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.971254:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.971256:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.971260:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.971265:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.971268:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513073.971318:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.971325:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.971328:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.971335:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.971342:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774ee0 00000800:00000200:26.0:1616513073.971345:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.971348:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513073.973143:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513073.973197:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.973207:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513073.973215:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.116@tcp1 : PUT - for me 00000400:00000200:8.0:1616513073.973224:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.116@tcp1 of length 224 into portal 12 MB=0x5ffde0bbd0080 00000400:00000200:8.0:1616513073.973230:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.116@tcp1 of length 224/224 into md 0x865b9f31 [1] + 70520 00000400:00000200:8.0:1616513073.973236:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513073.973238:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513073.973245:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.973248:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513073.973250:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513073.973252:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513073.973310:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704028311680 00010000:00000200:20.0:1616513073.973323:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6c00 x1688704028311680/t0(0) o400->7041f681-1641-5775-ea34-30746c810903@10.31.142.116@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513073.973337:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704028311680, offset 192 00000400:00000200:20.0:1616513073.973343:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.116@tcp1 00000400:00000200:20.0:1616513073.973351:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.116@tcp1 routed destination 00000400:00000200:20.0:1616513073.973367:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.116@tcp1(10.31.142.116@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.973373:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.973379:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.973382:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.973384:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.973389:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.973395:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.973458:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.973464:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513073.973467:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513073.973473:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513073.973477:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee227f8 00000800:00000200:28.0:1616513073.973480:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513073.973483:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.987781:0:48805:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.987836:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.987845:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.987854:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.53@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.987864:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.53@tcp1 of length 224 into portal 26 MB=0x5ffd493f69800 00000400:00000200:2.0:1616513073.987870:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.53@tcp1 of length 224/224 into md 0xf97c5689 [1] + 0 00000400:00000200:2.0:1616513073.987875:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241650 00000400:00000200:2.0:1616513073.987878:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.987881:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.987887:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241650 00000800:00000200:14.0:1616513073.987890:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.987890:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.987892:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.987894:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.987906:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663364114432 00010000:00000200:15.0:1616513073.987916:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688663364114432/t0(0) o400->344d2eb8-2e9c-193f-7eb3-4c0e1288eb45@10.31.138.53@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.987928:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663364114432, offset 192 00000400:00000200:15.0:1616513073.987934:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.53@tcp1 00000400:00000200:15.0:1616513073.987942:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.53@tcp1 routed destination 00000400:00000200:15.0:1616513073.987955:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.53@tcp1(10.31.138.53@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.987961:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.987967:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.987969:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.987971:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.987976:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.987980:0:48805:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.987981:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513073.988033:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.988041:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.988044:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.988050:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.988056:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77747f8 00000800:00000200:26.0:1616513073.988060:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.988063:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.988067:0:48805:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.988079:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513073.988087:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.988087:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513073.988096:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.53@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.988106:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.53@tcp1 of length 224 into portal 12 MB=0x5ffd493f69940 00000400:00000200:14.0:1616513073.988112:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.53@tcp1 of length 224/224 into md 0xf96450b9 [1] + 36864 00000400:00000200:14.0:1616513073.988118:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.988120:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.988130:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.988133:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.988135:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513073.988196:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663364114752 00010000:00000200:30.0:1616513073.988208:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cf980 x1688663364114752/t0(0) o400->5e9d9516-d614-a2da-0ca3-d8ae8869ed09@10.31.138.53@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513073.988223:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663364114752, offset 192 00000400:00000200:30.0:1616513073.988230:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.53@tcp1 00000400:00000200:30.0:1616513073.988237:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.53@tcp1 routed destination 00000400:00000200:30.0:1616513073.988252:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.53@tcp1(10.31.138.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513073.988259:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513073.988265:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513073.988267:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513073.988269:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513073.988274:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.988281:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.988336:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.988343:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.988346:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.988352:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.988356:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeaa0 00000800:00000200:8.0:1616513073.988361:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.988363:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513073.992774:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513073.992830:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513073.992838:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513073.992846:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.167@tcp1 : PUT - for me 00000400:00000200:28.0:1616513073.992856:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.167@tcp1 of length 224 into portal 26 MB=0x5ffd4a5483880 00000400:00000200:28.0:1616513073.992862:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.167@tcp1 of length 224/224 into md 0xf97c5699 [1] + 0 00000400:00000200:28.0:1616513073.992867:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241a08 00000400:00000200:28.0:1616513073.992869:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513073.992872:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513073.992877:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a08 00000800:00000200:28.0:1616513073.992879:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513073.992882:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513073.992882:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513073.992888:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513073.992891:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.992891:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513073.992899:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.167@tcp1 : PUT - for me 00000400:00000200:24.0:1616513073.992908:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.167@tcp1 of length 224 into portal 12 MB=0x5ffd4a5483900 00000400:00000200:24.0:1616513073.992914:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.167@tcp1 of length 224/224 into md 0x865b9f31 [1] + 70744 00000400:00000200:24.0:1616513073.992919:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513073.992921:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513073.992930:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513073.992933:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513073.992935:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513073.992943:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663654676608 00010000:00000200:15.0:1616513073.992952:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688663654676608/t0(0) o400->b7db667a-5779-a8b2-3462-ce9fb9df9d00@10.31.136.167@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.992965:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663654676608, offset 192 00000400:00000200:15.0:1616513073.992970:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.167@tcp1 00000400:00000200:15.0:1616513073.992978:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.167@tcp1 routed destination 00000100:00000200:20.0:1616513073.992988:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663654676736 00000400:00000200:15.0:1616513073.992991:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.167@tcp1(10.31.136.167@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.992997:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:20.0:1616513073.992999:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc7500 x1688663654676736/t0(0) o400->43c56fe8-2735-9cd0-1ef4-767701a88639@10.31.136.167@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513073.993002:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.993005:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.993007:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513073.993011:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663654676736, offset 192 00000800:00000200:15.0:1616513073.993011:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513073.993017:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.167@tcp1 00000800:00000200:7.2:1616513073.993017:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.993018:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513073.993024:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.167@tcp1 routed destination 00000400:00000200:20.0:1616513073.993038:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.167@tcp1(10.31.136.167@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513073.993043:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513073.993049:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513073.993051:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513073.993055:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513073.993059:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.993064:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513073.993070:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.993077:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.993080:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.993087:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.993093:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774880 00000800:00000200:26.0:1616513073.993096:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.993099:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513073.993116:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513073.993123:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513073.993126:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513073.993127:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513073.993133:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513073.993136:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23ba0 00000800:00000200:12.0:1616513073.993140:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513073.995981:0:48806:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513073.996036:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513073.996045:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513073.996054:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.124@tcp1 : PUT - for me 00000400:00000200:2.0:1616513073.996064:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.124@tcp1 of length 224 into portal 26 MB=0x604a3623a21c0 00000400:00000200:2.0:1616513073.996071:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.124@tcp1 of length 224/224 into md 0xf97c56c9 [1] + 0 00000400:00000200:2.0:1616513073.996075:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240880 00000400:00000200:2.0:1616513073.996078:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513073.996081:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513073.996087:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240880 00000800:00000200:14.0:1616513073.996089:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513073.996090:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513073.996092:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513073.996094:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513073.996106:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949634421184 00010000:00000200:15.0:1616513073.996116:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1693949634421184/t0(0) o400->fb56d8a0-f7ce-e68b-5d79-991ef6371ded@10.31.142.124@tcp1:30/0 lens 224/224 e 0 to 0 dl 1616513165 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513073.996128:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949634421184, offset 192 00000400:00000200:15.0:1616513073.996135:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.124@tcp1 00000400:00000200:15.0:1616513073.996143:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.124@tcp1 routed destination 00000400:00000200:15.0:1616513073.996157:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.124@tcp1(10.31.142.124@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513073.996163:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513073.996168:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513073.996171:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513073.996173:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513073.996177:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513073.996182:0:48806:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513073.996183:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513073.996236:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513073.996243:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513073.996247:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513073.996253:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513073.996260:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775ba0 00000800:00000200:26.0:1616513073.996263:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513073.996266:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513073.996718:0:48806:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513073.996730:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.996738:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513073.996740:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513073.996747:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.124@tcp1 : PUT - for me 00000400:00000200:14.0:1616513073.996756:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.124@tcp1 of length 224 into portal 12 MB=0x604a3623a2480 00000400:00000200:14.0:1616513073.996763:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.124@tcp1 of length 224/224 into md 0xf96450b9 [1] + 37088 00000400:00000200:14.0:1616513073.996768:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513073.996770:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513073.996777:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513073.996780:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513073.996781:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513073.996840:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949634421888 00010000:00000200:30.0:1616513073.996852:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cc380 x1693949634421888/t0(0) o400->93e86fd7-1a42-3643-39f0-0299740c53f6@10.31.142.124@tcp1:699/0 lens 224/224 e 0 to 0 dl 1616513079 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513073.996866:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949634421888, offset 192 00000400:00000200:30.0:1616513073.996873:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.124@tcp1 00000400:00000200:30.0:1616513073.996881:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.124@tcp1 routed destination 00000400:00000200:30.0:1616513073.996896:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.124@tcp1(10.31.142.124@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513073.996903:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513073.996909:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513073.996911:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513073.996913:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513073.996918:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513073.996923:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513073.996976:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513073.996983:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513073.996985:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513073.996992:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513073.996996:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efa08 00000800:00000200:8.0:1616513073.997000:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513073.997003:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.013284:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.013339:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.013348:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513074.013357:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.218@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.013367:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.218@tcp1 of length 224 into portal 26 MB=0x604f5f6337d40 00000400:00000200:28.0:1616513074.013374:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.218@tcp1 of length 224/224 into md 0xf97c56d9 [1] + 0 00000400:00000200:28.0:1616513074.013378:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241298 00000400:00000200:28.0:1616513074.013381:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.013384:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513074.013389:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241298 00000800:00000200:28.0:1616513074.013392:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.013394:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.013396:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.013398:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.013409:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304304332096 00010000:00000200:15.0:1616513074.013418:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1694304304332096/t0(0) o400->4d864bc3-b06d-b62c-b828-d7c17c3a1560@10.31.140.218@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513074.013429:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513074.013430:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694304304332096, offset 192 00000400:00000200:15.0:1616513074.013445:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.218@tcp1 00000400:00000200:15.0:1616513074.013452:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.218@tcp1 routed destination 00000400:00000200:15.0:1616513074.013464:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.218@tcp1(10.31.140.218@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.013470:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.013475:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.013478:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.013480:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.013484:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513074.013490:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.013490:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513074.013501:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.013511:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.013519:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.218@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.013528:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.218@tcp1 of length 224 into portal 12 MB=0x604f5f6337e80 00000400:00000200:8.0:1616513074.013534:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.218@tcp1 of length 224/224 into md 0x865b9f31 [1] + 70968 00000400:00000200:8.0:1616513074.013539:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.013542:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513074.013544:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513074.013550:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513074.013552:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:8.0:1616513074.013554:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513074.013555:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.013555:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.013558:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513074.013563:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.013569:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775760 00000800:00000200:26.0:1616513074.013573:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.013576:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.013616:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304304332416 00010000:00000200:20.0:1616513074.013628:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5a00 x1694304304332416/t0(0) o400->bb661ba6-79b0-e903-d77e-005947123f16@10.31.140.218@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.013643:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694304304332416, offset 192 00000400:00000200:20.0:1616513074.013649:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.218@tcp1 00000400:00000200:20.0:1616513074.013657:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.218@tcp1 routed destination 00000400:00000200:20.0:1616513074.013673:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.218@tcp1(10.31.140.218@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.013679:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.013686:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.013688:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.013690:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.013695:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.013699:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.013710:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.013715:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513074.013718:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513074.013719:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513074.013724:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.013728:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23540 00000800:00000200:12.0:1616513074.013732:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.018172:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513074.018225:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.018234:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.018243:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.42@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.018252:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.42@tcp1 of length 224 into portal 12 MB=0x5f90fb91eb440 00000400:00000200:8.0:1616513074.018258:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.42@tcp1 of length 224/224 into md 0x865b9f31 [1] + 71192 00000400:00000200:8.0:1616513074.018263:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.018266:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513074.018272:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.018275:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.018277:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513074.018279:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.018334:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681220809176128 00010000:00000200:20.0:1616513074.018347:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3600 x1681220809176128/t0(0) o400->a48896ee-5d20-43f5-a7cc-7150037a4ac0@10.31.132.42@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.018361:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681220809176128, offset 192 00000400:00000200:20.0:1616513074.018368:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.42@tcp1 00000400:00000200:20.0:1616513074.018375:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.42@tcp1 routed destination 00000400:00000200:20.0:1616513074.018390:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.42@tcp1(10.31.132.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.018396:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.018402:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.018405:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.018407:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.018412:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.018418:0:48808:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.018477:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.018483:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.018486:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.018493:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.018497:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23210 00000800:00000200:2.0:1616513074.018501:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.018504:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.018955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.019008:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.019018:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.019027:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.42@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.019035:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.42@tcp1 of length 224 into portal 26 MB=0x5f90fb91eb3c0 00000400:00000200:26.0:1616513074.019043:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.42@tcp1 of length 224/224 into md 0xf97c56e9 [1] + 0 00000400:00000200:26.0:1616513074.019048:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241980 00000400:00000200:26.0:1616513074.019050:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.019053:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.019058:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241980 00000800:00000200:26.0:1616513074.019061:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.019062:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.019064:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.019066:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.019131:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681220809176000 00010000:00000200:15.0:1616513074.019153:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1681220809176000/t0(0) o400->55ed0d37-8354-68fd-749a-338719d28815@10.31.132.42@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.019164:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681220809176000, offset 192 00000400:00000200:15.0:1616513074.019169:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.42@tcp1 00000400:00000200:15.0:1616513074.019176:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.42@tcp1 routed destination 00000400:00000200:15.0:1616513074.019188:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.42@tcp1(10.31.132.42@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.019197:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.019203:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.019205:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.019207:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.019211:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.019216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.019217:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513074.019270:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.019277:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.019280:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513074.019287:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.019293:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775dc0 00000800:00000200:28.0:1616513074.019297:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.019300:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.021688:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.021742:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.021751:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.021760:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.212@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.021768:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.212@tcp1 of length 224 into portal 26 MB=0x601a9d3b93a80 00000400:00000200:2.0:1616513074.021775:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.212@tcp1 of length 224/224 into md 0xf97c5709 [1] + 0 00000400:00000200:2.0:1616513074.021780:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782405d8 00000400:00000200:2.0:1616513074.021782:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.021785:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513074.021790:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782405d8 00000800:00000200:2.0:1616513074.021793:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.021796:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.021796:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.021800:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.021839:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513074.021849:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690678773496448 00000800:00000200:8.0:1616513074.021852:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:15.0:1616513074.021858:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1690678773496448/t0(0) o400->ff5a996d-7e98-150a-7f05-4a5e349b9210@10.31.140.212@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:8.0:1616513074.021862:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:15.0:1616513074.021870:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690678773496448, offset 192 00000400:00000200:8.0:1616513074.021871:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.212@tcp1 : PUT - for me 00000400:00000200:15.0:1616513074.021875:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.212@tcp1 00000400:00000200:15.0:1616513074.021882:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.212@tcp1 routed destination 00000400:00000200:8.0:1616513074.021883:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.212@tcp1 of length 224 into portal 12 MB=0x601a9d3b93b00 00000400:00000200:8.0:1616513074.021890:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.212@tcp1 of length 224/224 into md 0x865b9f31 [1] + 71416 00000400:00000200:15.0:1616513074.021895:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.212@tcp1(10.31.140.212@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513074.021895:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.021898:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:15.0:1616513074.021901:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513074.021904:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:15.0:1616513074.021906:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513074.021907:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:15.0:1616513074.021908:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513074.021909:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:15.0:1616513074.021910:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513074.021912:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:15.0:1616513074.021914:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.021918:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.021919:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:20.0:1616513074.021968:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690678773496576 00000800:00000200:22.0:1616513074.021975:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:20.0:1616513074.021981:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5100 x1690678773496576/t0(0) o400->eb88c265-3c15-831d-f5db-deb76cc8ad6a@10.31.140.212@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:22.0:1616513074.021982:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.021986:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.021992:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:20.0:1616513074.021996:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690678773496576, offset 192 00000400:00000200:22.0:1616513074.021998:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77744c8 00000800:00000200:22.0:1616513074.022002:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513074.022004:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.212@tcp1 00000800:00000200:22.0:1616513074.022005:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513074.022012:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.212@tcp1 routed destination 00000400:00000200:20.0:1616513074.022027:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.212@tcp1(10.31.140.212@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.022033:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.022039:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.022042:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.022044:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.022049:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.022054:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.022065:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.022072:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.022075:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513074.022081:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.022085:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23b18 00000800:00000200:28.0:1616513074.022089:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.022092:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513074.026544:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924384, portal 4 00000100:00000200:26.0:1616513074.026550:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924384, offset 0 00000400:00000200:26.0:1616513074.026557:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:26.0:1616513074.026562:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:26.0:1616513074.026575:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513074.026582:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:26.0:1616513074.026589:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513074.026592:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:26.0:1616513074.026594:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:26.0:1616513074.026599:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513074.026647:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.026667:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513074.026672:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513074.026676:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513074.026683:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513074.026689:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d5a00 x1693067725924384/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513074.026701:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afb18 00000800:00000200:27.0:1616513074.026706:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513074.026708:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513074.026835:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513074.026844:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.026851:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513074.026854:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513074.026858:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513074.026866:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626820 00000400:00000200:27.0:1616513074.026872:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716574d [1] + 192 00000400:00000200:27.0:1616513074.026877:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513074.026881:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d5a00 x1693067725924384/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513074.026896:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.026899:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513074.026903:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:26.0:1616513074.026906:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d5a00 x1693067725924384/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513074.026919:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d5a00 x1693067725924384/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513074.026929:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:11.2:1616513074.037519:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.037573:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.037583:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513074.037592:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.252@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.037602:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.252@tcp1 of length 224 into portal 26 MB=0x5ffe5df06f9c0 00000400:00000200:28.0:1616513074.037609:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.252@tcp1 of length 224/224 into md 0xf97c5719 [1] + 0 00000400:00000200:28.0:1616513074.037614:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240d48 00000400:00000200:28.0:1616513074.037617:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.037619:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513074.037624:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240d48 00000800:00000200:28.0:1616513074.037627:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.037627:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.037630:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.037634:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.037637:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.037638:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.037647:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.252@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.037656:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.252@tcp1 of length 224 into portal 12 MB=0x5ffe5df06fcc0 00000400:00000200:8.0:1616513074.037662:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.252@tcp1 of length 224/224 into md 0x865b9f31 [1] + 71640 00000400:00000200:8.0:1616513074.037668:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.037670:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513074.037679:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.037682:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513074.037683:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737637923264 00000800:00000200:8.0:1616513074.037684:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:15.0:1616513074.037692:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688737637923264/t0(0) o400->86ae17a5-c688-8ee2-3a61-4842e9b54a69@10.31.142.252@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.037700:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737637924032 00000100:00000200:15.0:1616513074.037704:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688737637923264, offset 192 00000400:00000200:15.0:1616513074.037709:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.252@tcp1 00010000:00000200:20.0:1616513074.037712:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0000 x1688737637924032/t0(0) o400->e3ca7a1a-65c0-c3d8-c0d3-13871fb8c6be@10.31.142.252@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513074.037716:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.252@tcp1 routed destination 00000100:00000200:20.0:1616513074.037725:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688737637924032, offset 192 00000400:00000200:15.0:1616513074.037729:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.252@tcp1(10.31.142.252@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513074.037731:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.252@tcp1 00000800:00000200:15.0:1616513074.037734:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513074.037739:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.252@tcp1 routed destination 00000800:00000200:15.0:1616513074.037739:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.037742:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.037745:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.037750:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513074.037753:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.252@tcp1(10.31.142.252@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:15.0:1616513074.037756:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.037756:0:48809:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.037760:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.037765:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.037768:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.037770:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.037774:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.037780:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513074.037826:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.037833:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.037836:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.037843:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513074.037847:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513074.037849:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774d48 00000800:00000200:14.0:1616513074.037853:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513074.037854:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513074.037855:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513074.037857:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513074.037858:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513074.037864:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.037868:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22220 00000800:00000200:12.0:1616513074.037872:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.039184:0:48809:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.039237:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.039247:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.039255:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.039264:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x60399133c4a00 00000400:00000200:22.0:1616513074.039271:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0xf96450b9 [1] + 37312 00000400:00000200:22.0:1616513074.039276:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.039281:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513074.039287:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.039290:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.039291:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.039297:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513074.039352:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805847861760 00010000:00000200:30.0:1616513074.039429:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad4380 x1692805847861760/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:700/0 lens 336/336 e 0 to 0 dl 1616513080 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513074.039459:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805847861760, offset 192 00000400:00000200:30.0:1616513074.039465:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:30.0:1616513074.039472:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:30.0:1616513074.039489:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.039495:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513074.039501:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.039504:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513074.039506:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513074.039511:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.039515:0:48809:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.039569:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.039576:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.039579:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513074.039581:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.039585:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.039589:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efa08 00000800:00000200:14.0:1616513074.039593:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.041130:0:48809:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.041183:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.041193:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.041202:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.54@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.041211:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.54@tcp1 of length 224 into portal 26 MB=0x5ffe490f26c40 00000400:00000200:26.0:1616513074.041218:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.54@tcp1 of length 224/224 into md 0xf97c5729 [1] + 0 00000400:00000200:26.0:1616513074.041222:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240ff0 00000400:00000200:26.0:1616513074.041225:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.041228:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.041233:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ff0 00000800:00000200:26.0:1616513074.041236:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.041236:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.041239:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.041240:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.041285:0:48809:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513074.041290:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688732032986176 00010000:00000200:15.0:1616513074.041299:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688732032986176/t0(0) o400->ce31160e-a90d-5e9b-8ee6-7d64444c3c4c@10.31.140.54@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.041311:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688732032986176, offset 192 00000400:00000200:15.0:1616513074.041316:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.54@tcp1 00000400:00000200:15.0:1616513074.041323:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.54@tcp1 routed destination 00000400:00000200:15.0:1616513074.041336:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.54@tcp1(10.31.140.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.041341:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.041346:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.041348:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.041350:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.041354:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513074.041357:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.041360:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513074.041361:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513074.041366:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.041375:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.54@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.041383:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.54@tcp1 of length 224 into portal 12 MB=0x5ffe490f26f00 00000400:00000200:2.0:1616513074.041390:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.54@tcp1 of length 224/224 into md 0xf96450b9 [1] + 37648 00000400:00000200:2.0:1616513074.041395:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.041398:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513074.041404:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.041407:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.041409:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.041414:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513074.041452:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.041459:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.041462:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.041469:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:30.0:1616513074.041470:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688732032986880 00000400:00000200:0.0:1616513074.041475:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775650 00000800:00000200:0.0:1616513074.041478:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.041481:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:30.0:1616513074.041482:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138f080 x1688732032986880/t0(0) o400->4df7d946-307b-8187-3749-e664cd13ecda@10.31.140.54@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.041497:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688732032986880, offset 192 00000400:00000200:30.0:1616513074.041504:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.54@tcp1 00000400:00000200:30.0:1616513074.041511:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.54@tcp1 routed destination 00000400:00000200:30.0:1616513074.041526:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.54@tcp1(10.31.140.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.041533:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513074.041539:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.041542:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513074.041544:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513074.041548:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.041552:0:48809:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.041563:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.041569:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.041571:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513074.041573:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.041578:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.041582:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeaa0 00000800:00000200:22.0:1616513074.041586:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.047534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.047588:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.047598:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513074.047607:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.197@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.047616:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.197@tcp1 of length 224 into portal 26 MB=0x5ffe15947cb80 00000400:00000200:28.0:1616513074.047625:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.197@tcp1 of length 224/224 into md 0xf97c5739 [1] + 0 00000400:00000200:28.0:1616513074.047630:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241dc0 00000400:00000200:28.0:1616513074.047633:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.047635:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513074.047640:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241dc0 00000800:00000200:12.0:1616513074.047642:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.047644:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.047646:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513074.047649:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.047652:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.047652:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513074.047661:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.197@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.047670:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.197@tcp1 of length 224 into portal 12 MB=0x5ffe15947cf00 00000400:00000200:12.0:1616513074.047676:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.197@tcp1 of length 224/224 into md 0x865b9f31 [1] + 71864 00000400:00000200:12.0:1616513074.047681:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.047684:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513074.047690:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.047693:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.047695:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.047700:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688718214155136 00010000:00000200:15.0:1616513074.047709:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688718214155136/t0(0) o400->d96470c3-0a35-7286-5727-1aa41147a4d0@10.31.142.197@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.047721:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688718214155136, offset 192 00000400:00000200:15.0:1616513074.047726:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.197@tcp1 00000400:00000200:15.0:1616513074.047733:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.197@tcp1 routed destination 00000400:00000200:15.0:1616513074.047746:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.197@tcp1(10.31.142.197@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.047751:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:20.0:1616513074.047754:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688718214156032 00000800:00000200:15.0:1616513074.047756:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.047758:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.047760:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.047764:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:20.0:1616513074.047766:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc579b00 x1688718214156032/t0(0) o400->374c80d0-215a-fc04-8cfd-fc0a0630ce33@10.31.142.197@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513074.047769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.047770:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:20.0:1616513074.047781:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688718214156032, offset 192 00000400:00000200:20.0:1616513074.047787:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.197@tcp1 00000400:00000200:20.0:1616513074.047795:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.197@tcp1 routed destination 00000400:00000200:20.0:1616513074.047819:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.197@tcp1(10.31.142.197@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.047825:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513074.047825:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513074.047830:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513074.047832:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513074.047834:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513074.047835:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513074.047836:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513074.047836:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.047840:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513074.047841:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513074.047846:0:48809:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513074.047847:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77746e8 00000800:00000200:8.0:1616513074.047850:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513074.047899:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.047906:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.047909:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.047916:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.047920:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23d38 00000800:00000200:14.0:1616513074.047925:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.047928:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.059003:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.059060:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.059069:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.059078:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.105@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.059087:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.105@tcp1 of length 224 into portal 26 MB=0x5ffdb9e1d6940 00000400:00000200:26.0:1616513074.059094:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.105@tcp1 of length 224/224 into md 0xf97c5769 [1] + 0 00000400:00000200:26.0:1616513074.059099:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241188 00000400:00000200:26.0:1616513074.059102:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.059104:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.059109:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241188 00000800:00000200:26.0:1616513074.059112:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.059112:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.059115:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.059117:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.059166:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693599201600 00010000:00000200:15.0:1616513074.059174:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688693599201600/t0(0) o400->b42741b1-d59f-bc83-5037-53ef758fcc94@10.31.136.105@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.059187:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693599201600, offset 192 00000400:00000200:15.0:1616513074.059192:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.105@tcp1 00000400:00000200:15.0:1616513074.059199:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.105@tcp1 routed destination 00000400:00000200:15.0:1616513074.059211:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.105@tcp1(10.31.136.105@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.059216:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.059221:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.059223:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.059224:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.059228:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513074.059234:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513074.059269:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.059325:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.059333:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.059346:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.059352:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.059358:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774990 00000800:00000200:0.0:1616513074.059362:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.059367:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.059414:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.059477:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.059485:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.059493:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.105@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.059502:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.105@tcp1 of length 224 into portal 12 MB=0x5ffdb9e1d6a00 00000400:00000200:14.0:1616513074.059508:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.105@tcp1 of length 224/224 into md 0xf96450b9 [1] + 37872 00000400:00000200:14.0:1616513074.059512:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.059515:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.059520:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.059523:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.059525:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.059528:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513074.059584:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693599201792 00010000:00000200:30.0:1616513074.059596:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c9680 x1688693599201792/t0(0) o400->bcaa42ac-df65-ef6c-0df3-14040ebf495e@10.31.136.105@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.059611:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693599201792, offset 192 00000400:00000200:30.0:1616513074.059617:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.105@tcp1 00000400:00000200:30.0:1616513074.059625:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.105@tcp1 routed destination 00000400:00000200:30.0:1616513074.059640:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.105@tcp1(10.31.136.105@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.059647:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513074.059653:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.059656:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513074.059660:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513074.059664:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.059669:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.059679:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.059685:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.059688:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513074.059689:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.059694:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.059698:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efed0 00000800:00000200:22.0:1616513074.059702:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.066892:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.066945:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.066954:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513074.066962:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.101@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.066971:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.101@tcp1 of length 224 into portal 26 MB=0x6018321eaaa80 00000400:00000200:28.0:1616513074.066977:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.101@tcp1 of length 224/224 into md 0xf97c5789 [1] + 0 00000400:00000200:28.0:1616513074.066982:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240c38 00000400:00000200:28.0:1616513074.066985:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.066988:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513074.066992:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240c38 00000800:00000200:28.0:1616513074.066995:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.066997:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.066998:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.067002:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.067045:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513074.067057:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512581634688 00010000:00000200:15.0:1616513074.067068:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1690512581634688/t0(0) o400->adb27f66-7339-4867-8775-37d28e80b640@10.31.136.101@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.067082:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512581634688, offset 192 00000400:00000200:15.0:1616513074.067088:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.101@tcp1 00000400:00000200:15.0:1616513074.067097:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.101@tcp1 routed destination 00000800:00000200:2.0:1616513074.067110:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:15.0:1616513074.067113:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.101@tcp1(10.31.136.101@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513074.067119:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.067120:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.067126:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.067129:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513074.067129:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.101@tcp1 : PUT - for me 00000800:00000200:15.0:1616513074.067132:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.067136:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513074.067140:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.101@tcp1 of length 224 into portal 12 MB=0x6018321eaac40 00000800:00000200:11.2:1616513074.067142:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.067143:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513074.067147:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.101@tcp1 of length 224/224 into md 0xf96450b9 [1] + 38096 00000400:00000200:2.0:1616513074.067152:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.067155:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513074.067161:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.067163:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.067164:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.067167:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513074.067197:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.067205:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.067208:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.067214:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.067220:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775188 00000800:00000200:0.0:1616513074.067224:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.067227:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513074.067230:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512581635136 00010000:00000200:30.0:1616513074.067242:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138f500 x1690512581635136/t0(0) o400->493a1a22-b11f-afb5-b537-4c08c733f43f@10.31.136.101@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.067256:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512581635136, offset 192 00000400:00000200:30.0:1616513074.067262:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.101@tcp1 00000400:00000200:30.0:1616513074.067269:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.101@tcp1 routed destination 00000400:00000200:30.0:1616513074.067283:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.101@tcp1(10.31.136.101@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.067289:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513074.067295:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.067297:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513074.067299:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513074.067304:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.067309:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.067362:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.067369:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.067372:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.067379:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.067383:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eebb0 00000800:00000200:26.0:1616513074.067386:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.067389:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.074107:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513074.074160:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.074169:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.074178:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.238@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.074187:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.238@tcp1 of length 224 into portal 26 MB=0x5ffd440bbf900 00000400:00000200:8.0:1616513074.074194:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.238@tcp1 of length 224/224 into md 0xf97c5799 [1] + 0 00000400:00000200:8.0:1616513074.074198:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240000 00000400:00000200:8.0:1616513074.074201:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.074204:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513074.074209:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240000 00000800:00000200:8.0:1616513074.074212:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.074213:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.074214:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.074217:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.074271:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661967763712 00010000:00000200:15.0:1616513074.074280:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688661967763712/t0(0) o400->2e942272-7d96-4510-f646-f43ab7369199@10.31.132.238@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513074.074284:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513074.074293:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661967763712, offset 192 00000400:00000200:15.0:1616513074.074298:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.238@tcp1 00000400:00000200:15.0:1616513074.074305:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.238@tcp1 routed destination 00000400:00000200:15.0:1616513074.074319:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.238@tcp1(10.31.132.238@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.074325:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.074330:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.074333:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.074335:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.074339:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513074.074342:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513074.074344:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.074345:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513074.074352:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.074361:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.238@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.074369:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.238@tcp1 of length 224 into portal 12 MB=0x5ffd440bbfa00 00000400:00000200:14.0:1616513074.074375:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.238@tcp1 of length 224/224 into md 0xf96450b9 [1] + 38320 00000400:00000200:14.0:1616513074.074381:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.074383:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.074390:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.074393:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.074394:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.074397:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513074.074399:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.074407:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.074410:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.074416:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.074422:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774550 00000800:00000200:0.0:1616513074.074426:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.074429:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513074.074466:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661967763968 00010000:00000200:30.0:1616513074.074477:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cec00 x1688661967763968/t0(0) o400->d8f2335c-59ae-d41b-37a6-2ad596fe1343@10.31.132.238@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.074491:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661967763968, offset 192 00000400:00000200:30.0:1616513074.074497:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.238@tcp1 00000400:00000200:30.0:1616513074.074504:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.238@tcp1 routed destination 00000400:00000200:30.0:1616513074.074518:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.238@tcp1(10.31.132.238@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.074524:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513074.074530:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.074533:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513074.074535:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513074.074539:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.074543:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.074597:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.074604:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.074607:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.074613:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.074617:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee660 00000800:00000200:26.0:1616513074.074621:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.074624:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513074.075485:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924400, portal 4 00000100:00000200:4.0:1616513074.075490:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924400, offset 0 00000400:00000200:4.0:1616513074.075497:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:4.0:1616513074.075503:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:4.0:1616513074.075516:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513074.075523:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:4.0:1616513074.075531:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513074.075534:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:4.0:1616513074.075536:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:4.0:1616513074.075541:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513074.075588:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.075607:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513074.075612:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513074.075615:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513074.075622:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513074.075629:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdc57c800 x1693067725924400/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513074.075641:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba3b8 00000800:00000200:27.0:1616513074.075645:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513074.075648:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513074.075756:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513074.075765:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.075771:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513074.075775:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513074.075779:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513074.075786:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626830 00000400:00000200:27.0:1616513074.075793:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371657bd [1] + 192 00000400:00000200:27.0:1616513074.075798:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513074.075802:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdc57c800 x1693067725924400/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513074.075817:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.075820:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513074.075821:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:4.0:1616513074.075825:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdc57c800 x1693067725924400/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513074.075838:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdc57c800 x1693067725924400/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513074.075849:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000800:00000200:7.2:1616513074.080213:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.080266:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.080276:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.080285:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.4@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.080294:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.4@tcp1 of length 224 into portal 26 MB=0x5ffe77f5bc8c0 00000400:00000200:14.0:1616513074.080301:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.4@tcp1 of length 224/224 into md 0xf97c57b9 [1] + 0 00000400:00000200:14.0:1616513074.080306:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782407f8 00000400:00000200:14.0:1616513074.080309:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.080314:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513074.080318:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.080319:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782407f8 00000800:00000200:14.0:1616513074.080322:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.080324:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.080326:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.080340:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513074.080378:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688744622803136 00010000:00000200:15.0:1616513074.080387:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688744622803136/t0(0) o400->dc93886c-707c-e5cb-d2d5-d68dd54273ea@10.31.142.4@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513074.080394:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.080400:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688744622803136, offset 192 00000800:00000200:26.0:1616513074.080403:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:15.0:1616513074.080405:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.4@tcp1 00000400:00000200:26.0:1616513074.080412:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.4@tcp1 : PUT - for me 00000400:00000200:15.0:1616513074.080413:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.4@tcp1 routed destination 00000400:00000200:26.0:1616513074.080421:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.4@tcp1 of length 224 into portal 12 MB=0x5ffe77f5bc9c0 00000400:00000200:15.0:1616513074.080427:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.4@tcp1(10.31.142.4@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513074.080428:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.4@tcp1 of length 224/224 into md 0xf96450b9 [1] + 38544 00000800:00000200:15.0:1616513074.080445:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513074.080451:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:15.0:1616513074.080451:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.080453:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513074.080455:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:15.0:1616513074.080455:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513074.080458:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:15.0:1616513074.080459:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.080464:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513074.080465:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.080466:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513074.080468:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.080470:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513074.080527:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688744622803392 00000800:00000200:28.0:1616513074.080532:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:30.0:1616513074.080539:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695580 x1688744622803392/t0(0) o400->f7c4dba8-9e74-a3d3-0991-a2c1683d6c9d@10.31.142.4@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513074.080540:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.080543:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513074.080550:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:30.0:1616513074.080554:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688744622803392, offset 192 00000400:00000200:28.0:1616513074.080556:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775320 00000800:00000200:28.0:1616513074.080559:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513074.080562:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.4@tcp1 00000800:00000200:28.0:1616513074.080562:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513074.080570:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.4@tcp1 routed destination 00000400:00000200:30.0:1616513074.080585:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.4@tcp1(10.31.142.4@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.080591:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513074.080597:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.080600:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513074.080602:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513074.080607:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.080611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.080621:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.080627:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.080629:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513074.080631:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513074.080636:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.080640:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efba0 00000800:00000200:2.0:1616513074.080644:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.085492:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513074.085547:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.085557:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.085566:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.182@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.085574:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.182@tcp1 of length 224 into portal 26 MB=0x60182cbd050c0 00000400:00000200:0.0:1616513074.085581:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.182@tcp1 of length 224/224 into md 0xf97c57c9 [1] + 0 00000400:00000200:0.0:1616513074.085586:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241b18 00000400:00000200:0.0:1616513074.085589:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.085592:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513074.085597:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241b18 00000800:00000200:12.0:1616513074.085600:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.085600:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.085603:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.085607:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.085610:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.085611:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513074.085619:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.182@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.085630:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.182@tcp1 of length 224 into portal 12 MB=0x60182cbd051c0 00000400:00000200:12.0:1616513074.085636:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.182@tcp1 of length 224/224 into md 0x865b9f31 [1] + 72088 00000400:00000200:12.0:1616513074.085641:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.085644:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513074.085650:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.085654:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513074.085656:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511137067200 00000800:00000200:12.0:1616513074.085656:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:15.0:1616513074.085666:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1690511137067200/t0(0) o400->6f45ea69-91c1-46c4-8f79-2a1ee50b9ffb@10.31.132.182@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.085679:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511137067200, offset 192 00000400:00000200:15.0:1616513074.085684:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.182@tcp1 00000400:00000200:15.0:1616513074.085691:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.182@tcp1 routed destination 00000400:00000200:15.0:1616513074.085705:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.182@tcp1(10.31.132.182@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.085710:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.085715:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.085719:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.085721:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.085726:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513074.085727:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511137067456 00000800:00000200:11.2:1616513074.085731:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.085732:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:20.0:1616513074.085740:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57f080 x1690511137067456/t0(0) o400->86e386e5-191b-6d80-42af-cff6ca42415e@10.31.132.182@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.085754:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511137067456, offset 192 00000400:00000200:20.0:1616513074.085761:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.182@tcp1 00000400:00000200:20.0:1616513074.085768:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.182@tcp1 routed destination 00000800:00000200:8.0:1616513074.085783:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513074.085784:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.182@tcp1(10.31.132.182@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513074.085790:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513074.085791:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513074.085793:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.085794:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513074.085797:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.085799:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513074.085800:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513074.085802:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.085806:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513074.085806:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774f68 00000800:00000200:8.0:1616513074.085810:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.085811:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.085864:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.085871:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.085873:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.085880:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.085884:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22440 00000800:00000200:22.0:1616513074.085888:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.085891:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.090566:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.090619:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.090628:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.090636:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.207@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.090644:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.207@tcp1 of length 224 into portal 12 MB=0x5ffe487b1f2c0 00000400:00000200:14.0:1616513074.090650:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.207@tcp1 of length 224/224 into md 0xf96450b9 [1] + 38768 00000400:00000200:14.0:1616513074.090655:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.090657:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.090663:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.090666:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.090667:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.090670:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513074.090725:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731877765824 00010000:00000200:30.0:1616513074.090737:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c9b00 x1688731877765824/t0(0) o400->16c45e79-e258-c76c-ec59-03e4c1ea3f7a@10.31.142.207@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.090752:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688731877765824, offset 192 00000400:00000200:30.0:1616513074.090759:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.207@tcp1 00000400:00000200:30.0:1616513074.090766:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.207@tcp1 routed destination 00000400:00000200:30.0:1616513074.090781:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.207@tcp1(10.31.142.207@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.090787:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.090793:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.090796:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513074.090798:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.090803:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.090808:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.090858:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.090865:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513074.090867:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513074.090873:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.090877:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee4c8 00000800:00000200:12.0:1616513074.090881:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.090883:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.090889:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513074.090896:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.207@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.090903:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.207@tcp1 of length 224 into portal 26 MB=0x5ffe487b1f240 00000400:00000200:12.0:1616513074.090909:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.207@tcp1 of length 224/224 into md 0xf97c57d9 [1] + 0 00000800:00000200:28.0:1616513074.090911:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513074.090913:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240110 00000400:00000200:12.0:1616513074.090916:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.090918:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513074.090922:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240110 00000800:00000200:12.0:1616513074.090925:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.090927:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.090929:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.090990:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731877765696 00010000:00000200:15.0:1616513074.091001:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688731877765696/t0(0) o400->cd1073ef-189e-0123-bf5f-c2fec4b2a0b2@10.31.142.207@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.091015:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688731877765696, offset 192 00000400:00000200:15.0:1616513074.091022:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.207@tcp1 00000400:00000200:15.0:1616513074.091030:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.207@tcp1 routed destination 00000400:00000200:15.0:1616513074.091045:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.207@tcp1(10.31.142.207@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.091052:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.091058:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.091061:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.091063:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.091068:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.091074:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.091075:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513074.091127:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.091134:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.091138:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.091144:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.091151:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775a08 00000800:00000200:22.0:1616513074.091154:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.091157:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.099068:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513074.099082:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.099091:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.099100:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.218@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.099111:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.218@tcp1 of length 224 into portal 26 MB=0x5fffcfdf21900 00000400:00000200:8.0:1616513074.099118:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.218@tcp1 of length 224/224 into md 0xf97c57f9 [1] + 0 00000400:00000200:8.0:1616513074.099124:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240990 00000400:00000200:8.0:1616513074.099127:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.099130:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513074.099136:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240990 00000800:00000200:8.0:1616513074.099139:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.099142:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.099144:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.099148:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.099203:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688836940896512 00010000:00000200:15.0:1616513074.099226:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688836940896512/t0(0) o400->76293125-6011-6399-c38c-77149b866ca0@10.31.136.218@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.099240:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688836940896512, offset 192 00000400:00000200:15.0:1616513074.099246:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.218@tcp1 00000400:00000200:15.0:1616513074.099255:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.218@tcp1 routed destination 00000400:00000200:15.0:1616513074.099269:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.218@tcp1(10.31.136.218@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.099275:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.099281:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.099283:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.099285:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.099290:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.099295:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.099296:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513074.099345:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.099352:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513074.099355:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513074.099361:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.099366:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77755c8 00000800:00000200:12.0:1616513074.099370:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.099372:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.099560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.099570:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.099578:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513074.099580:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.099589:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.218@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.099599:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.218@tcp1 of length 224 into portal 12 MB=0x5fffcfdf21ac0 00000400:00000200:28.0:1616513074.099605:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.218@tcp1 of length 224/224 into md 0x865b9f31 [1] + 72312 00000400:00000200:28.0:1616513074.099610:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.099612:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513074.099619:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.099622:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.099624:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.099682:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688836940896960 00010000:00000200:20.0:1616513074.099694:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf82d00 x1688836940896960/t0(0) o400->6ef5b0ae-c1b9-82fb-289e-e459ca0ff0a6@10.31.136.218@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.099707:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688836940896960, offset 192 00000400:00000200:20.0:1616513074.099713:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.218@tcp1 00000400:00000200:20.0:1616513074.099721:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.218@tcp1 routed destination 00000400:00000200:20.0:1616513074.099735:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.218@tcp1(10.31.136.218@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.099749:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.099755:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.099757:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.099759:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.099764:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.099772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.099828:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.099835:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.099838:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.099845:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.099849:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22000 00000800:00000200:2.0:1616513074.099853:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.099856:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.100721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513074.100777:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.100787:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513074.100794:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.224@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.100804:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.224@tcp1 of length 224 into portal 26 MB=0x5ffd4e8f3f700 00000400:00000200:12.0:1616513074.100810:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.224@tcp1 of length 224/224 into md 0xf97c5819 [1] + 0 00000400:00000200:12.0:1616513074.100814:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782415c8 00000400:00000200:12.0:1616513074.100817:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.100820:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513074.100824:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782415c8 00000800:00000200:12.0:1616513074.100827:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.100829:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.100831:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513074.100837:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.100840:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.100841:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.100850:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.224@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.100859:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.224@tcp1 of length 224 into portal 12 MB=0x5ffd4e8f3f780 00000400:00000200:0.0:1616513074.100865:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.224@tcp1 of length 224/224 into md 0x865b9f31 [1] + 72536 00000400:00000200:0.0:1616513074.100872:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.100875:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513074.100882:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.100885:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664790005504 00000800:00000200:0.0:1616513074.100885:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.100887:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.100893:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664790005632 00010000:00000200:15.0:1616513074.100896:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688664790005504/t0(0) o400->4b375354-1b75-1ba4-76f3-27a95bbc0b9c@10.31.132.224@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:20.0:1616513074.100901:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d7980 x1688664790005632/t0(0) o400->38b739d1-d3e0-25dd-9f2f-df3274750291@10.31.132.224@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513074.100909:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664790005504, offset 192 00000400:00000200:15.0:1616513074.100915:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.224@tcp1 00000100:00000200:20.0:1616513074.100923:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664790005632, offset 192 00000400:00000200:15.0:1616513074.100923:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.224@tcp1 routed destination 00000400:00000200:20.0:1616513074.100928:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.224@tcp1 00000400:00000200:15.0:1616513074.100938:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.224@tcp1(10.31.132.224@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.100944:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513074.100946:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.224@tcp1 routed destination 00000800:00000200:15.0:1616513074.100950:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.100953:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.100955:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.100959:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513074.100960:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.224@tcp1(10.31.132.224@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513074.100963:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.100966:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:15.0:1616513074.100966:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513074.100971:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.100973:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513074.100975:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513074.100975:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.100978:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513074.100982:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513074.100983:0:48813:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513074.100985:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513074.100985:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.100992:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.100998:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774bb0 00000800:00000200:28.0:1616513074.101002:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513074.101040:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.101047:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.101052:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.101059:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.101063:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22550 00000800:00000200:22.0:1616513074.101067:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.101070:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.102186:0:48813:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.102240:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.102250:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.102259:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.56@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.102269:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.56@tcp1 of length 224 into portal 26 MB=0x5ffdaf65f3140 00000400:00000200:2.0:1616513074.102275:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.56@tcp1 of length 224/224 into md 0xf97c5839 [1] + 0 00000400:00000200:2.0:1616513074.102287:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241430 00000400:00000200:2.0:1616513074.102290:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.102293:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513074.102298:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241430 00000800:00000200:2.0:1616513074.102301:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.102304:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.102305:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.102310:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.102360:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688690784940352 00010000:00000200:15.0:1616513074.102370:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688690784940352/t0(0) o400->7732c16b-a0bc-2ed1-4e5b-118e6944244c@10.31.140.56@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.102384:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688690784940352, offset 192 00000400:00000200:15.0:1616513074.102389:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.56@tcp1 00000400:00000200:15.0:1616513074.102398:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.56@tcp1 routed destination 00000400:00000200:15.0:1616513074.102412:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.56@tcp1(10.31.140.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.102418:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.102424:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.102427:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.102429:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:11.2:1616513074.102441:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:15.0:1616513074.102446:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:15.0:1616513074.102453:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513074.102510:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.102518:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.102521:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.102528:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513074.102532:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:8.0:1616513074.102534:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775078 00000800:00000200:8.0:1616513074.102538:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.102542:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.102542:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.102551:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.56@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.102561:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.56@tcp1 of length 224 into portal 12 MB=0x5ffdaf65f3340 00000400:00000200:0.0:1616513074.102567:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.56@tcp1 of length 224/224 into md 0x865b9f31 [1] + 72760 00000400:00000200:0.0:1616513074.102573:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.102575:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513074.102582:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.102585:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.102586:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.102645:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688690784940864 00010000:00000200:20.0:1616513074.102658:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d1b00 x1688690784940864/t0(0) o400->10468d6f-7217-2404-2f97-cf126b151746@10.31.140.56@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.102672:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688690784940864, offset 192 00000400:00000200:20.0:1616513074.102679:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.56@tcp1 00000400:00000200:20.0:1616513074.102686:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.56@tcp1 routed destination 00000400:00000200:20.0:1616513074.102701:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.56@tcp1(10.31.140.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.102708:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.102714:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.102717:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.102719:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.102724:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.102729:0:48813:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.102784:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.102791:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.102793:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.102800:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.102807:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee234b8 00000800:00000200:22.0:1616513074.102811:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.102814:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.115320:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513074.115358:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513074.115375:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.115394:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.115402:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.100@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.115411:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.100@tcp1 of length 224 into portal 26 MB=0x5ffdbe37a6400 00000400:00000200:2.0:1616513074.115417:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.100@tcp1 of length 224/224 into md 0xf97c5849 [1] + 0 00000400:00000200:2.0:1616513074.115422:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782404c8 00000400:00000200:2.0:1616513074.115424:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.115427:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513074.115432:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782404c8 00000800:00000200:14.0:1616513074.115447:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513074.115448:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513074.115450:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.115453:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513074.115455:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513074.115457:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513074.115457:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513074.115458:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513074.115466:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.100@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.115466:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.174@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.115475:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.100@tcp1 of length 224 into portal 12 MB=0x5ffdbe37a65c0 00000400:00000200:14.0:1616513074.115476:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.174@tcp1 of length 224 into portal 12 MB=0x5ffd48ea6ed40 00000400:00000200:28.0:1616513074.115481:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.100@tcp1 of length 224/224 into md 0x865b9f31 [1] + 72984 00000400:00000200:14.0:1616513074.115482:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.174@tcp1 of length 224/224 into md 0xf96450b9 [1] + 38992 00000400:00000200:28.0:1616513074.115486:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.115487:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.115488:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:14.0:1616513074.115490:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513074.115494:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:15.0:1616513074.115494:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694762923008 00000800:00000200:14.0:1616513074.115496:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513074.115497:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.115498:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513074.115499:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513074.115501:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513074.115501:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:15.0:1616513074.115504:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688694762923008/t0(0) o400->d6f437c7-1958-f14d-0d8e-174519f8cd5c@10.31.136.100@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.115517:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694762923008, offset 192 00000400:00000200:15.0:1616513074.115525:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.100@tcp1 00000400:00000200:15.0:1616513074.115533:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.100@tcp1 routed destination 00000800:00000200:7.2:1616513074.115541:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:15.0:1616513074.115548:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.100@tcp1(10.31.136.100@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.115554:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.115559:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.115562:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.115564:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.115569:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:30.0:1616513074.115571:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663275007296 00000100:00000200:20.0:1616513074.115571:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694762923456 00000800:00000200:11.2:1616513074.115572:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.115575:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:30.0:1616513074.115583:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cb600 x1688663275007296/t0(0) o400->12b574a5-ab14-551d-8636-5db4d7fab519@10.31.129.174@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:20.0:1616513074.115583:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf81f80 x1688694762923456/t0(0) o400->775ffaff-4345-93a2-5784-6c96430a772c@10.31.136.100@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:26.0:1616513074.115595:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:30.0:1616513074.115597:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663275007296, offset 192 00000100:00000200:20.0:1616513074.115597:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694762923456, offset 192 00000400:00000200:30.0:1616513074.115603:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.174@tcp1 00000400:00000200:20.0:1616513074.115603:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.100@tcp1 00000800:00000200:26.0:1616513074.115604:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513074.115605:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513074.115610:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.174@tcp1 routed destination 00000400:00000200:20.0:1616513074.115610:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.100@tcp1 routed destination 00000400:00000200:30.0:1616513074.115624:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.174@tcp1(10.31.129.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513074.115625:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513074.115630:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513074.115632:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513074.115633:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.174@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.115635:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513074.115638:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.100@tcp1(10.31.136.100@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513074.115641:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513074.115646:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513074.115647:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513074.115647:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775298 00000400:00000200:26.0:1616513074.115648:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.174@tcp1 of length 224 into portal 26 MB=0x5ffd48ea6e9c0 00000800:00000200:12.0:1616513074.115650:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513074.115651:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.115651:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.115653:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513074.115653:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:20.0:1616513074.115654:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513074.115655:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.174@tcp1 of length 224/224 into md 0xf97c5859 [1] + 0 00000800:00000200:30.0:1616513074.115658:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:20.0:1616513074.115658:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (24)++ 00000400:00000200:26.0:1616513074.115660:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782416d8 00000800:00000200:7.2:1616513074.115662:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (24)++ 00000400:00000200:26.0:1616513074.115663:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513074.115663:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:26.0:1616513074.115666:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.115671:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782416d8 00000800:00000200:22.0:1616513074.115673:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513074.115674:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513074.115677:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:26.0:1616513074.115679:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (25)-- 00000800:00000200:22.0:1616513074.115680:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:26.0:1616513074.115682:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000400:00000200:22.0:1616513074.115683:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:15.0:1616513074.115684:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663275006400 00000800:00000200:26.0:1616513074.115685:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513074.115686:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513074.115688:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.115690:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:15.0:1616513074.115690:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688663275006400/t0(0) o400->dc15b231-f190-c8d0-0ae8-1a7e37ff6d16@10.31.129.174@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513074.115693:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.115694:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef760 00000400:00000200:26.0:1616513074.115698:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22bb0 00000800:00000200:22.0:1616513074.115698:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.115701:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.115701:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663275006400, offset 192 00000400:00000200:15.0:1616513074.115707:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.174@tcp1 00000400:00000200:15.0:1616513074.115713:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.174@tcp1 routed destination 00000400:00000200:15.0:1616513074.115726:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.174@tcp1(10.31.129.174@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.115731:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.115736:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.115738:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.115740:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.115743:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513074.115747:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513074.115747:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.115758:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.115765:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.115768:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.115769:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.115776:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.115782:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774e58 00000800:00000200:8.0:1616513074.115786:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.119839:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513074.119892:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.119902:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513074.119911:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.119@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.119920:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.119@tcp1 of length 224 into portal 26 MB=0x5ffddac381780 00000400:00000200:12.0:1616513074.119927:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.119@tcp1 of length 224/224 into md 0xf97c5869 [1] + 0 00000400:00000200:12.0:1616513074.119932:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240b28 00000400:00000200:12.0:1616513074.119934:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.119937:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513074.119942:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240b28 00000800:00000200:12.0:1616513074.119945:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.119947:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.119948:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.119954:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.119957:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.119957:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:15.0:1616513074.119963:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702425765760 00000400:00000200:0.0:1616513074.119966:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.119@tcp1 : PUT - for me 00010000:00000200:15.0:1616513074.119972:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688702425765760/t0(0) o400->8477f066-53a5-6082-5d79-832969509c1c@10.31.142.119@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513074.119976:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.119@tcp1 of length 224 into portal 12 MB=0x5ffddac381a80 00000400:00000200:0.0:1616513074.119982:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.119@tcp1 of length 224/224 into md 0x865b9f31 [1] + 73208 00000100:00000200:15.0:1616513074.119985:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702425765760, offset 192 00000400:00000200:0.0:1616513074.119987:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:15.0:1616513074.119990:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.119@tcp1 00000100:00000200:0.0:1616513074.119990:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513074.119996:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513074.119997:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.119@tcp1 routed destination 00000800:00000200:0.0:1616513074.120000:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.120001:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:15.0:1616513074.120010:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.119@tcp1(10.31.142.119@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.120020:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.120026:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.120028:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.120030:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.120034:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513074.120040:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.120040:0:48815:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.120063:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702425766528 00010000:00000200:20.0:1616513074.120075:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d4800 x1688702425766528/t0(0) o400->4fc03b4b-e354-75c3-77bd-0833306055ea@10.31.142.119@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.120090:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702425766528, offset 192 00000800:00000200:14.0:1616513074.120095:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513074.120096:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.119@tcp1 00000800:00000200:14.0:1616513074.120102:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513074.120104:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.119@tcp1 routed destination 00000400:00000200:14.0:1616513074.120105:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.120112:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513074.120119:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.119@tcp1(10.31.142.119@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513074.120120:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775e48 00000800:00000200:14.0:1616513074.120124:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513074.120125:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513074.120127:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513074.120131:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.120134:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.120136:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.120141:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.120146:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.120198:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.120206:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.120208:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513074.120220:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.120225:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.120229:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee223b8 00000800:00000200:8.0:1616513074.120232:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.165882:0:48819:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.165939:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.165949:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.165958:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.54@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.165967:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.54@tcp1 of length 224 into portal 26 MB=0x601968cccebc0 00000400:00000200:22.0:1616513074.165974:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.54@tcp1 of length 224/224 into md 0xf97c5899 [1] + 0 00000400:00000200:22.0:1616513074.165979:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240440 00000400:00000200:22.0:1616513074.165982:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.165984:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513074.165989:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240440 00000800:00000200:26.0:1616513074.165992:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.165993:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.165995:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.165997:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.166051:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690595979226048 00010000:00000200:15.0:1616513074.166061:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1690595979226048/t0(0) o400->5c97b998-3e7c-f74c-cdbe-f3f3fdcac75e@10.31.141.54@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.166074:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690595979226048, offset 192 00000400:00000200:15.0:1616513074.166079:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.54@tcp1 00000400:00000200:15.0:1616513074.166086:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.54@tcp1 routed destination 00000400:00000200:15.0:1616513074.166100:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.54@tcp1(10.31.141.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.166105:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.166111:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.166113:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.166115:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.166120:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.166124:0:48819:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.166126:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513074.166178:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.166186:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.166189:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.166196:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.166202:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775298 00000800:00000200:14.0:1616513074.166206:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.166209:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.166266:0:48819:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.166277:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.166285:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513074.166287:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513074.166294:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.54@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.166302:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.54@tcp1 of length 224 into portal 12 MB=0x601968cccf180 00000400:00000200:26.0:1616513074.166309:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.54@tcp1 of length 224/224 into md 0xf96450b9 [1] + 39216 00000400:00000200:26.0:1616513074.166314:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.166317:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513074.166325:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.166327:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.166329:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513074.166390:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690595979227520 00010000:00000200:30.0:1616513074.166403:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695100 x1690595979227520/t0(0) o400->ac72237a-5417-11df-6bc4-ec48b932deaf@10.31.141.54@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.166418:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690595979227520, offset 192 00000400:00000200:30.0:1616513074.166425:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.54@tcp1 00000400:00000200:30.0:1616513074.166432:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.54@tcp1 routed destination 00000400:00000200:30.0:1616513074.166460:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.54@tcp1(10.31.141.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.166467:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.166473:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.166476:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513074.166478:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.166483:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.166491:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.166545:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.166551:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.166554:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.166560:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.166564:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee990 00000800:00000200:0.0:1616513074.166567:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.166570:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:21.0:1616513074.170818:0:6492:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924416, portal 4 00000100:00000200:21.0:1616513074.170824:0:6492:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924416, offset 0 00000400:00000200:21.0:1616513074.170831:0:6492:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:21.0:1616513074.170837:0:6492:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:21.0:1616513074.170850:0:6492:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:21.0:1616513074.170857:0:6492:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:21.0:1616513074.170863:0:6492:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513074.170866:0:6492:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:21.0:1616513074.170868:0:6492:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:21.0:1616513074.170873:0:6492:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513074.170919:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.170937:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513074.170942:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513074.170945:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513074.170951:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513074.170958:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d853e8ce300 x1693067725924416/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513074.170970:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51760 00000800:00000200:27.0:1616513074.170973:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513074.170976:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513074.171091:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513074.171100:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.171106:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513074.171110:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513074.171115:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513074.171124:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626840 00000400:00000200:27.0:1616513074.171130:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716586d [1] + 192 00000400:00000200:27.0:1616513074.171135:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513074.171139:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d853e8ce300 x1693067725924416/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513074.171153:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.171155:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513074.171157:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:21.0:1616513074.171161:0:6492:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d853e8ce300 x1693067725924416/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:21.0:1616513074.171174:0:6492:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d853e8ce300 x1693067725924416/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:21.0:1616513074.171184:0:6492:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51650 00000800:00000200:11.2:1616513074.175244:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513074.175287:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513074.175299:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513074.175300:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.175307:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:12.0:1616513074.175308:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:14.0:1616513074.175316:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.62@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.175317:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.62@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.175325:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.62@tcp1 of length 224 into portal 26 MB=0x5ffde4ec7a700 00000400:00000200:12.0:1616513074.175325:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.62@tcp1 of length 224 into portal 12 MB=0x5ffde4ec7a880 00000400:00000200:12.0:1616513074.175331:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.62@tcp1 of length 224/224 into md 0x865b9f31 [1] + 73432 00000400:00000200:14.0:1616513074.175332:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.62@tcp1 of length 224/224 into md 0xf97c58a9 [1] + 0 00000400:00000200:12.0:1616513074.175336:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.175337:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240660 00000100:00000200:12.0:1616513074.175338:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513074.175339:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.175342:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513074.175345:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513074.175347:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:12.0:1616513074.175347:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.175349:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513074.175350:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.175351:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513074.175352:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513074.175353:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.175355:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.175411:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705153083520 00000100:00000200:15.0:1616513074.175414:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705153083136 00010000:00000200:20.0:1616513074.175424:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57ec00 x1688705153083520/t0(0) o400->0bb054ff-0ef7-3936-0413-2ca2461b4e96@10.31.141.62@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:15.0:1616513074.175426:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688705153083136/t0(0) o400->f1b1ddb6-229d-a5c9-f1aa-96f695f1ba46@10.31.141.62@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.175451:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705153083520, offset 192 00000100:00000200:15.0:1616513074.175452:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705153083136, offset 192 00000400:00000200:20.0:1616513074.175458:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.62@tcp1 00000400:00000200:15.0:1616513074.175459:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.62@tcp1 00000400:00000200:20.0:1616513074.175466:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.62@tcp1 routed destination 00000400:00000200:15.0:1616513074.175468:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.62@tcp1 routed destination 00000400:00000200:20.0:1616513074.175483:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.62@tcp1(10.31.141.62@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:15.0:1616513074.175486:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.62@tcp1(10.31.141.62@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.175491:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.175493:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.175498:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.175500:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.175501:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.175503:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.175504:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.175507:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.175509:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:15.0:1616513074.175513:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513074.175513:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513074.175518:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.175520:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513074.175577:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.175584:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513074.175584:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513074.175587:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.175592:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.175595:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.175596:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.175602:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:28.0:1616513074.175603:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513074.175603:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22c38 00000800:00000200:22.0:1616513074.175607:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513074.175608:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774bb0 00000800:00000200:22.0:1616513074.175610:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513074.175612:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.177519:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513074.177571:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.177581:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513074.177590:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.92@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.177600:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.92@tcp1 of length 224 into portal 26 MB=0x5ffe2d4fd9040 00000400:00000200:12.0:1616513074.177607:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.92@tcp1 of length 224/224 into md 0xf97c58b9 [1] + 0 00000400:00000200:12.0:1616513074.177612:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241f58 00000400:00000200:12.0:1616513074.177614:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.177617:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513074.177622:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241f58 00000800:00000200:12.0:1616513074.177625:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.177625:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.177628:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.177632:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.177635:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.177636:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.177644:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.92@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.177653:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.92@tcp1 of length 224 into portal 12 MB=0x5ffe2d4fd92c0 00000400:00000200:8.0:1616513074.177660:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.92@tcp1 of length 224/224 into md 0x865b9f31 [1] + 73656 00000400:00000200:8.0:1616513074.177665:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.177668:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513074.177676:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.177679:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.177681:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.177687:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724584632384 00010000:00000200:15.0:1616513074.177698:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688724584632384/t0(0) o400->8a900185-ced5-149f-8d8b-7d8549a86284@10.31.141.92@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.177713:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688724584632384, offset 192 00000400:00000200:15.0:1616513074.177719:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.92@tcp1 00000400:00000200:15.0:1616513074.177728:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.92@tcp1 routed destination 00000100:00000200:20.0:1616513074.177740:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724584633024 00000400:00000200:15.0:1616513074.177743:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.92@tcp1(10.31.141.92@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.177750:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:20.0:1616513074.177752:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3f00 x1688724584633024/t0(0) o400->e7efad9e-ca5c-333f-9277-06585a85c5b4@10.31.141.92@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513074.177756:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.177759:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.177761:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.177766:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513074.177767:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688724584633024, offset 192 00000800:00000200:7.2:1616513074.177772:0:48820:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513074.177773:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.92@tcp1 00000100:00000200:15.0:1616513074.177773:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513074.177781:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.92@tcp1 routed destination 00000400:00000200:20.0:1616513074.177797:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.92@tcp1(10.31.141.92@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.177803:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.177809:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.177812:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.177814:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.177819:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.177824:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513074.177827:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.177834:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.177837:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.177839:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513074.177844:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513074.177846:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.177849:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513074.177850:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513074.177851:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775a08 00000800:00000200:2.0:1616513074.177854:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513074.177855:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513074.177857:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513074.177860:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22660 00000800:00000200:0.0:1616513074.177863:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.187418:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.187472:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.187481:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513074.187490:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.46@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.187500:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.46@tcp1 of length 224 into portal 12 MB=0x5ffde6d4d6ec0 00000400:00000200:28.0:1616513074.187506:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.46@tcp1 of length 224/224 into md 0x865b9f31 [1] + 73880 00000400:00000200:28.0:1616513074.187513:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.187515:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513074.187522:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.187524:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.187526:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.187529:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.187541:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513074.187586:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705665167040 00000800:00000200:12.0:1616513074.187597:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:20.0:1616513074.187598:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf83600 x1688705665167040/t0(0) o400->a4f7b493-be57-4bd3-a9c8-d9b49c60ea39@10.31.129.46@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513074.187606:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:20.0:1616513074.187612:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705665167040, offset 192 00000400:00000200:12.0:1616513074.187615:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.46@tcp1 : PUT - for me 00000400:00000200:20.0:1616513074.187618:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.46@tcp1 00000400:00000200:20.0:1616513074.187627:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.46@tcp1 routed destination 00000400:00000200:12.0:1616513074.187634:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.46@tcp1 of length 224 into portal 26 MB=0x5ffde6d4d6b40 00000400:00000200:20.0:1616513074.187641:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.46@tcp1(10.31.129.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513074.187641:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.46@tcp1 of length 224/224 into md 0xf97c58c9 [1] + 0 00000400:00000200:12.0:1616513074.187646:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241a90 00000800:00000200:20.0:1616513074.187648:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513074.187650:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.187651:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513074.187653:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513074.187654:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.187656:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513074.187658:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a90 00000800:00000200:20.0:1616513074.187659:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513074.187661:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.187663:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513074.187664:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.187666:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.187670:0:48821:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.187721:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705665166144 00000800:00000200:22.0:1616513074.187726:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:15.0:1616513074.187731:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688705665166144/t0(0) o400->934a677b-6e8b-3edb-3437-80547361b300@10.31.129.46@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513074.187732:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.187735:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.187742:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:15.0:1616513074.187744:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705665166144, offset 192 00000400:00000200:22.0:1616513074.187746:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22dd0 00000800:00000200:22.0:1616513074.187750:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:15.0:1616513074.187750:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.46@tcp1 00000800:00000200:22.0:1616513074.187753:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513074.187758:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.46@tcp1 routed destination 00000400:00000200:15.0:1616513074.187773:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.46@tcp1(10.31.129.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.187779:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.187785:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.187788:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.187790:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.187794:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513074.187800:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513074.187800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.187809:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.187814:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.187817:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.187818:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.187823:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.187828:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775320 00000800:00000200:28.0:1616513074.187831:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.190375:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513074.190430:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.190454:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.190463:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.28@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.190473:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.28@tcp1 of length 224 into portal 26 MB=0x5ffe85310efc0 00000400:00000200:0.0:1616513074.190480:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.28@tcp1 of length 224/224 into md 0xf97c58e9 [1] + 0 00000800:00000200:12.0:1616513074.190484:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513074.190485:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241100 00000400:00000200:0.0:1616513074.190487:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.190490:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513074.190495:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241100 00000800:00000200:0.0:1616513074.190498:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.190501:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.190503:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.190559:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748174667712 00010000:00000200:15.0:1616513074.190569:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688748174667712/t0(0) o400->eebacead-2b29-5693-3042-8e5acee250c3@10.31.142.28@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513074.190572:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513074.190587:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688748174667712, offset 192 00000400:00000200:15.0:1616513074.190593:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.28@tcp1 00000400:00000200:15.0:1616513074.190600:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.28@tcp1 routed destination 00000400:00000200:15.0:1616513074.190614:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.28@tcp1(10.31.142.28@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.190620:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.190626:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513074.190627:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.190629:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.190631:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.190635:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513074.190636:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:15.0:1616513074.190642:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.190642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513074.190645:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.28@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.190654:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.28@tcp1 of length 224 into portal 12 MB=0x5ffe85310f740 00000400:00000200:8.0:1616513074.190661:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.28@tcp1 of length 224/224 into md 0x865b9f31 [1] + 74104 00000400:00000200:8.0:1616513074.190666:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.190668:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513074.190674:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.190677:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.190680:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.190684:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513074.190703:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.190710:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.190713:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.190720:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.190726:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775188 00000800:00000200:2.0:1616513074.190730:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.190733:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.190738:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000100:00000200:20.0:1616513074.190741:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748174669632 00000800:00000200:2.0:1616513074.190741:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.190743:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.190744:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.190747:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:20.0:1616513074.190753:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d2d00 x1688748174669632/t0(0) o400->1ace4534-b689-e804-2f7f-b5333ea27576@10.31.142.28@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.190768:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688748174669632, offset 192 00000400:00000200:20.0:1616513074.190775:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.28@tcp1 00000400:00000200:20.0:1616513074.190782:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.28@tcp1 routed destination 00000400:00000200:20.0:1616513074.190799:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.28@tcp1(10.31.142.28@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.190805:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.190811:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.190814:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.190816:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.190821:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.190825:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.190836:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.190841:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513074.190844:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.190845:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513074.190850:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.190854:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23760 00000800:00000200:12.0:1616513074.190858:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.191194:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.191205:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.191213:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513074.191215:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.191221:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513074.191222:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.173@tcp1 : PUT - for me 00000800:00000200:0.0:1616513074.191223:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.191228:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.173@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.191232:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.173@tcp1 of length 224 into portal 26 MB=0x5ffde36295040 00000400:00000200:8.0:1616513074.191235:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.173@tcp1 of length 224 into portal 12 MB=0x5ffde362950c0 00000400:00000200:28.0:1616513074.191239:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.173@tcp1 of length 224/224 into md 0xf97c5909 [1] + 0 00000400:00000200:8.0:1616513074.191240:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.173@tcp1 of length 224/224 into md 0x865b9f31 [1] + 74328 00000400:00000200:28.0:1616513074.191243:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241ba0 00000400:00000200:8.0:1616513074.191244:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513074.191246:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.191246:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:28.0:1616513074.191249:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513074.191251:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.191253:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.191254:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241ba0 00000800:00000200:8.0:1616513074.191255:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.191257:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.191260:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.191261:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.191265:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704740053184 00000100:00000200:15.0:1616513074.191268:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704740053056 00010000:00000200:20.0:1616513074.191271:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6300 x1688704740053184/t0(0) o400->4928ffdb-936a-31da-4f80-d97dc942d40c@10.31.136.173@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:15.0:1616513074.191274:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688704740053056/t0(0) o400->0d5f9d53-bdc8-0591-9ddd-cba0150289cb@10.31.136.173@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.191283:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704740053184, offset 192 00000100:00000200:15.0:1616513074.191286:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704740053056, offset 192 00000400:00000200:20.0:1616513074.191289:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.173@tcp1 00000400:00000200:15.0:1616513074.191291:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.173@tcp1 00000400:00000200:20.0:1616513074.191295:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.173@tcp1 routed destination 00000400:00000200:15.0:1616513074.191297:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.173@tcp1 routed destination 00000400:00000200:20.0:1616513074.191312:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.173@tcp1(10.31.136.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:15.0:1616513074.191313:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.173@tcp1(10.31.136.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.191319:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.191319:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.191324:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.191325:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.191327:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.191328:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.191329:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.191330:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.191332:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513074.191334:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513074.191336:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513074.191338:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513074.191338:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513074.191346:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.191350:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513074.191352:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.191354:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513074.191358:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.191363:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77746e8 00000800:00000200:12.0:1616513074.191366:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513074.191390:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.191398:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.191401:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513074.191402:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513074.191407:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.191411:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22990 00000800:00000200:26.0:1616513074.191415:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.194699:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513074.194753:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.194762:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.194771:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.19@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.194780:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.19@tcp1 of length 224 into portal 26 MB=0x5ffd43fc750c0 00000400:00000200:8.0:1616513074.194787:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.19@tcp1 of length 224/224 into md 0xf97c5929 [1] + 0 00000400:00000200:8.0:1616513074.194792:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241cb0 00000400:00000200:8.0:1616513074.194795:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.194800:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513074.194805:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241cb0 00000800:00000200:28.0:1616513074.194806:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.194808:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.194810:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.194812:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.194866:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661951729856 00010000:00000200:15.0:1616513074.194876:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688661951729856/t0(0) o400->6d885dac-6066-4115-e1f8-c3b50e532469@10.31.133.19@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.194890:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661951729856, offset 192 00000400:00000200:15.0:1616513074.194895:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.19@tcp1 00000400:00000200:15.0:1616513074.194903:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.19@tcp1 routed destination 00000400:00000200:15.0:1616513074.194918:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.19@tcp1(10.31.133.19@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.194924:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.194930:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.194933:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.194935:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.194940:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513074.194946:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.194946:0:48822:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.195000:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.195007:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.195010:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.195017:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.195024:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774d48 00000800:00000200:14.0:1616513074.195027:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.195030:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.195053:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513074.195107:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.195117:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.195125:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.19@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.195134:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.19@tcp1 of length 224 into portal 12 MB=0x5ffd43fc75280 00000400:00000200:0.0:1616513074.195141:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.19@tcp1 of length 224/224 into md 0x865b9f31 [1] + 74552 00000400:00000200:0.0:1616513074.195151:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.195153:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513074.195159:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.195159:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.195163:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.195164:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.195220:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661951730304 00010000:00000200:20.0:1616513074.195231:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5e80 x1688661951730304/t0(0) o400->1437af55-3b45-50b4-e276-cffce3f3c1f6@10.31.133.19@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.195245:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661951730304, offset 192 00000400:00000200:20.0:1616513074.195251:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.19@tcp1 00000400:00000200:20.0:1616513074.195257:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.19@tcp1 routed destination 00000400:00000200:20.0:1616513074.195272:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.19@tcp1(10.31.133.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.195278:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.195283:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.195286:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.195289:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.195294:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.195299:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.195310:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.195316:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.195318:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.195320:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.195324:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.195328:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22088 00000800:00000200:28.0:1616513074.195332:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.195394:0:48822:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.195452:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.195461:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.195471:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.101@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.195480:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.101@tcp1 of length 224 into portal 26 MB=0x60595e0e17100 00000400:00000200:2.0:1616513074.195487:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.101@tcp1 of length 224/224 into md 0xf97c5949 [1] + 0 00000800:00000200:11.2:1616513074.195491:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513074.195492:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240aa0 00000400:00000200:2.0:1616513074.195495:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.195498:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513074.195501:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513074.195503:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240aa0 00000800:00000200:26.0:1616513074.195506:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.195506:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513074.195508:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513074.195509:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.195511:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513074.195511:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513074.195517:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694991141400832 00000400:00000200:12.0:1616513074.195517:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.101@tcp1 : PUT - for me 00010000:00000200:15.0:1616513074.195523:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1694991141400832/t0(0) o400->ee04726a-a802-4936-aaed-a633f7ecd23b@10.31.141.101@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513074.195525:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.101@tcp1 of length 224 into portal 12 MB=0x60595e0e173c0 00000400:00000200:12.0:1616513074.195531:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.101@tcp1 of length 224/224 into md 0x865b9f31 [1] + 74776 00000100:00000200:15.0:1616513074.195534:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694991141400832, offset 192 00000400:00000200:12.0:1616513074.195537:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:15.0:1616513074.195539:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.101@tcp1 00000100:00000200:12.0:1616513074.195539:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:15.0:1616513074.195545:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.101@tcp1 routed destination 00000800:00000200:12.0:1616513074.195545:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.195548:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.195550:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.195557:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694991141401536 00000400:00000200:15.0:1616513074.195558:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.101@tcp1(10.31.141.101@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:20.0:1616513074.195563:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc578000 x1694991141401536/t0(0) o400->c65ee785-46ce-4cbe-ada2-48254bf16804@10.31.141.101@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513074.195564:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.195569:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.195572:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.195573:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.195574:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694991141401536, offset 192 00000800:00000200:15.0:1616513074.195577:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513074.195579:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.101@tcp1 00000100:00000200:15.0:1616513074.195581:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.195581:0:48822:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513074.195585:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.101@tcp1 routed destination 00000800:00000200:14.0:1616513074.195592:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.195596:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513074.195597:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.101@tcp1(10.31.141.101@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513074.195598:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513074.195602:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513074.195603:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513074.195604:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513074.195607:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513074.195608:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775dc0 00000800:00000200:20.0:1616513074.195610:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.195611:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513074.195613:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513074.195614:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.195619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.195628:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.195631:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.195634:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513074.195636:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.195639:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.195643:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23e48 00000800:00000200:8.0:1616513074.195645:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.196460:0:48822:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.196472:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.196480:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513074.196483:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513074.196489:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.148@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.196499:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.148@tcp1 of length 224 into portal 26 MB=0x5ffdcb0a16180 00000400:00000200:26.0:1616513074.196505:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.148@tcp1 of length 224/224 into md 0xf97c5979 [1] + 0 00000400:00000200:26.0:1616513074.196510:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782402a8 00000400:00000200:26.0:1616513074.196513:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.196516:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.196521:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782402a8 00000800:00000200:26.0:1616513074.196524:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.196527:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.196528:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.196534:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698204807552 00010000:00000200:15.0:1616513074.196539:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688698204807552/t0(0) o400->edf6e12e-ef96-2feb-092f-868379edb889@10.31.131.148@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.196549:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698204807552, offset 192 00000400:00000200:15.0:1616513074.196554:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.148@tcp1 00000400:00000200:15.0:1616513074.196560:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.148@tcp1 routed destination 00000400:00000200:15.0:1616513074.196572:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.148@tcp1(10.31.131.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.196578:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.196582:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.196585:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.196587:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.196590:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513074.196594:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.196594:0:48822:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.196604:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.196610:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.196613:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513074.196614:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.196620:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.196626:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775ba0 00000800:00000200:22.0:1616513074.196630:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.196806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513074.196860:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.196869:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.196878:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.148@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.196886:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.148@tcp1 of length 224 into portal 12 MB=0x5ffdcb0a166c0 00000400:00000200:0.0:1616513074.196893:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.148@tcp1 of length 224/224 into md 0x865b9f31 [1] + 75000 00000400:00000200:0.0:1616513074.196898:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.196900:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513074.196907:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.196910:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.196912:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.196915:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.196968:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698204808896 00010000:00000200:20.0:1616513074.196980:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5a00 x1688698204808896/t0(0) o400->4b9f0925-4b31-0326-284d-db7817f901e1@10.31.131.148@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.196994:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698204808896, offset 192 00000400:00000200:20.0:1616513074.197000:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.148@tcp1 00000400:00000200:20.0:1616513074.197007:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.148@tcp1 routed destination 00000400:00000200:20.0:1616513074.197021:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.148@tcp1(10.31.131.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.197028:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.197033:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.197036:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.197038:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.197042:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.197047:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.197099:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.197106:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.197110:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513074.197116:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.197120:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ee0 00000800:00000200:28.0:1616513074.197123:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.197126:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.198750:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513074.198803:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.198812:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513074.198821:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.5@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.198831:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.5@tcp1 of length 224 into portal 26 MB=0x60002097d4e80 00000400:00000200:12.0:1616513074.198838:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.5@tcp1 of length 224/224 into md 0xf97c5989 [1] + 0 00000400:00000200:12.0:1616513074.198843:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240a18 00000400:00000200:12.0:1616513074.198845:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.198848:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513074.198853:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240a18 00000800:00000200:12.0:1616513074.198856:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.198857:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.198859:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.198864:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.198867:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.198868:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.198877:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.5@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.198885:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.5@tcp1 of length 224 into portal 12 MB=0x60002097d5140 00000400:00000200:0.0:1616513074.198892:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.5@tcp1 of length 224/224 into md 0x865b9f31 [1] + 75224 00000400:00000200:0.0:1616513074.198897:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.198900:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513074.198907:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.198910:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.198912:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.198915:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688858609405568 00010000:00000200:15.0:1616513074.198925:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688858609405568/t0(0) o400->2175d7ff-e247-7719-6f09-debc3b1b60ab@10.31.142.5@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.198939:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688858609405568, offset 192 00000400:00000200:15.0:1616513074.198945:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.5@tcp1 00000400:00000200:15.0:1616513074.198952:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.5@tcp1 routed destination 00000400:00000200:15.0:1616513074.198966:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.5@tcp1(10.31.142.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:20.0:1616513074.198971:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688858609406272 00000800:00000200:15.0:1616513074.198972:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.198978:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.198981:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.198982:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:20.0:1616513074.198983:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b193600 x1688858609406272/t0(0) o400->c291b7ff-a7c9-ec48-7228-774d27d45661@10.31.142.5@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513074.198987:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.198992:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.198994:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:20.0:1616513074.198998:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688858609406272, offset 192 00000400:00000200:20.0:1616513074.199006:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.5@tcp1 00000400:00000200:20.0:1616513074.199013:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.5@tcp1 routed destination 00000400:00000200:20.0:1616513074.199029:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.5@tcp1(10.31.142.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.199035:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.199041:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.199044:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.199046:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513074.199047:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.199061:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513074.199062:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.199062:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513074.199064:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.199071:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.199076:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77747f8 00000800:00000200:2.0:1616513074.199080:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.199083:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513074.199121:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.199128:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.199130:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513074.199131:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.199137:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.199141:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22f68 00000800:00000200:8.0:1616513074.199145:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.203397:0:48823:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.203452:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.203462:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.203471:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.76@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.203479:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.76@tcp1 of length 224 into portal 26 MB=0x5ffe5836ec640 00000400:00000200:14.0:1616513074.203486:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.76@tcp1 of length 224/224 into md 0xf97c5999 [1] + 0 00000400:00000200:14.0:1616513074.203491:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240dd0 00000400:00000200:14.0:1616513074.203493:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.203496:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513074.203501:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240dd0 00000800:00000200:14.0:1616513074.203504:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.203505:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.203507:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.203509:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.203567:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688736101221952 00010000:00000200:15.0:1616513074.203578:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688736101221952/t0(0) o400->f505842b-4c57-481a-86be-324a9e983b02@10.31.141.76@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.203592:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688736101221952, offset 192 00000400:00000200:15.0:1616513074.203599:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.76@tcp1 00000400:00000200:15.0:1616513074.203606:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.76@tcp1 routed destination 00000800:00000200:7.2:1616513074.203615:0:48823:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:15.0:1616513074.203634:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.76@tcp1(10.31.141.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.203639:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.203654:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.203657:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.203659:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513074.203663:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:15.0:1616513074.203669:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513074.203676:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.203685:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.203694:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.76@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.203703:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.76@tcp1 of length 224 into portal 12 MB=0x5ffe5836ec740 00000400:00000200:2.0:1616513074.203710:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.76@tcp1 of length 224/224 into md 0xf96450b9 [1] + 39440 00000400:00000200:2.0:1616513074.203715:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.203717:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513074.203724:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513074.203728:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513074.203728:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513074.203736:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513074.203738:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513074.203740:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513074.203741:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.203747:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.203753:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774330 00000800:00000200:26.0:1616513074.203756:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513074.203791:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688736101222208 00010000:00000200:30.0:1616513074.203803:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de138ba80 x1688736101222208/t0(0) o400->39fec0f1-6661-9814-6be8-b5578018ba4f@10.31.141.76@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.203818:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688736101222208, offset 192 00000400:00000200:30.0:1616513074.203825:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.76@tcp1 00000400:00000200:30.0:1616513074.203832:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.76@tcp1 routed destination 00000400:00000200:30.0:1616513074.203847:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.76@tcp1(10.31.141.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.203854:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.203860:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.203863:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513074.203865:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.203869:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.203877:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.203930:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.203936:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.203938:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513074.203945:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.203948:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efcb0 00000800:00000200:28.0:1616513074.203952:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.203955:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.224478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.224532:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.224542:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.224551:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.136@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.224560:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.136@tcp1 of length 224 into portal 26 MB=0x5ffd7f7506c00 00000400:00000200:14.0:1616513074.224567:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.136@tcp1 of length 224/224 into md 0xf97c59b9 [1] + 0 00000400:00000200:14.0:1616513074.224572:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241210 00000400:00000200:14.0:1616513074.224575:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.224577:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513074.224582:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241210 00000800:00000200:14.0:1616513074.224585:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.224586:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.224588:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.224592:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.224648:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688677915847680 00010000:00000200:15.0:1616513074.224658:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688677915847680/t0(0) o400->4556e14a-520c-86cd-8364-7e009a0f6e76@10.31.129.136@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.224671:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688677915847680, offset 192 00000400:00000200:15.0:1616513074.224676:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.136@tcp1 00000400:00000200:15.0:1616513074.224683:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.136@tcp1 routed destination 00000400:00000200:15.0:1616513074.224697:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.136@tcp1(10.31.129.136@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.224702:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.224708:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.224710:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.224712:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.224716:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.224721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.224722:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513074.224774:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.224781:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.224784:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.224791:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.224797:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775b18 00000800:00000200:22.0:1616513074.224801:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.224804:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.225281:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.225293:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.225301:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513074.225303:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513074.225310:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.136@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.225319:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.136@tcp1 of length 224 into portal 12 MB=0x5ffd7f7506d80 00000400:00000200:2.0:1616513074.225326:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.136@tcp1 of length 224/224 into md 0xf96450b9 [1] + 39664 00000400:00000200:2.0:1616513074.225331:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.225334:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513074.225341:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.225344:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.225346:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513074.225404:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688677915848064 00010000:00000200:30.0:1616513074.225417:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120bf00 x1688677915848064/t0(0) o400->2772fd00-1ff6-fd25-b26a-e4a3fc82c51d@10.31.129.136@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.225465:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688677915848064, offset 192 00000400:00000200:30.0:1616513074.225472:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.136@tcp1 00000400:00000200:30.0:1616513074.225480:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.136@tcp1 routed destination 00000400:00000200:30.0:1616513074.225495:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.136@tcp1(10.31.129.136@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.225502:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.225508:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.225511:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513074.225513:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.225518:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.225525:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.225578:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.225585:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513074.225587:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513074.225593:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.225597:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef4b8 00000800:00000200:12.0:1616513074.225601:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.225603:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.233720:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.233774:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.233783:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513074.233791:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.145@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.233799:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.145@tcp1 of length 224 into portal 26 MB=0x5ffd8cf8da7c0 00000400:00000200:28.0:1616513074.233806:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.145@tcp1 of length 224/224 into md 0xf97c59c9 [1] + 0 00000400:00000200:28.0:1616513074.233810:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240ee0 00000400:00000200:28.0:1616513074.233813:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.233815:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513074.233820:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ee0 00000800:00000200:28.0:1616513074.233822:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.233825:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.233827:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.233830:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.233907:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688681543739328 00010000:00000200:15.0:1616513074.233916:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688681543739328/t0(0) o400->2cd1c497-083e-8641-8356-8f67d090f2bf@10.31.131.145@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.233929:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688681543739328, offset 192 00000400:00000200:15.0:1616513074.233934:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.145@tcp1 00000400:00000200:15.0:1616513074.233941:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.145@tcp1 routed destination 00000400:00000200:15.0:1616513074.233954:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.145@tcp1(10.31.131.145@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.233960:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.233966:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.233968:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.233970:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.233974:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513074.233980:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.233981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513074.234033:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513074.234035:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.234043:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.234046:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.234052:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.234059:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774908 00000800:00000200:22.0:1616513074.234063:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.234066:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513074.234085:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.234095:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513074.234103:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.145@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.234112:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.145@tcp1 of length 224 into portal 12 MB=0x5ffd8cf8da9c0 00000400:00000200:12.0:1616513074.234118:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.145@tcp1 of length 224/224 into md 0x865b9f31 [1] + 75448 00000400:00000200:12.0:1616513074.234123:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.234126:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513074.234132:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.234135:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.234137:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.234140:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.234201:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688681543739840 00010000:00000200:20.0:1616513074.234214:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc578480 x1688681543739840/t0(0) o400->4fedd9c9-01eb-0abd-1d77-8f8965adc6b3@10.31.131.145@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.234228:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688681543739840, offset 192 00000400:00000200:20.0:1616513074.234234:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.145@tcp1 00000400:00000200:20.0:1616513074.234242:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.145@tcp1 routed destination 00000400:00000200:20.0:1616513074.234257:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.145@tcp1(10.31.131.145@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.234264:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.234270:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.234272:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.234274:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.234279:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.234284:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.234294:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.234300:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.234302:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513074.234304:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513074.234309:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.234313:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22330 00000800:00000200:0.0:1616513074.234318:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.235018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513074.235028:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.235034:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513074.235037:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513074.235041:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.131@tcp1 : PUT - for me 00000800:00000200:8.0:1616513074.235046:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513074.235047:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513074.235048:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.131@tcp1 of length 224 into portal 26 MB=0x5fff3b76381c0 00000400:00000200:12.0:1616513074.235054:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.131@tcp1 of length 224/224 into md 0xf97c59d9 [1] + 0 00000400:00000200:8.0:1616513074.235054:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.131@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.235060:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241320 00000400:00000200:12.0:1616513074.235062:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.235063:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.131@tcp1 of length 224 into portal 12 MB=0x5fff3b7638240 00000100:00000200:12.0:1616513074.235065:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513074.235069:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241320 00000400:00000200:8.0:1616513074.235069:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.131@tcp1 of length 224/224 into md 0x865b9f31 [1] + 75672 00000800:00000200:12.0:1616513074.235071:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.235073:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.235075:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.235075:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.235077:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513074.235084:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688797102440896 00000800:00000200:8.0:1616513074.235085:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.235088:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.235089:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:15.0:1616513074.235090:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688797102440896/t0(0) o400->76fee377-fc0e-dc19-4177-b6d1e517365c@10.31.142.131@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.235097:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688797102441024 00000100:00000200:15.0:1616513074.235102:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688797102440896, offset 192 00010000:00000200:20.0:1616513074.235103:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4800 x1688797102441024/t0(0) o400->6293f39a-d7e1-9036-2574-e1d8065f28fc@10.31.142.131@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513074.235109:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.131@tcp1 00000100:00000200:20.0:1616513074.235115:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688797102441024, offset 192 00000400:00000200:15.0:1616513074.235116:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.131@tcp1 routed destination 00000400:00000200:20.0:1616513074.235120:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.131@tcp1 00000400:00000200:20.0:1616513074.235127:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.131@tcp1 routed destination 00000400:00000200:15.0:1616513074.235131:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.131@tcp1(10.31.142.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.235138:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513074.235142:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.131@tcp1(10.31.142.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.235143:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.235146:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.235148:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.235149:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.235151:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513074.235154:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:15.0:1616513074.235156:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.235156:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.235158:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.235160:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.235163:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.235168:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.235177:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.235181:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.235183:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513074.235185:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513074.235188:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.235191:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee226e8 00000800:00000200:0.0:1616513074.235193:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513074.235222:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.235229:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.235232:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.235239:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.235245:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775870 00000800:00000200:14.0:1616513074.235249:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.235254:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.254309:0:48829:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.254365:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.254374:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.254384:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.40@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.254393:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.40@tcp1 of length 224 into portal 12 MB=0x604f6266ae7c0 00000400:00000200:22.0:1616513074.254400:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.40@tcp1 of length 224/224 into md 0xf96450b9 [1] + 39888 00000400:00000200:22.0:1616513074.254405:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.254408:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513074.254414:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.254417:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.254417:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.254421:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.254474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:30.0:1616513074.254477:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694305113270208 00010000:00000200:30.0:1616513074.254489:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad7500 x1694305113270208/t0(0) o400->470c2403-5e0c-1660-74ba-8f4174e7b9fe@10.31.142.40@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.254504:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694305113270208, offset 192 00000400:00000200:30.0:1616513074.254511:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.40@tcp1 00000400:00000200:30.0:1616513074.254519:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.40@tcp1 routed destination 00000400:00000200:30.0:1616513074.254533:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.40@tcp1(10.31.142.40@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.254540:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513074.254540:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.254546:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.254548:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513074.254549:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.254550:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513074.254555:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513074.254558:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.40@tcp1 : PUT - for me 00000800:00000200:7.2:1616513074.254559:0:48829:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513074.254567:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.40@tcp1 of length 224 into portal 26 MB=0x604f6266ae740 00000400:00000200:12.0:1616513074.254574:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.40@tcp1 of length 224/224 into md 0xf97c5a19 [1] + 0 00000400:00000200:12.0:1616513074.254578:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241c28 00000400:00000200:12.0:1616513074.254581:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.254584:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513074.254589:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241c28 00000800:00000200:12.0:1616513074.254592:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.254594:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.254595:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.254599:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513074.254618:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.254625:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.254628:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513074.254631:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.254635:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.254639:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef3a8 00000800:00000200:14.0:1616513074.254642:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.254652:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694305113270080 00010000:00000200:15.0:1616513074.254662:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1694305113270080/t0(0) o400->e4450eda-580f-fe4e-ab80-93f17d2d74ab@10.31.142.40@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.254675:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694305113270080, offset 192 00000400:00000200:15.0:1616513074.254681:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.40@tcp1 00000400:00000200:15.0:1616513074.254688:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.40@tcp1 routed destination 00000400:00000200:15.0:1616513074.254703:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.40@tcp1(10.31.142.40@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.254709:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.254715:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.254717:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.254719:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.254724:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.254729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.254730:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513074.254782:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.254789:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.254792:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513074.254799:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.254805:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774770 00000800:00000200:28.0:1616513074.254809:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.254812:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:3.0:1616513074.266517:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924432, portal 4 00000100:00000200:3.0:1616513074.266522:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924432, offset 0 00000400:00000200:3.0:1616513074.266529:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:3.0:1616513074.266535:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:3.0:1616513074.266548:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513074.266554:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:3.0:1616513074.266560:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513074.266563:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000100:00000200:22.0:1616513074.266565:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924448, portal 4 00000800:00000200:3.0:1616513074.266567:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:22.0:1616513074.266571:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924448, offset 0 00000800:00000200:3.0:1616513074.266572:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:22.0:1616513074.266577:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:27.2:1616513074.266579:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:22.0:1616513074.266583:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:27.0:1616513074.266593:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:22.0:1616513074.266597:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513074.266598:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513074.266600:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513074.266603:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000400:00000200:27.0:1616513074.266607:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:22.0:1616513074.266609:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513074.266612:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000100:00000200:27.0:1616513074.266613:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63c380 x1693067725924432/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513074.266615:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:22.0:1616513074.266619:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:27.0:1616513074.266625:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2000 00000800:00000200:27.0:1616513074.266629:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513074.266631:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:5.2:1616513074.266670:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513074.266727:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513074.266734:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513074.266737:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.266743:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513074.266752:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a78480 x1693067725924448/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513074.266764:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b210 00000800:00000200:8.0:1616513074.266768:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513074.266771:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.2:1616513074.266783:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513074.266792:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.266798:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513074.266802:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513074.266806:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513074.266814:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626850 00000400:00000200:27.0:1616513074.266821:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716595d [1] + 192 00000400:00000200:27.0:1616513074.266825:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513074.266829:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63c380 x1693067725924432/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513074.266843:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.266845:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513074.266847:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:3.0:1616513074.266850:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63c380 x1693067725924432/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:5.2:1616513074.266854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000100:00000200:3.0:1616513074.266863:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63c380 x1693067725924432/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513074.266873:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:28.0:1616513074.266912:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513074.266921:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:28.0:1616513074.266931:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513074.266939:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626860 00000400:00000200:28.0:1616513074.266945:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d3cc1 [1] + 192 00000400:00000200:28.0:1616513074.266951:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.266956:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a78480 x1693067725924448/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:0.0:1616513074.266969:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513074.266970:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513074.266973:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513074.266975:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:22.0:1616513074.266979:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a78480 x1693067725924448/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513074.266992:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a78480 x1693067725924448/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513074.267002:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:31.2:1616513074.278389:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:26.0:1616513074.278450:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:26.0:1616513074.278459:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:26.0:1616513074.278467:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513074.278475:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 224 into portal 24 MB=0x603db0154bcd0 00000400:00000200:26.0:1616513074.278482:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 18 from 12345-10.242.46.234@o2ib1 of length 224/224 into md 0xf9733ed9 [1] + 0 00000400:00000200:26.0:1616513074.278487:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d4553e826e8 00000400:00000200:26.0:1616513074.278490:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.278493:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt_out 00000400:00000200:26.0:1616513074.278498:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e826e8 00000800:00000200:26.0:1616513074.278501:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:22.0:1616513074.278503:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:26.0:1616513074.278504:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:26.0:1616513074.278508:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:6.0:1616513074.278522:0:185125:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693089015315664 00010000:00000200:6.0:1616513074.278544:0:185125:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d741ca81050 x1693089015315664/t0(0) o41->boslfs02-MDT0000-mdtlov_UUID@10.242.46.234@o2ib1:700/0 lens 224/368 e 0 to 0 dl 1616513080 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:6.0:1616513074.278558:0:185125:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 4, xid 1693089015315664, offset 192 00000400:00000200:6.0:1616513074.278565:0:185125:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:6.0:1616513074.278572:0:185125:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:6.0:1616513074.278586:0:185125:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513074.278592:0:185125:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:6.0:1616513074.278598:0:185125:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513074.278601:0:185125:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:6.0:1616513074.278603:0:185125:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:6.0:1616513074.278608:0:185125:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513074.278613:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:6.0:1616513074.278616:0:185125:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000800:00000200:2.0:1616513074.278667:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:2.0:1616513074.278673:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:2.0:1616513074.278676:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.278683:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.278688:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01430 00000800:00000200:2.0:1616513074.278692:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:2.0:1616513074.278695:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:7.2:1616513074.279193:0:48829:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.279206:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.279214:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513074.279215:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.279223:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.55@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.279233:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.55@tcp1 of length 224 into portal 26 MB=0x5ffd4ccf62f00 00000400:00000200:22.0:1616513074.279240:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.55@tcp1 of length 224/224 into md 0xf97c5a29 [1] + 0 00000400:00000200:22.0:1616513074.279244:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241e48 00000400:00000200:22.0:1616513074.279249:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.279252:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513074.279257:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241e48 00000800:00000200:22.0:1616513074.279260:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.279263:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.279265:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.279322:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664320388864 00010000:00000200:15.0:1616513074.279333:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688664320388864/t0(0) o400->cde7966b-8814-287b-9086-bd7740c82573@10.31.129.55@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.279347:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664320388864, offset 192 00000400:00000200:15.0:1616513074.279354:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.55@tcp1 00000400:00000200:15.0:1616513074.279362:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.55@tcp1 routed destination 00000400:00000200:15.0:1616513074.279378:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.55@tcp1(10.31.129.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.279384:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.279393:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.279396:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.279398:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.279403:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.279408:0:48829:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.279411:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513074.279417:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.279422:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.279424:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513074.279427:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513074.279429:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.279445:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774198 00000800:00000200:2.0:1616513074.279448:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.279602:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513074.279657:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.279671:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.279680:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.55@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.279689:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.55@tcp1 of length 224 into portal 12 MB=0x5ffd4ccf63440 00000400:00000200:8.0:1616513074.279696:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.55@tcp1 of length 224/224 into md 0x865b9f31 [1] + 75896 00000400:00000200:8.0:1616513074.279701:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.279703:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513074.279710:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.279711:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.279713:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.279715:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.279777:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664320390208 00010000:00000200:20.0:1616513074.279789:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3180 x1688664320390208/t0(0) o400->524eb99e-768e-f190-41e4-0cd7f3cfb0f7@10.31.129.55@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.279803:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664320390208, offset 192 00000400:00000200:20.0:1616513074.279810:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.55@tcp1 00000400:00000200:20.0:1616513074.279818:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.55@tcp1 routed destination 00000400:00000200:20.0:1616513074.279832:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.55@tcp1(10.31.129.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.279839:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.279845:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.279848:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.279850:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.279855:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.279859:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.279871:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.279878:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.279881:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.279887:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.279891:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22aa0 00000800:00000200:0.0:1616513074.279895:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.279898:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.292472:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513074.292524:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.292533:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513074.292542:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.90@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.292552:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.90@tcp1 of length 224 into portal 26 MB=0x60051a7597a80 00000400:00000200:12.0:1616513074.292559:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.90@tcp1 of length 224/224 into md 0xf97c5a49 [1] + 0 00000400:00000200:12.0:1616513074.292564:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782406e8 00000400:00000200:12.0:1616513074.292567:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.292570:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513074.292574:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782406e8 00000800:00000200:12.0:1616513074.292577:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.292577:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.292580:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.292582:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.292639:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689200560274048 00010000:00000200:15.0:1616513074.292650:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1689200560274048/t0(0) o400->8be1b25c-c93f-cc59-0e64-a68c0df52f36@10.31.136.90@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.292664:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689200560274048, offset 192 00000400:00000200:15.0:1616513074.292670:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.90@tcp1 00000400:00000200:15.0:1616513074.292679:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.90@tcp1 routed destination 00000400:00000200:15.0:1616513074.292694:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.90@tcp1(10.31.136.90@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.292701:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.292707:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.292710:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.292713:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.292718:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.292723:0:48829:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.292724:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513074.292776:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.292783:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.292786:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.292793:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.292799:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775870 00000800:00000200:26.0:1616513074.292802:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.292805:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.292865:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513074.292918:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.292928:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.292936:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.90@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.292945:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.90@tcp1 of length 224 into portal 12 MB=0x60051a7597c40 00000400:00000200:0.0:1616513074.292952:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.90@tcp1 of length 224/224 into md 0x865b9f31 [1] + 76120 00000400:00000200:0.0:1616513074.292957:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.292959:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513074.292966:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.292969:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.292971:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.292974:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.293028:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689200560274496 00010000:00000200:20.0:1616513074.293039:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b197980 x1689200560274496/t0(0) o400->a1521856-b976-15b2-f5fd-7e8d8e3be3c3@10.31.136.90@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.293052:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689200560274496, offset 192 00000400:00000200:20.0:1616513074.293058:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.90@tcp1 00000400:00000200:20.0:1616513074.293065:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.90@tcp1 routed destination 00000400:00000200:20.0:1616513074.293080:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.90@tcp1(10.31.136.90@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.293086:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.293092:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.293094:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.293096:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.293101:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.293106:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.293116:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.293122:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.293124:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513074.293126:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.293131:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.293135:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23650 00000800:00000200:8.0:1616513074.293139:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.295803:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513074.295856:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.295865:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.295874:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.151@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.295883:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.151@tcp1 of length 224 into portal 26 MB=0x603e13bde3440 00000400:00000200:0.0:1616513074.295890:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.151@tcp1 of length 224/224 into md 0xf97c5a69 [1] + 0 00000400:00000200:0.0:1616513074.295895:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782417e8 00000400:00000200:0.0:1616513074.295897:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.295900:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513074.295907:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782417e8 00000800:00000200:28.0:1616513074.295909:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.295910:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.295913:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.295915:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.295968:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115767206976 00010000:00000200:15.0:1616513074.295979:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1693115767206976/t0(0) o400->c7724465-873a-0b89-032b-1f8ffcc755e3@10.31.136.151@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.295992:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693115767206976, offset 192 00000400:00000200:15.0:1616513074.295998:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.151@tcp1 00000400:00000200:15.0:1616513074.296005:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.151@tcp1 routed destination 00000400:00000200:15.0:1616513074.296020:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.151@tcp1(10.31.136.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.296026:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.296032:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.296034:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.296036:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.296041:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.296045:0:48829:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.296049:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513074.296099:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.296107:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.296110:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.296116:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.296122:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775b18 00000800:00000200:14.0:1616513074.296126:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.296129:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.296198:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513074.296250:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.296259:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513074.296268:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.151@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.296276:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.151@tcp1 of length 224 into portal 12 MB=0x603e13bde3580 00000400:00000200:12.0:1616513074.296283:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.151@tcp1 of length 224/224 into md 0x865b9f31 [1] + 76344 00000400:00000200:12.0:1616513074.296288:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513074.296290:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513074.296297:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.296300:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.296302:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.296304:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.296361:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115767207296 00010000:00000200:20.0:1616513074.296373:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57ad00 x1693115767207296/t0(0) o400->990c94b6-1ab2-a258-15e0-035dfcb7a042@10.31.136.151@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.296398:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693115767207296, offset 192 00000400:00000200:20.0:1616513074.296404:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.151@tcp1 00000400:00000200:20.0:1616513074.296411:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.151@tcp1 routed destination 00000400:00000200:20.0:1616513074.296426:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.151@tcp1(10.31.136.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.296445:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.296451:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.296454:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.296456:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.296461:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.296466:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.296477:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.296482:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.296484:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.296487:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.296491:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.296495:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23100 00000800:00000200:28.0:1616513074.296499:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.313441:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.313495:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.313504:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.313513:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.18@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.313522:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.18@tcp1 of length 224 into portal 26 MB=0x5ffd35e236bc0 00000400:00000200:26.0:1616513074.313529:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.18@tcp1 of length 224/224 into md 0xf97c5a79 [1] + 0 00000400:00000200:26.0:1616513074.313533:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42990 00000400:00000200:26.0:1616513074.313536:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.313541:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.313546:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42990 00000800:00000200:26.0:1616513074.313549:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.313549:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.313552:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.313553:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.313610:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658166115264 00010000:00000200:15.0:1616513074.313620:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688658166115264/t0(0) o400->ecac4034-9862-b757-476f-9d1b94203985@10.31.133.18@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.313634:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658166115264, offset 192 00000400:00000200:15.0:1616513074.313640:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.18@tcp1 00000400:00000200:15.0:1616513074.313647:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.18@tcp1 routed destination 00000400:00000200:15.0:1616513074.313661:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.18@tcp1(10.31.133.18@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.313667:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.313673:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.313676:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.313678:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.313683:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.313686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.313689:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513074.313739:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.313747:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.313750:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.313757:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.313763:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77747f8 00000800:00000200:14.0:1616513074.313766:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.313769:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.313775:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513074.313777:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.313783:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.18@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.313791:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.18@tcp1 of length 224 into portal 12 MB=0x5ffd35e236cc0 00000400:00000200:14.0:1616513074.313797:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.18@tcp1 of length 224/224 into md 0xf96450b9 [1] + 40112 00000400:00000200:14.0:1616513074.313802:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.313805:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.313810:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.313813:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.313815:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513074.313876:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658166115520 00010000:00000200:30.0:1616513074.313888:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53ca880 x1688658166115520/t0(0) o400->b0487875-2027-ab85-8594-6dc919876239@10.31.133.18@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.313903:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658166115520, offset 192 00000400:00000200:30.0:1616513074.313909:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.18@tcp1 00000400:00000200:30.0:1616513074.313917:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.18@tcp1 routed destination 00000400:00000200:30.0:1616513074.313931:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.18@tcp1(10.31.133.18@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.313940:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.313946:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.313949:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513074.313951:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.313956:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.313961:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.314013:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.314020:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513074.314022:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513074.314029:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.314033:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee908 00000800:00000200:12.0:1616513074.314037:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.314040:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:27.0:1616513074.314565:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924464, portal 4 00000100:00000200:27.0:1616513074.314571:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924464, offset 0 00000400:00000200:27.0:1616513074.314577:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:27.0:1616513074.314585:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:27.0:1616513074.314599:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513074.314605:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:27.0:1616513074.314612:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513074.314615:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:27.0:1616513074.314617:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:27.0:1616513074.314623:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513074.314669:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513074.314688:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513074.314693:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513074.314695:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513074.314696:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:26.0:1616513074.314701:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513074.314707:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600d80 x1693067725924464/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513074.314719:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249a90 00000800:00000200:26.0:1616513074.314723:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513074.314851:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513074.314865:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513074.314875:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:14.0:1616513074.314878:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513074.314884:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513074.314892:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626870 00000400:00000200:2.0:1616513074.314899:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c6039 [1] + 192 00000400:00000200:2.0:1616513074.314904:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.314910:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600d80 x1693067725924464/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513074.314925:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513074.314929:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513074.314932:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:27.0:1616513074.314935:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600d80 x1693067725924464/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513074.314948:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600d80 x1693067725924464/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513074.314959:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248e58 00000800:00000200:11.2:1616513074.316072:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513074.316131:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.316140:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.316149:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.147@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.316158:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.147@tcp1 of length 224 into portal 26 MB=0x5ffd78e390980 00000400:00000200:0.0:1616513074.316165:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.147@tcp1 of length 224/224 into md 0xf97c5a89 [1] + 0 00000400:00000200:0.0:1616513074.316170:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a436d8 00000400:00000200:0.0:1616513074.316173:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.316176:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513074.316181:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a436d8 00000800:00000200:28.0:1616513074.316184:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.316184:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.316187:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.316188:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.316217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513074.316245:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676152707456 00010000:00000200:15.0:1616513074.316256:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688676152707456/t0(0) o400->dea41fd8-e60b-ef82-4706-40ae9da394da@10.31.129.147@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513074.316277:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.316281:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688676152707456, offset 192 00000400:00000200:15.0:1616513074.316286:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.147@tcp1 00000800:00000200:12.0:1616513074.316287:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:15.0:1616513074.316294:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.147@tcp1 routed destination 00000400:00000200:12.0:1616513074.316296:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.147@tcp1 : PUT - for me 00000400:00000200:12.0:1616513074.316305:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.147@tcp1 of length 224 into portal 12 MB=0x5ffd78e390e00 00000400:00000200:15.0:1616513074.316308:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.147@tcp1(10.31.129.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513074.316311:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.147@tcp1 of length 224/224 into md 0x865b9f31 [1] + 76568 00000800:00000200:15.0:1616513074.316314:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513074.316317:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:15.0:1616513074.316319:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:12.0:1616513074.316319:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:15.0:1616513074.316322:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.316324:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513074.316326:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:15.0:1616513074.316328:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513074.316329:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.316331:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513074.316333:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.316334:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513074.316334:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.316390:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676152708608 00000800:00000200:22.0:1616513074.316393:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.316400:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:20.0:1616513074.316402:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57c380 x1688676152708608/t0(0) o400->ec90872c-7948-3431-54dd-5302cac9c9a6@10.31.129.147@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:22.0:1616513074.316404:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.316410:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:20.0:1616513074.316416:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688676152708608, offset 192 00000400:00000200:22.0:1616513074.316417:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775dc0 00000800:00000200:22.0:1616513074.316421:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.316423:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513074.316423:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.147@tcp1 00000400:00000200:20.0:1616513074.316430:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.147@tcp1 routed destination 00000400:00000200:20.0:1616513074.316459:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.147@tcp1(10.31.129.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.316466:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.316472:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.316475:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.316477:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.316482:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.316487:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.316497:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.316503:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.316505:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.316507:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.316512:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.316516:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22cc0 00000800:00000200:28.0:1616513074.316520:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.319531:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.319584:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.319593:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.319603:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.2@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.319612:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.2@tcp1 of length 224 into portal 26 MB=0x604a468814b00 00000400:00000200:14.0:1616513074.319619:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.2@tcp1 of length 224/224 into md 0xf97c5aa9 [1] + 0 00000400:00000200:14.0:1616513074.319624:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a422a8 00000400:00000200:14.0:1616513074.319626:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.319629:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513074.319634:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a422a8 00000800:00000200:14.0:1616513074.319637:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.319637:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.319640:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.319644:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.319647:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.319648:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.319656:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.2@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.319665:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.2@tcp1 of length 224 into portal 12 MB=0x604a468814d80 00000400:00000200:2.0:1616513074.319671:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.2@tcp1 of length 224/224 into md 0xf96450b9 [1] + 40336 00000400:00000200:2.0:1616513074.319677:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.319679:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513074.319686:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.319689:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.319691:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.319695:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954034715392 00010000:00000200:15.0:1616513074.319704:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1693954034715392/t0(0) o400->cb43a1f7-2b03-fd61-7824-0945869e7aae@10.31.141.2@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.319717:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954034715392, offset 192 00000400:00000200:15.0:1616513074.319724:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.2@tcp1 00000400:00000200:15.0:1616513074.319731:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.2@tcp1 routed destination 00000400:00000200:15.0:1616513074.319744:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.2@tcp1(10.31.141.2@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:30.0:1616513074.319749:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954034716032 00000800:00000200:15.0:1616513074.319750:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.319756:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.319758:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.319760:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:30.0:1616513074.319761:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461209200 x1693954034716032/t0(0) o400->06e75f19-fe2f-5530-8182-3f87356ddd0e@10.31.141.2@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513074.319765:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.319769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.319771:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:30.0:1616513074.319776:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954034716032, offset 192 00000400:00000200:30.0:1616513074.319782:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.2@tcp1 00000400:00000200:30.0:1616513074.319790:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.2@tcp1 routed destination 00000400:00000200:30.0:1616513074.319806:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.2@tcp1(10.31.141.2@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.319812:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.319818:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.319821:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513074.319821:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513074.319823:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.319828:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513074.319828:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.319831:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513074.319832:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513074.319833:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513074.319838:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.319844:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77746e8 00000800:00000200:26.0:1616513074.319847:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513074.319887:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.319894:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.319896:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.319903:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.319907:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee110 00000800:00000200:8.0:1616513074.319911:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.319914:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.319919:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.319927:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.152@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.319936:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.152@tcp1 of length 224 into portal 26 MB=0x5ffd53446b780 00000800:00000200:12.0:1616513074.319944:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:8.0:1616513074.319944:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.152@tcp1 of length 224/224 into md 0xf97c5ac9 [1] + 0 00000400:00000200:8.0:1616513074.319949:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43100 00000400:00000200:8.0:1616513074.319952:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513074.319954:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:8.0:1616513074.319954:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513074.319959:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43100 00000800:00000200:8.0:1616513074.319962:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:12.0:1616513074.319963:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.152@tcp1 : PUT - for me 00000800:00000200:8.0:1616513074.319964:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513074.319966:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.319969:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513074.319972:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666053719936 00000400:00000200:12.0:1616513074.319972:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.152@tcp1 of length 224 into portal 12 MB=0x5ffd53446b880 00010000:00000200:15.0:1616513074.319978:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688666053719936/t0(0) o400->99e720d7-0aa0-969d-95d4-9dcd27a3911d@10.31.132.152@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513074.319980:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.152@tcp1 of length 224/224 into md 0x865b9f31 [1] + 76792 00000400:00000200:12.0:1616513074.319985:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:15.0:1616513074.319988:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666053719936, offset 192 00000100:00000200:12.0:1616513074.319988:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:15.0:1616513074.319993:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.152@tcp1 00000800:00000200:12.0:1616513074.319994:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.319997:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:15.0:1616513074.319999:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.152@tcp1 routed destination 00000800:00000200:12.0:1616513074.319999:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:15.0:1616513074.320011:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.152@tcp1(10.31.132.152@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:20.0:1616513074.320015:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666053720192 00000800:00000200:15.0:1616513074.320016:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.320021:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.320023:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.320025:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:20.0:1616513074.320027:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57da00 x1688666053720192/t0(0) o400->e2071adb-3246-a2e2-ba36-6dc28fc51552@10.31.132.152@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513074.320028:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513074.320032:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.320032:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.320040:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666053720192, offset 192 00000800:00000200:22.0:1616513074.320043:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513074.320046:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.152@tcp1 00000800:00000200:22.0:1616513074.320051:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513074.320053:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.152@tcp1 routed destination 00000400:00000200:22.0:1616513074.320054:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513074.320054:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.320061:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.320067:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775320 00000400:00000200:20.0:1616513074.320067:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.152@tcp1(10.31.132.152@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513074.320071:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513074.320075:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.320081:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.320083:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.320085:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.320090:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.320094:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.320104:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.320108:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.320109:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.320113:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.320115:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.320118:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22770 00000800:00000200:8.0:1616513074.320120:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.324752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.324805:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.324814:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.324823:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.254@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.324831:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.254@tcp1 of length 224 into portal 26 MB=0x601aa11a2dc40 00000400:00000200:14.0:1616513074.324838:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.254@tcp1 of length 224/224 into md 0xf97c5ae9 [1] + 0 00000400:00000200:14.0:1616513074.324843:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42b28 00000400:00000200:14.0:1616513074.324846:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.324848:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513074.324853:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42b28 00000800:00000200:14.0:1616513074.324856:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.324857:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.324859:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.324861:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.324913:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690679812217920 00010000:00000200:15.0:1616513074.324923:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1690679812217920/t0(0) o400->2cc4444a-3bdf-0f3e-f3d9-a8f86d2bb9ed@10.31.140.254@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.324936:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690679812217920, offset 192 00000400:00000200:15.0:1616513074.324942:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.254@tcp1 00000400:00000200:15.0:1616513074.324949:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.254@tcp1 routed destination 00000400:00000200:15.0:1616513074.324962:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.254@tcp1(10.31.140.254@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.324968:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.324975:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.324978:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.324980:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513074.324983:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.324984:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.324988:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.324990:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513074.325049:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513074.325055:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513074.325058:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513074.325067:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:12.0:1616513074.325069:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.254@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.325070:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.325077:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.325078:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.254@tcp1 of length 224 into portal 12 MB=0x601aa11a2dcc0 00000400:00000200:2.0:1616513074.325083:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774bb0 00000400:00000200:12.0:1616513074.325084:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.254@tcp1 of length 224/224 into md 0x865b9f31 [1] + 77016 00000800:00000200:2.0:1616513074.325087:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513074.325090:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513074.325090:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513074.325092:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513074.325098:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.325101:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513074.325103:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.325106:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.325164:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690679812218048 00010000:00000200:20.0:1616513074.325175:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57c800 x1690679812218048/t0(0) o400->0bca165b-bcf5-d5aa-c7c1-442b7ec13697@10.31.140.254@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.325188:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690679812218048, offset 192 00000400:00000200:20.0:1616513074.325194:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.254@tcp1 00000400:00000200:20.0:1616513074.325201:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.254@tcp1 routed destination 00000400:00000200:20.0:1616513074.325215:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.254@tcp1(10.31.140.254@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.325221:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.325227:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.325230:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.325232:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.325236:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.325241:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.325253:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.325260:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.325263:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.325270:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.325274:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ff0 00000800:00000200:0.0:1616513074.325277:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.325280:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.335250:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.335306:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.335315:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.335323:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.63@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.335331:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.63@tcp1 of length 224 into portal 26 MB=0x5ffdc93839980 00000400:00000200:26.0:1616513074.335338:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.63@tcp1 of length 224/224 into md 0xf97c5b09 [1] + 0 00000400:00000200:26.0:1616513074.335342:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43d38 00000400:00000200:26.0:1616513074.335345:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.335347:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.335352:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43d38 00000800:00000200:26.0:1616513074.335355:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.335357:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.335359:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513074.335365:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.335368:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.335368:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:15.0:1616513074.335372:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697716316544 00000400:00000200:14.0:1616513074.335376:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.63@tcp1 : PUT - for me 00010000:00000200:15.0:1616513074.335382:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688697716316544/t0(0) o400->637f0d4a-779f-b529-7bc0-458412abce15@10.31.141.63@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513074.335384:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.63@tcp1 of length 224 into portal 12 MB=0x5ffdc93839cc0 00000400:00000200:14.0:1616513074.335390:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.63@tcp1 of length 224/224 into md 0xf96450b9 [1] + 40560 00000100:00000200:15.0:1616513074.335394:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697716316544, offset 192 00000400:00000200:14.0:1616513074.335395:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.335398:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:15.0:1616513074.335400:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.63@tcp1 00000800:00000200:14.0:1616513074.335403:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:15.0:1616513074.335407:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.63@tcp1 routed destination 00000800:00000200:14.0:1616513074.335408:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.335410:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513074.335421:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.63@tcp1(10.31.141.63@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.335426:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.335432:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.335434:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.335455:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.335459:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.335463:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.335466:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:30.0:1616513074.335485:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697716317376 00010000:00000200:30.0:1616513074.335497:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cd580 x1688697716317376/t0(0) o400->9b736eef-ce59-c2f6-3c9b-2d95ad6e4a3e@10.31.141.63@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.335512:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697716317376, offset 192 00000800:00000200:22.0:1616513074.335515:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:30.0:1616513074.335519:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.63@tcp1 00000800:00000200:22.0:1616513074.335522:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.335525:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.335525:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513074.335526:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.63@tcp1 routed destination 00000400:00000200:22.0:1616513074.335531:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.335537:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775e48 00000800:00000200:22.0:1616513074.335541:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:30.0:1616513074.335542:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.63@tcp1(10.31.141.63@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.335548:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.335554:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.335557:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513074.335559:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.335564:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.335569:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.335621:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.335627:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513074.335630:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513074.335636:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.335639:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee220 00000800:00000200:12.0:1616513074.335643:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.335645:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.341491:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513074.341547:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.341557:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.341566:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.141@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.341575:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.141@tcp1 of length 224 into portal 26 MB=0x603fcd6dba140 00000400:00000200:0.0:1616513074.341582:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.141@tcp1 of length 224/224 into md 0xf97c5b19 [1] + 0 00000400:00000200:0.0:1616513074.341586:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42aa0 00000400:00000200:0.0:1616513074.341589:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.341594:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513074.341598:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42aa0 00000800:00000200:8.0:1616513074.341600:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.341601:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.341604:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513074.341607:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.341609:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.341610:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.341618:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.141@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.341626:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.141@tcp1 of length 224 into portal 12 MB=0x603fcd6dba1c0 00000400:00000200:8.0:1616513074.341632:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.141@tcp1 of length 224/224 into md 0x865b9f31 [1] + 77240 00000400:00000200:8.0:1616513074.341637:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.341639:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513074.341646:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.341648:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.341650:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.341664:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234331623744 00010000:00000200:15.0:1616513074.341675:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1693234331623744/t0(0) o400->47d376d6-81af-db3e-02e5-afb3b9d94c10@10.31.138.141@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.341689:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234331623744, offset 192 00000400:00000200:15.0:1616513074.341695:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.141@tcp1 00000400:00000200:15.0:1616513074.341704:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.141@tcp1 routed destination 00000100:00000200:20.0:1616513074.341707:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234331623872 00010000:00000200:20.0:1616513074.341719:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d2880 x1693234331623872/t0(0) o400->cf85d758-d77e-f012-9b7b-7b5825cb3828@10.31.138.141@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513074.341720:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.141@tcp1(10.31.138.141@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.341727:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:20.0:1616513074.341732:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234331623872, offset 192 00000800:00000200:15.0:1616513074.341734:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.341737:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513074.341738:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.141@tcp1 00000800:00000200:15.0:1616513074.341739:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.341743:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513074.341746:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.141@tcp1 routed destination 00000800:00000200:7.2:1616513074.341751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.341753:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513074.341760:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.141@tcp1(10.31.138.141@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.341767:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.341772:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.341775:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.341777:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.341782:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.341787:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513074.341805:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.341813:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.341816:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.341823:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.341829:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775320 00000800:00000200:2.0:1616513074.341833:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.341836:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513074.341840:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.341847:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.341850:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.341851:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.341857:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.341861:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23320 00000800:00000200:28.0:1616513074.341864:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.350282:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.350338:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.350347:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.350356:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.187@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.350366:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.187@tcp1 of length 224 into portal 26 MB=0x5ffd487f2e880 00000400:00000200:26.0:1616513074.350373:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.187@tcp1 of length 224/224 into md 0xf97c5b39 [1] + 0 00000400:00000200:26.0:1616513074.350377:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42c38 00000400:00000200:26.0:1616513074.350380:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.350383:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.350388:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42c38 00000800:00000200:26.0:1616513074.350391:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.350391:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.350394:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.350395:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.350459:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663162546304 00010000:00000200:15.0:1616513074.350469:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688663162546304/t0(0) o400->c3ff8d28-3b62-ed04-be23-e5fdcc630261@10.31.136.187@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.350481:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663162546304, offset 192 00000400:00000200:15.0:1616513074.350486:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.187@tcp1 00000400:00000200:15.0:1616513074.350494:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.187@tcp1 routed destination 00000400:00000200:15.0:1616513074.350508:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.187@tcp1(10.31.136.187@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.350514:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.350519:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.350522:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.350524:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.350528:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.350533:0:48832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.350534:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513074.350588:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.350595:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.350598:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.350605:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.350611:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775dc0 00000800:00000200:2.0:1616513074.350615:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.350618:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.351671:0:48832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.351725:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.351734:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.351743:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.187@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.351753:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.187@tcp1 of length 224 into portal 12 MB=0x5ffd487f2e9c0 00000400:00000200:22.0:1616513074.351759:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.187@tcp1 of length 224/224 into md 0xf96450b9 [1] + 40784 00000400:00000200:22.0:1616513074.351764:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.351769:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513074.351775:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.351778:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.351778:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.351782:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513074.351795:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663162546624 00010000:00000200:30.0:1616513074.351808:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad1b00 x1688663162546624/t0(0) o400->b9762d49-4c14-6be0-aa43-65a3903561b6@10.31.136.187@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.351822:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663162546624, offset 192 00000400:00000200:30.0:1616513074.351829:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.187@tcp1 00000400:00000200:30.0:1616513074.351847:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.187@tcp1 routed destination 00000400:00000200:30.0:1616513074.351861:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.187@tcp1(10.31.136.187@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.351867:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.351873:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.351875:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513074.351877:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.351881:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513074.351911:0:48832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.351924:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513074.351926:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513074.351933:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:8.0:1616513074.351940:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513074.351941:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.123@tcp1 : PUT - for me 00000800:00000200:8.0:1616513074.351946:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.351949:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.351951:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.123@tcp1 of length 224 into portal 26 MB=0x60182b3ea1840 00000400:00000200:8.0:1616513074.351956:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.351958:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.123@tcp1 of length 224/224 into md 0xf97c5b59 [1] + 0 00000400:00000200:2.0:1616513074.351962:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a433a8 00000400:00000200:8.0:1616513074.351965:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eecc0 00000400:00000200:2.0:1616513074.351965:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.351968:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513074.351969:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.351972:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513074.351973:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a433a8 00000800:00000200:14.0:1616513074.351976:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.351976:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.351979:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.351981:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.352035:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510736103488 00010000:00000200:15.0:1616513074.352045:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1690510736103488/t0(0) o400->0a13c080-c62b-918e-1dc3-ca3657e0e279@10.31.132.123@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.352059:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510736103488, offset 192 00000400:00000200:15.0:1616513074.352065:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.123@tcp1 00000400:00000200:15.0:1616513074.352073:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.123@tcp1 routed destination 00000400:00000200:15.0:1616513074.352087:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.123@tcp1(10.31.132.123@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.352094:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.352100:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.352102:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.352104:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.352109:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.352112:0:48832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.352115:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513074.352123:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.352128:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.352131:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513074.352133:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513074.352138:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.352145:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775b18 00000800:00000200:26.0:1616513074.352148:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.352270:0:48832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.352281:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.352289:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513074.352291:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.352298:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.123@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.352307:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.123@tcp1 of length 224 into portal 12 MB=0x60182b3ea1980 00000400:00000200:14.0:1616513074.352314:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.123@tcp1 of length 224/224 into md 0xf96450b9 [1] + 41008 00000400:00000200:14.0:1616513074.352319:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.352322:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.352329:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.352332:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.352334:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513074.352338:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510736103808 00010000:00000200:30.0:1616513074.352343:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c9200 x1690510736103808/t0(0) o400->3b7fa5d4-2431-a3b0-e3a8-58918066dc43@10.31.132.123@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513074.352354:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510736103808, offset 192 00000400:00000200:30.0:1616513074.352359:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.123@tcp1 00000400:00000200:30.0:1616513074.352364:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.123@tcp1 routed destination 00000400:00000200:30.0:1616513074.352377:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.123@tcp1(10.31.132.123@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.352382:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.352386:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.352388:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513074.352390:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.352393:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.352398:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.352454:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.352461:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.352463:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.352470:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.352474:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeff0 00000800:00000200:0.0:1616513074.352478:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.352481:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.358352:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513074.358365:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.358373:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.358381:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.190@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.358390:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.190@tcp1 of length 224 into portal 26 MB=0x5ffd4b7e38700 00000400:00000200:8.0:1616513074.358396:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.190@tcp1 of length 224/224 into md 0xf97c5b69 [1] + 0 00000400:00000200:8.0:1616513074.358400:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43430 00000400:00000200:8.0:1616513074.358402:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.358405:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513074.358409:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43430 00000800:00000200:8.0:1616513074.358412:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.358414:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.358416:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.358418:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.358474:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663966844672 00010000:00000200:15.0:1616513074.358485:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688663966844672/t0(0) o400->b1c30e64-1ded-8b5d-210e-387b421bb2de@10.31.136.190@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.358499:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663966844672, offset 192 00000400:00000200:15.0:1616513074.358507:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.190@tcp1 00000800:00000200:11.2:1616513074.358509:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:15.0:1616513074.358516:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.190@tcp1 routed destination 00000400:00000200:15.0:1616513074.358531:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.190@tcp1(10.31.136.190@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.358538:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.358544:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.358547:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.358549:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.358554:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513074.358561:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.358561:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513074.358563:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.358573:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.358581:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.190@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.358590:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.190@tcp1 of length 224 into portal 12 MB=0x5ffd4b7e387c0 00000400:00000200:0.0:1616513074.358597:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.190@tcp1 of length 224/224 into md 0x865b9f31 [1] + 77464 00000400:00000200:0.0:1616513074.358602:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.358604:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513074.358611:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.358614:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.358616:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513074.358617:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513074.358619:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513074.358624:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.358627:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.358634:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.358640:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774198 00000800:00000200:22.0:1616513074.358643:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.358646:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.358692:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663966844864 00010000:00000200:20.0:1616513074.358703:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b196300 x1688663966844864/t0(0) o400->a16dc293-2df5-62e1-9a68-170205a27e56@10.31.136.190@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.358717:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663966844864, offset 192 00000400:00000200:20.0:1616513074.358722:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.190@tcp1 00000400:00000200:20.0:1616513074.358730:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.190@tcp1 routed destination 00000400:00000200:20.0:1616513074.358744:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.190@tcp1(10.31.136.190@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.358750:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.358756:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.358758:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.358760:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.358765:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.358770:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513074.358781:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513074.358788:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513074.358791:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.358791:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513074.358798:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513074.358802:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23f58 00000800:00000200:12.0:1616513074.358805:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513074.362575:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924480, portal 4 00000100:00000200:23.0:1616513074.362581:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924480, offset 0 00000400:00000200:23.0:1616513074.362588:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:23.0:1616513074.362594:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:23.0:1616513074.362607:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513074.362614:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:23.0:1616513074.362628:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513074.362631:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:23.0:1616513074.362633:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:23.0:1616513074.362638:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513074.362685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513074.362741:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:0.0:1616513074.362749:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:0.0:1616513074.362752:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.362759:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:0.0:1616513074.362766:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be42880 x1693067725924480/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513074.362778:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fba0 00000800:00000200:0.0:1616513074.362782:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513074.362785:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513074.362860:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513074.362868:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513074.362873:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513074.362880:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513074.362886:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626880 00000400:00000200:8.0:1616513074.362891:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d3d51 [1] + 192 00000400:00000200:8.0:1616513074.362895:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.362900:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be42880 x1693067725924480/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513074.362913:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513074.362915:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:8.0:1616513074.362916:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513074.362918:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:23.0:1616513074.362923:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be42880 x1693067725924480/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513074.362937:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be42880 x1693067725924480/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513074.362947:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eee0 00000800:00000200:7.2:1616513074.387276:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.387334:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.387343:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.387352:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.114@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.387361:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.114@tcp1 of length 224 into portal 26 MB=0x6001dc28d3700 00000400:00000200:2.0:1616513074.387368:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.114@tcp1 of length 224/224 into md 0xf97c5b89 [1] + 0 00000400:00000200:2.0:1616513074.387373:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43870 00000400:00000200:2.0:1616513074.387375:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.387378:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513074.387383:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43870 00000800:00000200:2.0:1616513074.387386:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.387388:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.387389:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.387396:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.387399:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513074.387399:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.387408:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.114@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.387417:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.114@tcp1 of length 224 into portal 12 MB=0x6001dc28d3780 00000400:00000200:14.0:1616513074.387423:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.114@tcp1 of length 224/224 into md 0xf96450b9 [1] + 41232 00000400:00000200:14.0:1616513074.387428:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.387431:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.387453:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.387457:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.387459:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.387461:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688977678350080 00000100:00000200:30.0:1616513074.387472:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688977678350208 00010000:00000200:15.0:1616513074.387472:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688977678350080/t0(0) o400->343ff8ba-2f36-bb1f-6904-4ea1723e2309@10.31.142.114@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513074.387484:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53ca400 x1688977678350208/t0(0) o400->34c579d2-8968-92c5-dce4-d725ac2fcd17@10.31.142.114@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513074.387487:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688977678350080, offset 192 00000400:00000200:15.0:1616513074.387493:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.114@tcp1 00000100:00000200:30.0:1616513074.387498:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688977678350208, offset 192 00000400:00000200:15.0:1616513074.387501:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.114@tcp1 routed destination 00000400:00000200:30.0:1616513074.387504:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.114@tcp1 00000400:00000200:15.0:1616513074.387517:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.114@tcp1(10.31.142.114@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.387523:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:30.0:1616513074.387526:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.114@tcp1 routed destination 00000800:00000200:15.0:1616513074.387529:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.387532:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.387535:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.387540:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513074.387542:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.114@tcp1(10.31.142.114@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513074.387544:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.387547:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513074.387548:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513074.387554:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.387556:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513074.387558:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513074.387563:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513074.387597:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.387608:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.387611:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513074.387611:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513074.387612:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513074.387618:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.387624:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775dc0 00000800:00000200:26.0:1616513074.387627:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513074.387629:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.387635:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.387638:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.387644:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.387648:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef430 00000800:00000200:0.0:1616513074.387652:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.387655:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.395347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513074.395349:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513074.395404:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.395413:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.395422:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.224@tcp1 : PUT - for me 00000800:00000200:12.0:1616513074.395427:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513074.395431:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.224@tcp1 of length 224 into portal 12 MB=0x5fffd5808b440 00000800:00000200:8.0:1616513074.395442:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513074.395450:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.224@tcp1 of length 224/224 into md 0xf96450b9 [1] + 41456 00000800:00000200:12.0:1616513074.395451:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513074.395455:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.395458:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513074.395460:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.224@tcp1 : PUT - for me 00000800:00000200:22.0:1616513074.395465:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.395465:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.395468:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.395470:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513074.395470:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.224@tcp1 of length 224 into portal 26 MB=0x5fffd5808b380 00000400:00000200:12.0:1616513074.395477:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.224@tcp1 of length 224/224 into md 0xf97c5b99 [1] + 0 00000100:00000200:30.0:1616513074.395478:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688838452327488 00000400:00000200:12.0:1616513074.395482:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42e58 00000400:00000200:12.0:1616513074.395485:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:30.0:1616513074.395487:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0000 x1688838452327488/t0(0) o400->c43c42f0-6db6-7699-82e9-8be017a42b37@10.31.136.224@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513074.395488:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513074.395492:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42e58 00000800:00000200:12.0:1616513074.395495:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:30.0:1616513074.395498:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688838452327488, offset 192 00000800:00000200:12.0:1616513074.395498:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513074.395500:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513074.395503:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.224@tcp1 00000400:00000200:30.0:1616513074.395510:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.224@tcp1 routed destination 00000400:00000200:30.0:1616513074.395522:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.224@tcp1(10.31.136.224@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513074.395528:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513074.395533:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513074.395535:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513074.395537:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513074.395541:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.395546:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.395557:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688838452327296 00010000:00000200:15.0:1616513074.395568:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688838452327296/t0(0) o400->f112f123-998b-b65a-4277-38662078d66b@10.31.136.224@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513074.395571:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.395577:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.395580:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:15.0:1616513074.395583:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688838452327296, offset 192 00000800:00000200:26.0:1616513074.395584:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513074.395587:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:15.0:1616513074.395589:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.224@tcp1 00000400:00000200:2.0:1616513074.395591:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee000 00000800:00000200:2.0:1616513074.395595:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513074.395597:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.224@tcp1 routed destination 00000400:00000200:15.0:1616513074.395613:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.224@tcp1(10.31.136.224@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.395624:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513074.395630:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.395633:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513074.395635:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.395640:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.395646:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.395647:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513074.395661:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.395668:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513074.395671:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513074.395678:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513074.395684:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775e48 00000800:00000200:0.0:1616513074.395688:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.395691:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.402323:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.402381:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.402389:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.402398:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.48@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.402407:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.48@tcp1 of length 224 into portal 26 MB=0x5fff348c7abc0 00000400:00000200:14.0:1616513074.402413:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.48@tcp1 of length 224/224 into md 0xf97c5bb9 [1] + 0 00000400:00000200:14.0:1616513074.402417:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a425d8 00000400:00000200:14.0:1616513074.402420:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.402422:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513074.402429:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a425d8 00000800:00000200:14.0:1616513074.402432:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.402433:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.402446:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.402448:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.402493:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795246734272 00010000:00000200:15.0:1616513074.402504:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688795246734272/t0(0) o400->6b1bca5a-1ae6-e972-fd65-371ce2d2bb7f@10.31.136.48@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.402518:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688795246734272, offset 192 00000400:00000200:15.0:1616513074.402524:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.48@tcp1 00000400:00000200:15.0:1616513074.402533:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.48@tcp1 routed destination 00000800:00000200:7.2:1616513074.402535:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:15.0:1616513074.402548:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.48@tcp1(10.31.136.48@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.402555:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.402561:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.402564:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.402566:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513074.402571:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:15.0:1616513074.402578:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513074.402588:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.402598:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.402606:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.48@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.402615:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.48@tcp1 of length 224 into portal 12 MB=0x5fff348c7ac80 00000400:00000200:26.0:1616513074.402622:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.48@tcp1 of length 224/224 into md 0xf96450b9 [1] + 41680 00000400:00000200:26.0:1616513074.402627:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.402630:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513074.402641:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513074.402642:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513074.402644:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513074.402649:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513074.402650:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513074.402652:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.402654:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.402660:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:6.0:1616513074.402662:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795246734464 00000400:00000200:2.0:1616513074.402666:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774198 00000800:00000200:2.0:1616513074.402670:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:6.0:1616513074.402672:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8480 x1688795246734464/t0(0) o400->3c4ba62a-54bb-34cd-1953-851b7d717e1d@10.31.136.48@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513074.402685:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688795246734464, offset 192 00000400:00000200:6.0:1616513074.402691:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.48@tcp1 00000400:00000200:6.0:1616513074.402698:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.48@tcp1 routed destination 00000400:00000200:6.0:1616513074.402711:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.48@tcp1(10.31.136.48@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513074.402717:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:6.0:1616513074.402722:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513074.402725:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:6.0:1616513074.402726:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513074.402731:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.402779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.402801:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.402806:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.402808:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513074.402812:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.402813:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.402816:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf003b8 00000800:00000200:8.0:1616513074.402820:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513074.426519:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924496, portal 4 00000100:00000200:26.0:1616513074.426525:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924496, offset 0 00000400:00000200:26.0:1616513074.426532:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:26.0:1616513074.426538:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:26.0:1616513074.426550:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513074.426557:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:26.0:1616513074.426563:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513074.426566:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513074.426568:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513074.426573:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513074.426619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513074.426683:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513074.426690:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513074.426693:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513074.426694:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513074.426699:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513074.426705:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a7ec00 x1693067725924496/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513074.426724:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af540 00000800:00000200:14.0:1616513074.426728:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513074.426789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513074.426801:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513074.426811:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513074.426819:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513074.426827:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626890 00000400:00000200:22.0:1616513074.426834:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c6179 [1] + 192 00000400:00000200:22.0:1616513074.426839:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.426844:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a7ec00 x1693067725924496/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513074.426855:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513074.426859:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513074.426864:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:26.0:1616513074.426865:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a7ec00 x1693067725924496/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513074.426866:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:26.0:1616513074.426877:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a7ec00 x1693067725924496/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513074.426887:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:11.2:1616513074.438633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513074.438692:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.438701:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513074.438710:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.118@tcp1 : PUT - for me 00000400:00000200:0.0:1616513074.438719:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.118@tcp1 of length 224 into portal 26 MB=0x5f90ff8cc5800 00000400:00000200:0.0:1616513074.438726:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.118@tcp1 of length 224/224 into md 0xf97c5bc9 [1] + 0 00000400:00000200:0.0:1616513074.438730:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43540 00000400:00000200:0.0:1616513074.438733:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.438736:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513074.438741:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43540 00000800:00000200:0.0:1616513074.438744:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.438746:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513074.438747:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.438753:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513074.438756:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.438757:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513074.438767:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.118@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.438776:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.118@tcp1 of length 224 into portal 12 MB=0x5f90ff8cc5880 00000400:00000200:28.0:1616513074.438782:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.118@tcp1 of length 224/224 into md 0x865b9f31 [1] + 77688 00000400:00000200:28.0:1616513074.438787:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.438790:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513074.438796:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.438800:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.438802:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.438807:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681221877520384 00010000:00000200:15.0:1616513074.438819:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1681221877520384/t0(0) o400->251a1a76-3509-326a-abfe-1ae67dc0bd2c@10.31.143.118@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.438834:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681221877520384, offset 192 00000400:00000200:15.0:1616513074.438841:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.118@tcp1 00000400:00000200:15.0:1616513074.438849:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.118@tcp1 routed destination 00000100:00000200:20.0:1616513074.438862:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681221877520512 00000400:00000200:15.0:1616513074.438864:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.118@tcp1(10.31.143.118@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.438871:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:20.0:1616513074.438875:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf85a00 x1681221877520512/t0(0) o400->f55c6b9a-9fc1-df9d-a215-903027630866@10.31.143.118@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513074.438877:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.438881:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.438885:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.438889:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681221877520512, offset 192 00000800:00000200:15.0:1616513074.438890:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513074.438896:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.118@tcp1 00000100:00000200:15.0:1616513074.438897:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.438898:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513074.438903:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.118@tcp1 routed destination 00000800:00000200:26.0:1616513074.438917:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513074.438919:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.118@tcp1(10.31.143.118@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513074.438924:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513074.438925:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513074.438927:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513074.438931:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:26.0:1616513074.438934:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513074.438934:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.438936:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513074.438939:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774198 00000800:00000200:20.0:1616513074.438941:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.438946:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513074.438951:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.438955:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513074.438960:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.438967:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.438970:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513074.438971:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.438976:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.438979:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee235c8 00000800:00000200:8.0:1616513074.438983:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.470260:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513074.470317:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513074.470327:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513074.470336:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.96@tcp1 : PUT - for me 00000400:00000200:16.0:1616513074.470346:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.96@tcp1 of length 224 into portal 26 MB=0x5ffdd2139d740 00000400:00000200:16.0:1616513074.470353:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.96@tcp1 of length 224/224 into md 0xf97c5bd9 [1] + 0 00000400:00000200:16.0:1616513074.470357:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42dd0 00000400:00000200:16.0:1616513074.470360:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513074.470363:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513074.470368:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42dd0 00000800:00000200:28.0:1616513074.470371:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.470371:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.470374:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513074.470376:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.470445:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700093847360 00000800:00000200:11.2:1616513074.470449:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:15.0:1616513074.470456:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688700093847360/t0(0) o400->f88a3759-d38e-99b7-bc62-d4b55a750e72@10.31.128.96@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.470471:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700093847360, offset 192 00000400:00000200:15.0:1616513074.470477:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.96@tcp1 00000400:00000200:15.0:1616513074.470486:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.96@tcp1 routed destination 00000800:00000200:0.0:1616513074.470495:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513074.470501:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.96@tcp1(10.31.128.96@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513074.470505:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513074.470507:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:15.0:1616513074.470508:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513074.470514:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.96@tcp1 : PUT - for me 00000800:00000200:15.0:1616513074.470515:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.470518:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.470520:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513074.470524:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.96@tcp1 of length 224 into portal 12 MB=0x5ffdd2139da40 00000800:00000200:15.0:1616513074.470525:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513074.470530:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.96@tcp1 of length 224/224 into md 0x865b9f31 [1] + 77912 00000800:00000200:7.2:1616513074.470531:0:48845:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.470532:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513074.470540:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513074.470542:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513074.470549:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513074.470552:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513074.470554:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513074.470587:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.470595:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.470598:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.470605:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.470611:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d650 00000800:00000200:2.0:1616513074.470615:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.470618:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.470623:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700093848128 00010000:00000200:20.0:1616513074.470635:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b190900 x1688700093848128/t0(0) o400->a955a133-dd2c-fd56-6399-ef4a29cce99c@10.31.128.96@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.470649:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700093848128, offset 192 00000400:00000200:20.0:1616513074.470656:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.96@tcp1 00000400:00000200:20.0:1616513074.470663:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.96@tcp1 routed destination 00000400:00000200:20.0:1616513074.470680:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.96@tcp1(10.31.128.96@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.470687:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.470693:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.470696:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.470698:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.470703:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.470708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.470724:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.470729:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.470732:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513074.470734:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.470739:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.470743:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22e58 00000800:00000200:28.0:1616513074.470746:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513074.474487:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924528, portal 4 00000100:00000200:10.0:1616513074.474492:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924528, offset 0 00000400:00000200:10.0:1616513074.474497:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:10.0:1616513074.474502:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:10.0:1616513074.474512:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513074.474517:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:10.0:1616513074.474523:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513074.474525:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000100:00000200:11.0:1616513074.474526:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924512, portal 4 00000800:00000200:10.0:1616513074.474527:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:11.0:1616513074.474531:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924512, offset 0 00000800:00000200:10.0:1616513074.474531:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:11.0:1616513074.474539:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:11.0:1616513074.474545:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:11.0:1616513074.474558:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513074.474565:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:11.0:1616513074.474572:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513074.474575:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.2:1616513074.474577:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:11.0:1616513074.474577:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:11.0:1616513074.474583:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (24)-- 00000100:00000200:23.0:1616513074.474590:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924544, portal 4 00000100:00000200:23.0:1616513074.474595:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924544, offset 0 00000800:00000200:27.0:1616513074.474597:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:27.0:1616513074.474602:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:23.0:1616513074.474602:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:27.0:1616513074.474605:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513074.474608:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:23.0:1616513074.474608:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:27.0:1616513074.474612:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:31.0:1616513074.474615:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:31.0:1616513074.474618:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:27.0:1616513074.474619:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a7d100 x1693067725924528/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513074.474622:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:31.0:1616513074.474625:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:23.0:1616513074.474628:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000100:00000200:31.0:1616513074.474631:0:6476:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8f30a8ad00 x1693067725924512/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513074.474631:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e825d8 00000800:00000200:27.0:1616513074.474635:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:23.0:1616513074.474635:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513074.474638:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:23.0:1616513074.474638:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:23.0:1616513074.474640:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000400:00000200:31.0:1616513074.474642:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a2a8 00000800:00000200:31.0:1616513074.474646:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:23.0:1616513074.474646:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513074.474691:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513074.474748:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513074.474755:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513074.474758:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513074.474760:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:26.0:1616513074.474765:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:27.2:1616513074.474772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000100:00000200:26.0:1616513074.474772:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc567080 x1693067725924544/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513074.474780:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.474787:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513074.474789:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513074.474795:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513074.474795:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513074.474795:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f4b8 00000800:00000200:3.0:1616513074.474799:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:26.0:1616513074.474801:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000400:00000200:31.0:1616513074.474803:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513074.474803:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6268a0 00000400:00000200:31.0:1616513074.474811:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6268b0 00000400:00000200:27.0:1616513074.474811:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37165acd [1] + 192 00000400:00000200:27.0:1616513074.474817:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:31.0:1616513074.474819:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37165abd [1] + 192 00000400:00000200:31.0:1616513074.474824:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513074.474825:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8f30a8ad00 x1693067725924512/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513074.474839:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000100:00000200:31.0:1616513074.474840:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a7d100 x1693067725924528/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513074.474841:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513074.474845:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:31.0:1616513074.474855:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513074.474857:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:11.0:1616513074.474857:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8f30a8ad00 x1693067725924512/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513074.474859:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:25.2:1616513074.474867:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000100:00000200:11.0:1616513074.474870:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8f30a8ad00 x1693067725924512/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513074.474880:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bc28 00000800:00000200:2.0:1616513074.474882:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000100:00000200:10.0:1616513074.474885:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a7d100 x1693067725924528/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513074.474891:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000100:00000200:10.0:1616513074.474896:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a7d100 x1693067725924528/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513074.474900:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:10.0:1616513074.474904:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83870 00000400:00000200:2.0:1616513074.474909:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6268c0 00000400:00000200:2.0:1616513074.474916:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c61c9 [1] + 192 00000400:00000200:2.0:1616513074.474921:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.474926:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc567080 x1693067725924544/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513074.474935:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513074.474942:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513074.474945:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513074.474947:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:23.0:1616513074.474951:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc567080 x1693067725924544/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513074.474964:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc567080 x1693067725924544/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513074.474982:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eee0 00000800:00000200:7.2:1616513074.482713:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.482768:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.482778:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.482787:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.252@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.482797:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.252@tcp1 of length 224 into portal 26 MB=0x6048d685ac040 00000400:00000200:14.0:1616513074.482803:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.252@tcp1 of length 224/224 into md 0xf97c5be9 [1] + 0 00000400:00000200:14.0:1616513074.482808:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42000 00000400:00000200:14.0:1616513074.482811:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.482813:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513074.482818:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42000 00000800:00000200:14.0:1616513074.482821:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.482822:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.482824:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.482826:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.482880:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513074.482884:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855247941696 00010000:00000200:15.0:1616513074.482895:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1693855247941696/t0(0) o400->a4e01450-2dfe-e0b5-ea92-e3a5b34cc07b@10.31.140.252@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.482909:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855247941696, offset 192 00000400:00000200:15.0:1616513074.482916:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.252@tcp1 00000400:00000200:15.0:1616513074.482924:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.252@tcp1 routed destination 00000400:00000200:15.0:1616513074.482940:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.252@tcp1(10.31.140.252@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.482946:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513074.482948:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.482952:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.482955:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513074.482957:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513074.482958:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.482962:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:22.0:1616513074.482966:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.252@tcp1 : PUT - for me 00000100:00000200:15.0:1616513074.482969:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513074.482976:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.252@tcp1 of length 224 into portal 12 MB=0x6048d685ac180 00000400:00000200:22.0:1616513074.482983:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.252@tcp1 of length 224/224 into md 0xf96450b9 [1] + 41904 00000400:00000200:22.0:1616513074.482988:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.482990:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513074.482999:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513074.483001:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513074.483002:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513074.483008:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513074.483010:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513074.483011:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.483013:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:10.0:1616513074.483015:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855247942016 00000400:00000200:2.0:1616513074.483020:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:10.0:1616513074.483024:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad6c00 x1693855247942016/t0(0) o400->5da67e50-8b7b-b8a5-08ef-afa4179160a7@10.31.140.252@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513074.483025:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d8f8 00000800:00000200:2.0:1616513074.483029:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513074.483036:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855247942016, offset 192 00000400:00000200:10.0:1616513074.483041:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.252@tcp1 00000400:00000200:10.0:1616513074.483046:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.252@tcp1 routed destination 00000400:00000200:10.0:1616513074.483058:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.252@tcp1(10.31.140.252@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513074.483063:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513074.483068:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513074.483070:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513074.483071:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513074.483075:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.483122:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.483134:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.483139:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.483141:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.483146:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513074.483147:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.483149:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83870 00000800:00000200:8.0:1616513074.483152:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.522668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.522724:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513074.522726:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513074.522734:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.522743:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.52@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.522752:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.52@tcp1 of length 224 into portal 26 MB=0x5ffdc65c33100 00000400:00000200:14.0:1616513074.522759:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.52@tcp1 of length 224/224 into md 0xf97c5bf9 [1] + 0 00000400:00000200:14.0:1616513074.522764:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43320 00000400:00000200:14.0:1616513074.522766:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.522769:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513074.522776:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43320 00000800:00000200:22.0:1616513074.522778:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.522779:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.522782:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513074.522785:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.522788:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513074.522788:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.522797:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.52@tcp1 : PUT - for me 00000800:00000200:16.0:1616513074.522799:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513074.522806:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.52@tcp1 of length 224 into portal 12 MB=0x5ffdc65c33480 00000800:00000200:16.0:1616513074.522808:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513074.522813:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.52@tcp1 of length 224/224 into md 0xf96450b9 [1] + 42128 00000400:00000200:16.0:1616513074.522817:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.236@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.522818:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.522821:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513074.522827:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.236@tcp1 of length 224 into portal 26 MB=0x5ffd43e6ec000 00000800:00000200:22.0:1616513074.522829:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.522832:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513074.522833:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.236@tcp1 of length 224/224 into md 0xf97c5c09 [1] + 0 00000800:00000200:22.0:1616513074.522834:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513074.522840:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43cb0 00000100:00000200:15.0:1616513074.522840:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696948732160 00000400:00000200:16.0:1616513074.522843:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513074.522845:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00010000:00000200:15.0:1616513074.522850:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688696948732160/t0(0) o400->8777dfc2-c70b-ea12-57a0-0382c375581b@10.31.129.52@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513074.522851:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43cb0 00000100:00000200:14.0:1616513074.522851:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696948733056 00000800:00000200:16.0:1616513074.522854:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.522855:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.522857:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:14.0:1616513074.522862:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad3f00 x1688696948733056/t0(0) o400->c5756d1a-9abe-f41b-9e7d-5df0ede84854@10.31.129.52@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513074.522863:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513074.522866:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513074.522866:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696948732160, offset 192 00000800:00000200:28.0:1616513074.522867:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:15.0:1616513074.522872:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.52@tcp1 00000400:00000200:28.0:1616513074.522875:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.236@tcp1 : PUT - for me 00000100:00000200:14.0:1616513074.522876:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696948733056, offset 192 00000400:00000200:15.0:1616513074.522879:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.52@tcp1 routed destination 00000400:00000200:14.0:1616513074.522882:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.52@tcp1 00000400:00000200:28.0:1616513074.522885:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.236@tcp1 of length 224 into portal 12 MB=0x5ffd43e6ec080 00000400:00000200:28.0:1616513074.522892:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.236@tcp1 of length 224/224 into md 0x865b9f31 [1] + 78136 00000400:00000200:15.0:1616513074.522894:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.52@tcp1(10.31.129.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513074.522897:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.522900:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:15.0:1616513074.522901:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.522903:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.52@tcp1 routed destination 00000100:00000200:20.0:1616513074.522906:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661929148416 00000800:00000200:28.0:1616513074.522907:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513074.522907:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513074.522910:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:15.0:1616513074.522910:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513074.522912:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:15.0:1616513074.522912:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.522917:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:20.0:1616513074.522918:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688661929148416/t0(0) o400->494f0376-9872-c634-c02c-66a3357800d1@10.31.132.236@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513074.522919:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.52@tcp1(10.31.129.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513074.522921:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.522924:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513074.522925:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513074.522931:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:20.0:1616513074.522932:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661929148416, offset 192 00000800:00000200:14.0:1616513074.522934:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513074.522935:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513074.522939:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.236@tcp1 00000800:00000200:14.0:1616513074.522940:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.522945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513074.522947:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.236@tcp1 routed destination 00000800:00000200:8.0:1616513074.522954:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.522959:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.522961:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513074.522962:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.236@tcp1(10.31.132.236@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513074.522964:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.522967:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513074.522969:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:24.0:1616513074.522971:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661929148544 00000400:00000200:8.0:1616513074.522971:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757980 00000800:00000200:8.0:1616513074.522975:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:20.0:1616513074.522976:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513074.522977:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513074.522978:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513074.522981:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (24)++ 00010000:00000200:24.0:1616513074.522985:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf80900 x1688661929148544/t0(0) o400->31f40405-d867-5319-fb21-3fa430c839f1@10.31.132.236@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513074.522986:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:2.0:1616513074.522987:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:14.0:1616513074.522988:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513074.522989:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513074.522990:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:20.0:1616513074.522993:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513074.522997:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:24.0:1616513074.523000:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661929148544, offset 192 00000800:00000200:26.0:1616513074.523001:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:2.0:1616513074.523003:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d4b8 00000400:00000200:24.0:1616513074.523006:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.236@tcp1 00000800:00000200:2.0:1616513074.523007:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513074.523008:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513074.523010:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513074.523011:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513074.523014:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.236@tcp1 routed destination 00000400:00000200:26.0:1616513074.523018:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.523024:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23dc0 00000800:00000200:26.0:1616513074.523027:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513074.523028:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.236@tcp1(10.31.132.236@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.523035:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513074.523041:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.523044:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.523046:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513074.523050:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.523055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.523065:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.523069:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.523072:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513074.523075:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.523077:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.523080:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000800:00000200:28.0:1616513074.523084:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.534883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513074.534940:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513074.534949:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513074.534951:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513074.534958:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.77@tcp1 : PUT - for me 00000800:00000200:8.0:1616513074.534960:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513074.534962:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513074.534970:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.77@tcp1 of length 224 into portal 26 MB=0x5ffddb1d7b000 00000400:00000200:8.0:1616513074.534972:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.77@tcp1 : PUT - for me 00000400:00000200:16.0:1616513074.534976:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.77@tcp1 of length 224/224 into md 0xf97c5c19 [1] + 0 00000400:00000200:16.0:1616513074.534981:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42ee0 00000400:00000200:8.0:1616513074.534982:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.77@tcp1 of length 224 into portal 12 MB=0x5ffddb1d7b080 00000400:00000200:16.0:1616513074.534984:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513074.534987:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513074.534988:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.77@tcp1 of length 224/224 into md 0x865b9f31 [1] + 78360 00000400:00000200:16.0:1616513074.534991:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ee0 00000400:00000200:8.0:1616513074.534993:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513074.534994:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:8.0:1616513074.534996:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513074.534997:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513074.534999:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.535003:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.535006:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.535007:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.535054:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702520111104 00010000:00000200:20.0:1616513074.535066:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688702520111104/t0(0) o400->b574779a-a2fd-3c92-5ef1-e6c753a91dc6@10.31.136.77@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513074.535074:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702520111232 00000100:00000200:20.0:1616513074.535080:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702520111104, offset 192 00010000:00000200:24.0:1616513074.535086:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6780 x1688702520111232/t0(0) o400->30917b0d-49e4-1f11-9763-cd9baca72293@10.31.136.77@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513074.535087:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.77@tcp1 00000400:00000200:20.0:1616513074.535095:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.77@tcp1 routed destination 00000100:00000200:24.0:1616513074.535101:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702520111232, offset 192 00000400:00000200:24.0:1616513074.535107:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.77@tcp1 00000400:00000200:20.0:1616513074.535110:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.77@tcp1(10.31.136.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513074.535115:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.77@tcp1 routed destination 00000800:00000200:20.0:1616513074.535116:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513074.535129:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.77@tcp1(10.31.136.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.535136:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.535137:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.535139:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513074.535142:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.535143:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513074.535145:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.535147:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.535148:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513074.535151:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513074.535155:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.535155:0:48853:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513074.535156:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513074.535209:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.535216:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.535219:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.535226:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513074.535228:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513074.535231:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22d48 00000800:00000200:28.0:1616513074.535235:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513074.535235:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513074.535237:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513074.535238:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513074.535240:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513074.535246:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.535250:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2e58 00000800:00000200:28.0:1616513074.535254:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.548620:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.548675:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.548685:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.548694:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.54@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.548704:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.54@tcp1 of length 224 into portal 12 MB=0x5ffd4fadaff00 00000400:00000200:22.0:1616513074.548710:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.54@tcp1 of length 224/224 into md 0xf96450b9 [1] + 42352 00000400:00000200:22.0:1616513074.548716:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.548718:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513074.548725:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.548730:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.548730:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.548734:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513074.548787:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665090359040 00010000:00000200:14.0:1616513074.548798:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad7080 x1688665090359040/t0(0) o400->9e15cef9-5b14-e347-ab31-735c678409eb@10.31.138.54@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513074.548811:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665090359040, offset 192 00000400:00000200:14.0:1616513074.548817:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.54@tcp1 00000400:00000200:14.0:1616513074.548825:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.54@tcp1 routed destination 00000400:00000200:14.0:1616513074.548838:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.54@tcp1(10.31.138.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513074.548846:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513074.548851:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513074.548854:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.548856:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.548861:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.548865:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.548879:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.548883:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.548885:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.548890:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.548893:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47565d8 00000800:00000200:14.0:1616513074.548895:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.548898:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.551388:0:48855:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.551459:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.551472:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.551481:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.54@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.551489:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.54@tcp1 of length 224 into portal 26 MB=0x5ffd4fadafc40 00000400:00000200:26.0:1616513074.551496:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.54@tcp1 of length 224/224 into md 0xf97c5c29 [1] + 0 00000400:00000200:26.0:1616513074.551501:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43c28 00000400:00000200:26.0:1616513074.551503:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.551506:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.551510:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43c28 00000800:00000200:26.0:1616513074.551513:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.551516:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.551518:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.551520:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.551573:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665090358336 00010000:00000200:20.0:1616513074.551583:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688665090358336/t0(0) o400->e26fda8b-1507-bd80-9be2-4e08a9c1997b@10.31.138.54@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.551597:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665090358336, offset 192 00000400:00000200:20.0:1616513074.551602:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.54@tcp1 00000400:00000200:20.0:1616513074.551610:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.54@tcp1 routed destination 00000400:00000200:20.0:1616513074.551624:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.54@tcp1(10.31.138.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.551630:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.551635:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.551638:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.551640:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.551644:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513074.551650:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513074.551653:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513074.551708:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.551715:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513074.551718:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.551719:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513074.551725:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513074.551730:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22880 00000800:00000200:16.0:1616513074.551734:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.555134:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.555189:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.555199:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.555207:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.187@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.555217:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.187@tcp1 of length 224 into portal 26 MB=0x5ffd569e03d00 00000400:00000200:14.0:1616513074.555224:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.187@tcp1 of length 224/224 into md 0xf97c5c39 [1] + 0 00000400:00000200:14.0:1616513074.555229:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77756d8 00000400:00000200:14.0:1616513074.555231:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.555234:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513074.555238:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77756d8 00000800:00000200:14.0:1616513074.555242:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.555243:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.555245:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.555250:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.555253:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.555253:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.555262:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.187@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.555271:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.187@tcp1 of length 224 into portal 12 MB=0x5ffd569e03d80 00000400:00000200:2.0:1616513074.555277:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.187@tcp1 of length 224/224 into md 0xf96450b9 [1] + 42576 00000400:00000200:2.0:1616513074.555283:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.555285:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513074.555292:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.555295:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.555297:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.555303:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666952973568 00000100:00000200:14.0:1616513074.555312:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666952973696 00010000:00000200:20.0:1616513074.555314:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688666952973568/t0(0) o400->2766fb6a-34f2-bf76-8ff3-9e8bbaeed2ee@10.31.132.187@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:14.0:1616513074.555322:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461208000 x1688666952973696/t0(0) o400->77077ba3-7b17-2abc-32a2-7661f82a38e4@10.31.132.187@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.555328:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666952973568, offset 192 00000400:00000200:20.0:1616513074.555335:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.187@tcp1 00000100:00000200:14.0:1616513074.555335:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666952973696, offset 192 00000400:00000200:14.0:1616513074.555341:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.187@tcp1 00000400:00000200:20.0:1616513074.555343:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.187@tcp1 routed destination 00000400:00000200:14.0:1616513074.555348:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.187@tcp1 routed destination 00000400:00000200:20.0:1616513074.555363:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.187@tcp1(10.31.132.187@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513074.555363:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.187@tcp1(10.31.132.187@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.555370:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513074.555371:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.555376:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513074.555377:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.555379:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.555379:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.555381:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.555381:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513074.555385:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513074.555386:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.555391:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.555392:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513074.555431:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513074.555456:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.555462:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.555465:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513074.555467:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.555472:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.555477:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23078 00000800:00000200:22.0:1616513074.555481:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513074.555486:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513074.555493:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513074.555495:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513074.555502:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513074.555505:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47565d8 00000800:00000200:4.0:1616513074.555509:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.555512:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.566700:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.566757:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.566766:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.566775:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.11@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.566784:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.11@tcp1 of length 224 into portal 26 MB=0x5ffd3ffcc16c0 00000400:00000200:2.0:1616513074.566791:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.11@tcp1 of length 224/224 into md 0xf97c5c49 [1] + 0 00000400:00000200:2.0:1616513074.566796:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774b28 00000400:00000200:2.0:1616513074.566798:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.566801:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513074.566805:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774b28 00000800:00000200:2.0:1616513074.566808:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.566810:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.566811:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513074.566818:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.566821:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.566822:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.566830:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.11@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.566839:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.11@tcp1 of length 224 into portal 12 MB=0x5ffd3ffcc1740 00000400:00000200:26.0:1616513074.566845:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.11@tcp1 of length 224/224 into md 0xf96450b9 [1] + 42800 00000400:00000200:26.0:1616513074.566850:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.566853:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513074.566861:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.566864:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.566866:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.566885:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660878300864 00010000:00000200:20.0:1616513074.566896:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688660878300864/t0(0) o400->eeda493e-593f-9529-a59e-92b1f2918b12@10.31.133.11@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.566912:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660878300864, offset 192 00000400:00000200:20.0:1616513074.566918:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.11@tcp1 00000100:00000200:18.0:1616513074.566924:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660878300992 00000400:00000200:20.0:1616513074.566925:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.11@tcp1 routed destination 00010000:00000200:18.0:1616513074.566936:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dd580 x1688660878300992/t0(0) o400->90164de7-b050-eb54-47b6-c229745e83e4@10.31.133.11@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513074.566940:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.11@tcp1(10.31.133.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.566947:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:18.0:1616513074.566951:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660878300992, offset 192 00000800:00000200:20.0:1616513074.566953:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.566955:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:18.0:1616513074.566957:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.11@tcp1 00000800:00000200:20.0:1616513074.566958:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.566962:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513074.566966:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.11@tcp1 routed destination 00000800:00000200:7.2:1616513074.566968:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.566969:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513074.566980:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513074.566981:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.11@tcp1(10.31.133.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513074.566987:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513074.566987:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513074.566990:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513074.566992:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513074.566993:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.566996:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513074.566997:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513074.566998:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513074.567002:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23c28 00000800:00000200:18.0:1616513074.567002:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513074.567006:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.567049:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.567062:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.567066:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.567069:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.567074:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.567077:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccff0 00000800:00000200:8.0:1616513074.567081:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.567084:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.567089:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513074.567091:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.567093:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.567094:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.567096:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.568722:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.568775:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.568783:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.568792:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.34@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.568800:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.34@tcp1 of length 224 into portal 26 MB=0x5ffdd1432eb80 00000400:00000200:26.0:1616513074.568806:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.34@tcp1 of length 224/224 into md 0xf97c5c59 [1] + 0 00000400:00000200:26.0:1616513074.568811:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774110 00000400:00000200:26.0:1616513074.568813:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.568816:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.568819:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774110 00000800:00000200:26.0:1616513074.568822:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.568824:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.568826:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.568828:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.568838:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.568846:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.34@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.568854:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.34@tcp1 of length 224 into portal 12 MB=0x5ffdd1432ec00 00000400:00000200:14.0:1616513074.568860:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.34@tcp1 of length 224/224 into md 0xf96450b9 [1] + 43024 00000400:00000200:14.0:1616513074.568865:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.568867:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.568873:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.568876:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.568878:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.568880:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.568881:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699875289984 00010000:00000200:20.0:1616513074.568891:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688699875289984/t0(0) o400->88fe00e5-2f2b-734c-b786-00df6186c441@10.31.143.34@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.568904:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699875289984, offset 192 00000400:00000200:20.0:1616513074.568910:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.34@tcp1 00000400:00000200:20.0:1616513074.568917:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.34@tcp1 routed destination 00000400:00000200:20.0:1616513074.568931:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.34@tcp1(10.31.143.34@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.568937:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:18.0:1616513074.568937:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699875290112 00000800:00000200:20.0:1616513074.568942:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.568945:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.568947:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:18.0:1616513074.568948:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cd100 x1688699875290112/t0(0) o400->e3930ce4-289a-3f02-cf1b-c941c536ab4b@10.31.143.34@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513074.568951:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.568956:0:48857:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.568958:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513074.568962:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699875290112, offset 192 00000400:00000200:18.0:1616513074.568968:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.34@tcp1 00000400:00000200:18.0:1616513074.568975:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.34@tcp1 routed destination 00000800:00000200:22.0:1616513074.568988:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513074.568988:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.34@tcp1(10.31.143.34@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.568994:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513074.568995:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.568998:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513074.568999:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513074.569000:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.569002:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513074.569004:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513074.569004:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.569008:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513074.569010:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee237e8 00000800:00000200:22.0:1616513074.569013:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.569054:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.569111:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.569118:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.569120:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.569120:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.569127:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.569131:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc6e8 00000800:00000200:28.0:1616513074.569135:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.576117:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513074.576179:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513074.576189:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513074.576198:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.66@tcp1 : PUT - for me 00000400:00000200:16.0:1616513074.576207:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.66@tcp1 of length 224 into portal 26 MB=0x5ffdd14fecb40 00000400:00000200:16.0:1616513074.576214:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.66@tcp1 of length 224/224 into md 0xf97c5c69 [1] + 0 00000400:00000200:16.0:1616513074.576218:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77753a8 00000400:00000200:16.0:1616513074.576223:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513074.576226:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513074.576231:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77753a8 00000800:00000200:4.0:1616513074.576232:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.576234:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.576237:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.576238:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.576290:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699888651072 00010000:00000200:20.0:1616513074.576301:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688699888651072/t0(0) o400->6d88d4cd-b414-c95b-bf39-555316e73dd1@10.31.129.66@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.576314:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699888651072, offset 192 00000400:00000200:20.0:1616513074.576320:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.66@tcp1 00000400:00000200:20.0:1616513074.576327:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.66@tcp1 routed destination 00000400:00000200:20.0:1616513074.576342:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.66@tcp1(10.31.129.66@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.576348:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.576353:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.576355:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.576357:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.576362:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513074.576368:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.576369:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.576424:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.576431:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.576446:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.576452:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.576457:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23cb0 00000800:00000200:14.0:1616513074.576460:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.576463:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.576527:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513074.576536:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.576542:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.576549:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.66@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.576557:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.66@tcp1 of length 224 into portal 12 MB=0x5ffdd14fecf40 00000400:00000200:8.0:1616513074.576563:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.66@tcp1 of length 224/224 into md 0x865b9f31 [1] + 78584 00000400:00000200:8.0:1616513074.576567:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.576570:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513074.576575:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.576578:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.576579:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.576581:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513074.576640:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699888652096 00010000:00000200:24.0:1616513074.576652:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0d80 x1688699888652096/t0(0) o400->2721c4a9-bd89-7cf5-260c-19b1bc035eda@10.31.129.66@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513074.576666:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699888652096, offset 192 00000400:00000200:24.0:1616513074.576673:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.66@tcp1 00000400:00000200:24.0:1616513074.576680:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.66@tcp1 routed destination 00000400:00000200:24.0:1616513074.576695:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.66@tcp1(10.31.129.66@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.576703:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513074.576709:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.576711:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.576714:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513074.576718:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.576723:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.576734:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513074.576739:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513074.576742:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513074.576744:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513074.576749:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513074.576753:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:4.0:1616513074.576757:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.581228:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.581281:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.581290:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513074.581291:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513074.581298:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.165@tcp1 : PUT - for me 00000800:00000200:8.0:1616513074.581302:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513074.581306:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.165@tcp1 of length 224 into portal 12 MB=0x6018272847580 00000800:00000200:8.0:1616513074.581307:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513074.581312:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.165@tcp1 of length 224/224 into md 0xf96450b9 [1] + 43248 00000400:00000200:8.0:1616513074.581313:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.165@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.581317:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.581319:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.165@tcp1 of length 224 into portal 26 MB=0x6018272847500 00000100:00000200:26.0:1616513074.581320:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513074.581326:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:8.0:1616513074.581327:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.165@tcp1 of length 224/224 into md 0xf97c5c79 [1] + 0 00000800:00000200:26.0:1616513074.581328:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.581330:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513074.581331:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775100 00000800:00000200:26.0:1616513074.581332:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513074.581333:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.581335:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513074.581339:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775100 00000800:00000200:8.0:1616513074.581341:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.581343:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.581344:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.581346:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.581403:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509638923520 00000100:00000200:18.0:1616513074.581410:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509638923648 00010000:00000200:20.0:1616513074.581413:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1690509638923520/t0(0) o400->4c47c298-2a83-622f-a5e9-000de2df312a@10.31.138.165@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:18.0:1616513074.581422:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647de300 x1690509638923648/t0(0) o400->9791340c-c8d8-055a-a502-ea08adf28da5@10.31.138.165@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.581427:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690509638923520, offset 192 00000100:00000200:18.0:1616513074.581449:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690509638923648, offset 192 00000400:00000200:20.0:1616513074.581451:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.165@tcp1 00000400:00000200:18.0:1616513074.581455:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.165@tcp1 00000400:00000200:20.0:1616513074.581458:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.165@tcp1 routed destination 00000400:00000200:18.0:1616513074.581462:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.165@tcp1 routed destination 00000400:00000200:18.0:1616513074.581476:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.165@tcp1(10.31.138.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513074.581477:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.165@tcp1(10.31.138.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.581483:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513074.581483:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.581488:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.581489:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.581491:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513074.581491:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.581493:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513074.581495:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.581498:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513074.581499:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513074.581502:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.581505:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513074.581505:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513074.581526:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.581532:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.581534:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.581541:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.581546:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee225d8 00000800:00000200:14.0:1616513074.581549:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.581552:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513074.581558:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.581565:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513074.581568:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513074.581574:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513074.581578:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc220 00000800:00000200:16.0:1616513074.581582:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.581585:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.586575:0:6487:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924560, portal 4 00000100:00000200:20.0:1616513074.586580:0:6487:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924560, offset 0 00000400:00000200:20.0:1616513074.586585:0:6487:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:20.0:1616513074.586590:0:6487:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:20.0:1616513074.586601:0:6487:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.586607:0:6487:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:20.0:1616513074.586612:0:6487:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.586615:0:6487:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513074.586616:0:6487:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513074.586622:0:6487:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513074.586669:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513074.586682:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513074.586687:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513074.586689:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513074.586693:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:8.0:1616513074.586694:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513074.586699:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf80000 x1693067725924560/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513074.586711:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23c28 00000800:00000200:8.0:1616513074.586714:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:11.2:1616513074.586813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:5.2:1616513074.586857:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513074.586870:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.586876:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513074.586879:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513074.586884:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.586886:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513074.586886:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:16.0:1616513074.586887:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.67@tcp1 : PUT - for me 00000400:00000200:4.0:1616513074.586895:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513074.586897:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.67@tcp1 of length 224 into portal 12 MB=0x5ffd47cc31a00 00000400:00000200:16.0:1616513074.586903:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.67@tcp1 of length 224/224 into md 0x865b9f31 [1] + 78808 00000400:00000200:4.0:1616513074.586903:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6268d0 00000400:00000200:16.0:1616513074.586908:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:4.0:1616513074.586910:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d3e31 [1] + 192 00000100:00000200:16.0:1616513074.586915:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:4.0:1616513074.586915:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513074.586921:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513074.586923:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf80000 x1693067725924560/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513074.586924:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.586925:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:4.0:1616513074.586939:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513074.586942:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513074.586944:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:20.0:1616513074.586945:0:6487:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf80000 x1693067725924560/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513074.586956:0:6487:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf80000 x1693067725924560/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513074.586965:0:6487:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee237e8 00000100:00000200:24.0:1616513074.586984:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662974863872 00010000:00000200:24.0:1616513074.586996:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd6780 x1688662974863872/t0(0) o400->850e322d-46c8-756c-1c3c-af0c6e0f9e3c@10.31.138.67@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513074.587010:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662974863872, offset 192 00000400:00000200:24.0:1616513074.587017:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.67@tcp1 00000400:00000200:24.0:1616513074.587024:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.67@tcp1 routed destination 00000400:00000200:24.0:1616513074.587039:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.67@tcp1(10.31.138.67@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.587045:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513074.587051:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.587054:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513074.587056:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513074.587061:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.587066:0:48859:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.587133:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.587140:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.587143:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.587149:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.587153:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a08 00000800:00000200:22.0:1616513074.587157:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.587160:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.587755:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513074.587764:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.587770:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513074.587772:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.587777:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.67@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.587785:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.67@tcp1 of length 224 into portal 26 MB=0x5ffd47cc31980 00000400:00000200:8.0:1616513074.587790:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.67@tcp1 of length 224/224 into md 0xf97c5c89 [1] + 0 00000400:00000200:8.0:1616513074.587794:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774aa0 00000400:00000200:8.0:1616513074.587797:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.587799:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513074.587803:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774aa0 00000800:00000200:8.0:1616513074.587805:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.587807:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.587809:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.587821:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662974863744 00010000:00000200:20.0:1616513074.587830:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688662974863744/t0(0) o400->5a3e4405-aebe-5ba6-959c-66676941bb7e@10.31.138.67@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.587842:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662974863744, offset 192 00000400:00000200:20.0:1616513074.587847:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.67@tcp1 00000400:00000200:20.0:1616513074.587854:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.67@tcp1 routed destination 00000400:00000200:20.0:1616513074.587868:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.67@tcp1(10.31.138.67@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.587874:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.587879:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.587881:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.587883:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.587886:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.587891:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513074.587892:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513074.587901:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.587905:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513074.587907:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513074.587910:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513074.587913:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513074.587918:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee237e8 00000800:00000200:16.0:1616513074.587921:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.603903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.603959:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.603969:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513074.603970:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.603978:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.52@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.603987:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.52@tcp1 of length 224 into portal 26 MB=0x604b7d843cd00 00000400:00000200:28.0:1616513074.603994:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.52@tcp1 of length 224/224 into md 0xf97c5ca9 [1] + 0 00000400:00000200:28.0:1616513074.603999:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775cb0 00000400:00000200:28.0:1616513074.604002:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.604004:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513074.604009:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775cb0 00000800:00000200:28.0:1616513074.604012:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.604015:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.604016:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.604043:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513074.604075:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694037514112256 00010000:00000200:20.0:1616513074.604085:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1694037514112256/t0(0) o400->f4c7647c-0cb1-ccea-8cc6-a7f9711dca14@10.31.140.52@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.604097:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694037514112256, offset 192 00000800:00000200:4.0:1616513074.604098:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513074.604103:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.52@tcp1 00000800:00000200:4.0:1616513074.604107:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513074.604109:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.52@tcp1 routed destination 00000400:00000200:4.0:1616513074.604119:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.52@tcp1 : PUT - for me 00000400:00000200:20.0:1616513074.604123:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.52@tcp1(10.31.140.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.604128:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:4.0:1616513074.604129:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.52@tcp1 of length 224 into portal 12 MB=0x604b7d843cf80 00000800:00000200:20.0:1616513074.604133:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:4.0:1616513074.604135:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.52@tcp1 of length 224/224 into md 0x865b9f31 [1] + 79032 00000800:00000200:20.0:1616513074.604136:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.604138:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513074.604140:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513074.604143:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513074.604143:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:20.0:1616513074.604149:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513074.604150:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513074.604150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513074.604150:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513074.604153:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.604155:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513074.604211:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694037514112896 00000800:00000200:14.0:1616513074.604215:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:24.0:1616513074.604222:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08e780 x1694037514112896/t0(0) o400->a777836d-f578-2230-6313-d94c2c9af772@10.31.140.52@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513074.604222:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.604226:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.604232:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.604237:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22880 00000100:00000200:24.0:1616513074.604240:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694037514112896, offset 192 00000800:00000200:14.0:1616513074.604240:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.604243:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513074.604246:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.52@tcp1 00000400:00000200:24.0:1616513074.604253:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.52@tcp1 routed destination 00000400:00000200:24.0:1616513074.604267:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.52@tcp1(10.31.140.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.604273:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513074.604278:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.604281:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.604283:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513074.604287:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.604292:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.604300:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.604304:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.604306:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513074.604309:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.604311:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.604314:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:8.0:1616513074.604317:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513074.618559:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924576, portal 4 00000100:00000200:10.0:1616513074.618563:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924576, offset 0 00000400:00000200:10.0:1616513074.618572:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:10.0:1616513074.618576:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:10.0:1616513074.618585:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513074.618590:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:10.0:1616513074.618595:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513074.618597:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:10.0:1616513074.618599:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:10.0:1616513074.618603:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513074.618651:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513074.618670:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513074.618677:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513074.618689:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513074.618697:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513074.618714:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d746120ba80 x1693067725924576/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513074.618725:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513074.618726:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e836d8 00000800:00000200:1.0:1616513074.618731:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513074.618815:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513074.618824:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513074.618831:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513074.618832:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513074.618839:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513074.618847:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6268e0 00000400:00000200:1.0:1616513074.618854:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff9d5 [1] + 192 00000400:00000200:1.0:1616513074.618859:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513074.618864:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d746120ba80 x1693067725924576/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513074.618878:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513074.618881:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513074.618883:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:10.0:1616513074.618884:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d746120ba80 x1693067725924576/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513074.618893:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d746120ba80 x1693067725924576/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513074.618901:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83b18 00000100:00000200:18.0:1616513074.634512:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924592, portal 4 00000100:00000200:18.0:1616513074.634518:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924592, offset 0 00000400:00000200:18.0:1616513074.634525:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:18.0:1616513074.634531:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:18.0:1616513074.634545:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.634551:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:18.0:1616513074.634557:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.634560:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:18.0:1616513074.634562:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:18.0:1616513074.634568:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513074.634614:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.634635:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513074.634641:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513074.634643:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513074.634650:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513074.634657:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a7de80 x1693067725924592/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513074.634669:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc550 00000800:00000200:27.0:1616513074.634673:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513074.634676:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513074.634700:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513074.634709:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513074.634715:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513074.634722:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513074.634730:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6268f0 00000400:00000200:27.0:1616513074.634736:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37165b1d [1] + 192 00000400:00000200:27.0:1616513074.634741:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513074.634746:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a7de80 x1693067725924592/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513074.634760:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513074.634760:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513074.634763:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513074.634765:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:18.0:1616513074.634770:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a7de80 x1693067725924592/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:18.0:1616513074.634784:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a7de80 x1693067725924592/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513074.634794:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdb18 00000800:00000200:11.2:1616513074.635813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513074.635871:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513074.635881:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513074.635890:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.198@tcp1 : PUT - for me 00000400:00000200:16.0:1616513074.635901:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.198@tcp1 of length 224 into portal 26 MB=0x5ffdbd93aed80 00000400:00000200:16.0:1616513074.635907:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.198@tcp1 of length 224/224 into md 0xf97c5cb9 [1] + 0 00000400:00000200:16.0:1616513074.635912:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774000 00000400:00000200:16.0:1616513074.635915:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513074.635918:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513074.635922:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774000 00000800:00000200:4.0:1616513074.635924:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.635925:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.635928:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513074.635930:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513074.635933:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.635933:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513074.635942:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.198@tcp1 : PUT - for me 00000400:00000200:4.0:1616513074.635949:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.198@tcp1 of length 224 into portal 12 MB=0x5ffdbd93aeec0 00000400:00000200:4.0:1616513074.635955:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.198@tcp1 of length 224/224 into md 0x865b9f31 [1] + 79256 00000400:00000200:4.0:1616513074.635960:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513074.635963:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513074.635969:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.635972:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.635973:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.635982:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694590991744 00010000:00000200:20.0:1616513074.635991:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688694590991744/t0(0) o400->ffde25b8-3db5-d2c7-ad08-5c63a0da17b3@10.31.141.198@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.636004:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694590991744, offset 192 00000400:00000200:20.0:1616513074.636009:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.198@tcp1 00000400:00000200:20.0:1616513074.636015:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.198@tcp1 routed destination 00000400:00000200:20.0:1616513074.636028:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.198@tcp1(10.31.141.198@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.636034:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:24.0:1616513074.636037:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694590992064 00000800:00000200:20.0:1616513074.636039:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.636041:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.636043:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.636047:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:24.0:1616513074.636050:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08a880 x1688694590992064/t0(0) o400->0c9b7013-7241-1c88-8910-c227283dcc66@10.31.141.198@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.636053:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.636055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513074.636064:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694590992064, offset 192 00000400:00000200:24.0:1616513074.636070:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.198@tcp1 00000400:00000200:24.0:1616513074.636077:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.198@tcp1 routed destination 00000400:00000200:24.0:1616513074.636092:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.198@tcp1(10.31.141.198@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.636099:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513074.636104:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.636107:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.636109:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513074.636114:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.636119:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513074.636120:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.636127:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.636130:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.636136:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.636141:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23dc0 00000800:00000200:22.0:1616513074.636144:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.636147:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.636155:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513074.636157:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.636159:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.636160:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.636163:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513074.636171:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.636178:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.636180:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513074.636182:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513074.636186:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.636190:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3430 00000800:00000200:8.0:1616513074.636194:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.638657:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.638711:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.638720:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.638729:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.92@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.638738:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.92@tcp1 of length 224 into portal 26 MB=0x5ffd615060600 00000400:00000200:2.0:1616513074.638745:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.92@tcp1 of length 224/224 into md 0xf97c5cd9 [1] + 0 00000400:00000200:2.0:1616513074.638751:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77754b8 00000400:00000200:2.0:1616513074.638754:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.638757:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513074.638765:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77754b8 00000800:00000200:2.0:1616513074.638768:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.638771:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.638772:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.638775:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.638832:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688669824353792 00010000:00000200:20.0:1616513074.638843:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688669824353792/t0(0) o400->43cde11d-2567-1968-493b-8b39570d8e34@10.31.136.92@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.638857:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688669824353792, offset 192 00000400:00000200:20.0:1616513074.638863:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.92@tcp1 00000400:00000200:20.0:1616513074.638871:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.92@tcp1 routed destination 00000400:00000200:20.0:1616513074.638886:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.92@tcp1(10.31.136.92@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.638893:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.638899:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.638901:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.638903:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.638908:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.638913:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.638915:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513074.638967:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.638974:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.638977:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.638983:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.638989:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee235c8 00000800:00000200:26.0:1616513074.638992:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.638995:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.639093:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513074.639152:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.639161:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513074.639170:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.92@tcp1 : PUT - for me 00000400:00000200:4.0:1616513074.639179:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.92@tcp1 of length 224 into portal 12 MB=0x5ffd6150607c0 00000400:00000200:4.0:1616513074.639185:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.92@tcp1 of length 224/224 into md 0x865b9f31 [1] + 79480 00000400:00000200:4.0:1616513074.639190:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513074.639195:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513074.639201:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.639204:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513074.639204:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513074.639208:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513074.639262:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688669824354240 00010000:00000200:24.0:1616513074.639273:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08ba80 x1688669824354240/t0(0) o400->c4f00af6-fd4c-ae33-f853-50441749c967@10.31.136.92@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513074.639286:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688669824354240, offset 192 00000400:00000200:24.0:1616513074.639292:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.92@tcp1 00000400:00000200:24.0:1616513074.639299:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.92@tcp1 routed destination 00000400:00000200:24.0:1616513074.639313:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.92@tcp1(10.31.136.92@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.639319:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513074.639324:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.639327:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.639329:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513074.639333:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.639339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513074.639393:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.639400:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513074.639403:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.639404:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513074.639410:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513074.639414:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:16.0:1616513074.639418:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.648950:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.649007:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.649016:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513074.649025:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.27.195@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.649034:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.27.195@tcp1 of length 224 into portal 26 MB=0x5f90a481c8a40 00000400:00000200:28.0:1616513074.649041:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.27.195@tcp1 of length 224/224 into md 0xf97c5ce9 [1] + 0 00000400:00000200:28.0:1616513074.649050:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775c28 00000400:00000200:28.0:1616513074.649053:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.649056:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:4.0:1616513074.649060:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513074.649061:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775c28 00000800:00000200:28.0:1616513074.649064:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513074.649067:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513074.649069:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513074.649071:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513074.649072:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513074.649080:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.27.195@tcp1 : PUT - for me 00000400:00000200:4.0:1616513074.649088:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.27.195@tcp1 of length 224 into portal 12 MB=0x5f90a481c8ac0 00000400:00000200:4.0:1616513074.649094:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.27.195@tcp1 of length 224/224 into md 0x865b9f31 [1] + 79704 00000400:00000200:4.0:1616513074.649100:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513074.649102:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513074.649109:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.649112:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.649114:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.649123:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681197438372416 00010000:00000200:20.0:1616513074.649134:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1681197438372416/t0(0) o400->044a89da-07c8-7a0f-0a9b-ebb778721829@10.31.27.195@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.649159:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681197438372416, offset 192 00000400:00000200:20.0:1616513074.649165:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.27.195@tcp1 00000100:00000200:24.0:1616513074.649171:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681197438372544 00000400:00000200:20.0:1616513074.649172:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.27.195@tcp1 routed destination 00010000:00000200:24.0:1616513074.649182:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b088d80 x1681197438372544/t0(0) o400->71511a59-6905-19e8-2e8f-642cadd69125@10.31.27.195@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513074.649187:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.27.195@tcp1(10.31.27.195@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.649194:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:24.0:1616513074.649196:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681197438372544, offset 192 00000800:00000200:20.0:1616513074.649200:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513074.649201:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.27.195@tcp1 00000800:00000200:20.0:1616513074.649202:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.649205:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513074.649208:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.27.195@tcp1 routed destination 00000800:00000200:20.0:1616513074.649209:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513074.649217:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.649217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513074.649224:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.27.195@tcp1(10.31.27.195@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.649230:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513074.649235:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.649237:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.649239:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513074.649244:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.649249:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513074.649270:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.649277:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.649279:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.649286:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.649291:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23320 00000800:00000200:22.0:1616513074.649294:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.649297:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513074.649303:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.649310:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513074.649313:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.649313:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513074.649320:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513074.649324:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2990 00000800:00000200:16.0:1616513074.649328:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:13.0:1616513074.650648:0:6493:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924608, portal 4 00000100:00000200:13.0:1616513074.650654:0:6493:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924608, offset 0 00000400:00000200:13.0:1616513074.650661:0:6493:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:13.0:1616513074.650667:0:6493:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:13.0:1616513074.650680:0:6493:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:13.0:1616513074.650686:0:6493:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:13.0:1616513074.650693:0:6493:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513074.650698:0:6493:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:13.0:1616513074.650701:0:6493:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:13.0:1616513074.650706:0:6493:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513074.650751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513074.650808:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513074.650815:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513074.650819:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513074.650826:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513074.650832:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d853e8c8480 x1693067725924608/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513074.650845:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345a90 00000800:00000200:28.0:1616513074.650849:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513074.650852:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513074.650914:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513074.650923:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513074.650930:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513074.650938:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513074.650946:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626900 00000400:00000200:8.0:1616513074.650953:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d3ef1 [1] + 192 00000400:00000200:8.0:1616513074.650958:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.650963:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d853e8c8480 x1693067725924608/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513074.650975:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513074.650978:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513074.650981:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513074.650983:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:13.0:1616513074.650988:0:6493:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d853e8c8480 x1693067725924608/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:13.0:1616513074.651001:0:6493:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d853e8c8480 x1693067725924608/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:13.0:1616513074.651012:0:6493:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345210 00000800:00000200:11.2:1616513074.652308:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513074.652362:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.652371:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513074.652380:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.30@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.652388:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.30@tcp1 of length 224 into portal 26 MB=0x5f9222f4e8180 00000400:00000200:28.0:1616513074.652395:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.30@tcp1 of length 224/224 into md 0xf97c5cf9 [1] + 0 00000400:00000200:28.0:1616513074.652399:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77757e8 00000400:00000200:28.0:1616513074.652402:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513074.652404:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513074.652408:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77757e8 00000800:00000200:28.0:1616513074.652411:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.652414:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513074.652414:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513074.652421:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513074.652423:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.652424:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513074.652432:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.30@tcp1 : PUT - for me 00000400:00000200:4.0:1616513074.652453:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.30@tcp1 of length 224 into portal 12 MB=0x5f9222f4e8200 00000400:00000200:4.0:1616513074.652458:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.30@tcp1 of length 224/224 into md 0x865b9f31 [1] + 79928 00000400:00000200:4.0:1616513074.652463:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513074.652466:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:20.0:1616513074.652469:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681300101431680 00000800:00000200:4.0:1616513074.652472:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.652475:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.652477:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:20.0:1616513074.652479:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1681300101431680/t0(0) o400->79c66309-7d6f-c68b-bbbe-751309566cea@10.31.132.30@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.652493:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681300101431680, offset 192 00000400:00000200:20.0:1616513074.652498:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.30@tcp1 00000400:00000200:20.0:1616513074.652505:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.30@tcp1 routed destination 00000400:00000200:20.0:1616513074.652519:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.30@tcp1(10.31.132.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.652525:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.652531:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:24.0:1616513074.652533:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681300101431808 00000800:00000200:20.0:1616513074.652533:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.652535:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.652540:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:24.0:1616513074.652544:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b089f80 x1681300101431808/t0(0) o400->bfa24568-4a53-e2d6-4629-380f944cc8ce@10.31.132.30@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.652546:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.652546:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513074.652556:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681300101431808, offset 192 00000400:00000200:24.0:1616513074.652562:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.30@tcp1 00000400:00000200:24.0:1616513074.652568:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.30@tcp1 routed destination 00000400:00000200:24.0:1616513074.652581:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.30@tcp1(10.31.132.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.652587:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513074.652592:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.652594:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.652596:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513074.652600:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513074.652604:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513074.652606:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513074.652611:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:8.0:1616513074.652614:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513074.652616:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.652618:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.652620:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513074.652622:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513074.652623:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.652625:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.652628:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22770 00000400:00000200:8.0:1616513074.652631:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3c28 00000800:00000200:26.0:1616513074.652632:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513074.652634:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513074.652635:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.662564:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.662618:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.662627:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.662635:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.25@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.662644:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.25@tcp1 of length 224 into portal 26 MB=0x5ffd45c976500 00000400:00000200:22.0:1616513074.662650:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.25@tcp1 of length 224/224 into md 0xf97c5d19 [1] + 0 00000400:00000200:22.0:1616513074.662654:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774660 00000400:00000200:22.0:1616513074.662657:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.662659:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513074.662663:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774660 00000800:00000200:22.0:1616513074.662666:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.662668:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.662670:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.662672:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.662724:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662435128576 00010000:00000200:20.0:1616513074.662734:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688662435128576/t0(0) o400->3e51cc37-434d-2c1d-876d-880432dcaa23@10.31.133.25@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.662748:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662435128576, offset 192 00000400:00000200:20.0:1616513074.662754:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.25@tcp1 00000400:00000200:20.0:1616513074.662760:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.25@tcp1 routed destination 00000400:00000200:20.0:1616513074.662776:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.25@tcp1(10.31.133.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.662782:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.662787:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.662790:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.662792:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.662796:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.662802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.662803:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513074.662860:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.662867:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.662870:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.662876:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.662882:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23100 00000800:00000200:26.0:1616513074.662885:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.662888:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.662992:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.663002:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.663008:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513074.663010:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.663015:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.25@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.663021:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.25@tcp1 of length 224 into portal 12 MB=0x5ffd45c976600 00000400:00000200:22.0:1616513074.663026:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.25@tcp1 of length 224/224 into md 0xf96450b9 [1] + 43472 00000400:00000200:22.0:1616513074.663030:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.663033:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513074.663039:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.663041:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.663044:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513074.663106:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662435128832 00010000:00000200:18.0:1616513074.663118:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad1680 x1688662435128832/t0(0) o400->f59e4614-b4a7-8da4-c43c-42739f053e29@10.31.133.25@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513074.663132:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662435128832, offset 192 00000400:00000200:18.0:1616513074.663139:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.25@tcp1 00000400:00000200:18.0:1616513074.663147:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.25@tcp1 routed destination 00000400:00000200:18.0:1616513074.663161:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.25@tcp1(10.31.133.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.663168:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513074.663174:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.663176:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.663178:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.663183:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.663229:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513074.663286:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.663293:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513074.663296:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513074.663303:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513074.663307:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdb18 00000800:00000200:16.0:1616513074.663311:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.663314:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.669757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513074.669811:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513074.669821:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513074.669829:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.184@tcp1 : PUT - for me 00000400:00000200:26.0:1616513074.669839:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.184@tcp1 of length 224 into portal 26 MB=0x60182aebeda00 00000400:00000200:26.0:1616513074.669846:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.184@tcp1 of length 224/224 into md 0xf97c5d39 [1] + 0 00000400:00000200:26.0:1616513074.669850:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775210 00000400:00000200:26.0:1616513074.669853:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.669856:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513074.669860:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775210 00000800:00000200:26.0:1616513074.669863:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.669866:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.669867:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513074.669871:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.669922:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510649383424 00010000:00000200:20.0:1616513074.669932:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1690510649383424/t0(0) o400->cbb44cff-3eb4-74c5-0422-7cc29f1b7bbe@10.31.131.184@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.669944:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510649383424, offset 192 00000400:00000200:20.0:1616513074.669949:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.184@tcp1 00000400:00000200:20.0:1616513074.669956:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.184@tcp1 routed destination 00000400:00000200:20.0:1616513074.669970:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.184@tcp1(10.31.131.184@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.669975:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.669980:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.669983:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.669985:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.669989:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513074.669994:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.669994:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.670047:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.670054:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.670057:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.670064:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.670070:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22aa0 00000800:00000200:2.0:1616513074.670073:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.670076:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.670590:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.670602:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.670610:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513074.670612:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.670619:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.184@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.670628:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.184@tcp1 of length 224 into portal 12 MB=0x60182aebedb80 00000400:00000200:14.0:1616513074.670634:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.184@tcp1 of length 224/224 into md 0xf96450b9 [1] + 43696 00000400:00000200:14.0:1616513074.670640:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.670642:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.670650:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.670653:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.670654:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513074.670713:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510649383808 00010000:00000200:18.0:1616513074.670727:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e0000 x1690510649383808/t0(0) o400->d3fbfc92-0d72-6355-df6e-f88231c36663@10.31.131.184@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513074.670741:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510649383808, offset 192 00000400:00000200:18.0:1616513074.670747:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.184@tcp1 00000400:00000200:18.0:1616513074.670755:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.184@tcp1 routed destination 00000400:00000200:18.0:1616513074.670770:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.184@tcp1(10.31.131.184@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.670777:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513074.670782:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.670785:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.670787:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.670792:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.670848:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.670905:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.670911:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.670914:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.670915:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.670925:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.670929:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc908 00000800:00000200:28.0:1616513074.670932:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.672729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.672786:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.672795:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.672803:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.5@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.672812:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.5@tcp1 of length 224 into portal 12 MB=0x5ffd480cf4380 00000400:00000200:2.0:1616513074.672819:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.5@tcp1 of length 224/224 into md 0xf96450b9 [1] + 43920 00000400:00000200:2.0:1616513074.672824:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.672827:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513074.672833:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.672836:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.672838:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.672840:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.672870:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513074.672896:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663042769792 00010000:00000200:18.0:1616513074.672908:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461209b00 x1688663042769792/t0(0) o400->c84454f2-c675-3de5-556e-3a0677341032@10.31.138.5@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513074.672922:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663042769792, offset 192 00000800:00000200:16.0:1616513074.672927:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513074.672929:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.5@tcp1 00000400:00000200:18.0:1616513074.672936:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.5@tcp1 routed destination 00000800:00000200:16.0:1616513074.672937:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513074.672946:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.5@tcp1 : PUT - for me 00000400:00000200:18.0:1616513074.672951:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.5@tcp1(10.31.138.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513074.672957:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.5@tcp1 of length 224 into portal 26 MB=0x5ffd480cf4240 00000800:00000200:18.0:1616513074.672958:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513074.672963:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.5@tcp1 of length 224/224 into md 0xf97c5d59 [1] + 0 00000800:00000200:18.0:1616513074.672964:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.672966:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513074.672968:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513074.672968:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775d38 00000400:00000200:16.0:1616513074.672971:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513074.672973:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513074.672974:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513074.672978:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775d38 00000800:00000200:7.2:1616513074.672978:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513074.672981:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.672984:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513074.672986:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.672989:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513074.673037:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663042769472 00010000:00000200:20.0:1616513074.673046:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688663042769472/t0(0) o400->5ace6d3f-debe-e5b4-afe7-e061e0f08e47@10.31.138.5@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513074.673049:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.673056:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.673058:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:20.0:1616513074.673059:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663042769472, offset 192 00000400:00000200:20.0:1616513074.673064:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.5@tcp1 00000400:00000200:26.0:1616513074.673065:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.673069:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccd48 00000400:00000200:20.0:1616513074.673071:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.5@tcp1 routed destination 00000800:00000200:26.0:1616513074.673073:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.673076:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513074.673084:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.5@tcp1(10.31.138.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.673089:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.673094:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.673098:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.673100:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.673104:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.673109:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513074.673110:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513074.673119:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.673122:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.673124:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.673129:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.673133:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22330 00000800:00000200:8.0:1616513074.673136:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.673138:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.676335:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.676389:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.676398:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.676407:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.69@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.676417:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.69@tcp1 of length 224 into portal 26 MB=0x6036602d2ad00 00000400:00000200:2.0:1616513074.676424:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.69@tcp1 of length 224/224 into md 0xf97c5d69 [1] + 0 00000400:00000200:2.0:1616513074.676429:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775980 00000800:00000200:22.0:1616513074.676444:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513074.676446:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.676449:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513074.676453:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775980 00000800:00000200:2.0:1616513074.676456:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.676459:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.676461:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.676518:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692586529172736 00010000:00000200:20.0:1616513074.676529:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1692586529172736/t0(0) o400->0ccbe3c9-8754-fc3a-470d-b202f669e270@10.31.129.69@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.676543:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692586529172736, offset 192 00000400:00000200:20.0:1616513074.676550:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.69@tcp1 00000400:00000200:20.0:1616513074.676557:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.69@tcp1 routed destination 00000400:00000200:20.0:1616513074.676572:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.69@tcp1(10.31.129.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.676578:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.676584:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.676587:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.676589:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.676594:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513074.676598:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513074.676598:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.676600:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513074.676651:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.676658:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.676663:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.676669:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513074.676669:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513074.676675:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ee0 00000800:00000200:26.0:1616513074.676678:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513074.676678:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513074.676681:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513074.676688:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.69@tcp1 : PUT - for me 00000400:00000200:16.0:1616513074.676696:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.69@tcp1 of length 224 into portal 12 MB=0x6036602d2b140 00000400:00000200:16.0:1616513074.676703:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.69@tcp1 of length 224/224 into md 0x865b9f31 [1] + 80152 00000400:00000200:16.0:1616513074.676708:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513074.676711:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513074.676717:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.676720:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513074.676722:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.676724:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513074.676781:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692586529173824 00010000:00000200:24.0:1616513074.676794:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5e80 x1692586529173824/t0(0) o400->b4436b00-f27c-46d9-4184-16f04742f9b0@10.31.129.69@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513074.676808:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692586529173824, offset 192 00000400:00000200:24.0:1616513074.676815:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.69@tcp1 00000400:00000200:24.0:1616513074.676822:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.69@tcp1 routed destination 00000400:00000200:24.0:1616513074.676837:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.69@tcp1(10.31.129.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.676844:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513074.676849:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.676852:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.676856:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513074.676861:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.676866:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.676876:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.676880:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.676882:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.676888:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.676891:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2880 00000800:00000200:8.0:1616513074.676893:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.676896:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.682386:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513074.682403:0:48870:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513074.682465:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513074.682470:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513074.682473:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513074.682479:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513074.682482:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.24@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.682487:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.24@tcp1 : PUT - for me 00000400:00000200:16.0:1616513074.682491:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.24@tcp1 of length 224 into portal 26 MB=0x60182c15d7d40 00000400:00000200:22.0:1616513074.682496:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.24@tcp1 of length 224 into portal 12 MB=0x60182c15d7e80 00000400:00000200:16.0:1616513074.682497:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.24@tcp1 of length 224/224 into md 0xf97c5d89 [1] + 0 00000400:00000200:16.0:1616513074.682501:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774cc0 00000400:00000200:16.0:1616513074.682504:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.682506:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.24@tcp1 of length 224/224 into md 0xf96450b9 [1] + 44144 00000100:00000200:16.0:1616513074.682510:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513074.682511:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513074.682514:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774cc0 00000100:00000200:22.0:1616513074.682516:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513074.682517:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513074.682518:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.682520:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513074.682522:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513074.682523:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513074.682525:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.682526:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.682529:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.682589:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510961769792 00000100:00000200:18.0:1616513074.682595:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510961770112 00010000:00000200:20.0:1616513074.682598:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1690510961769792/t0(0) o400->e30bdbe9-0868-93ea-f39e-cabd2505ed09@10.31.133.24@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:18.0:1616513074.682607:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5a00 x1690510961770112/t0(0) o400->cb7c3ab1-0eb9-f528-eff4-72a08299a157@10.31.133.24@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513074.682610:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510961769792, offset 192 00000400:00000200:20.0:1616513074.682615:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.24@tcp1 00000100:00000200:18.0:1616513074.682620:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510961770112, offset 192 00000400:00000200:20.0:1616513074.682621:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.24@tcp1 routed destination 00000400:00000200:18.0:1616513074.682626:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.24@tcp1 00000400:00000200:20.0:1616513074.682633:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.24@tcp1(10.31.133.24@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.682638:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513074.682640:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.24@tcp1 routed destination 00000800:00000200:20.0:1616513074.682643:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.682645:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.682647:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.682651:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513074.682654:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.24@tcp1(10.31.133.24@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:20.0:1616513074.682656:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513074.682657:0:48870:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513074.682662:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513074.682667:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.682670:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.682672:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.682677:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.682682:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.682690:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.682693:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.682695:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.682700:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.682702:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd980 00000800:00000200:8.0:1616513074.682705:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.682707:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513074.682709:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.682716:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.682718:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.682724:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.682729:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22088 00000800:00000200:26.0:1616513074.682733:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.682736:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.732796:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513074.732832:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513074.732862:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.732872:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513074.732882:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.24@tcp1 : PUT - for me 00000800:00000200:22.0:1616513074.732890:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513074.732892:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.24@tcp1 of length 224 into portal 26 MB=0x6018c3fc34a00 00000400:00000200:4.0:1616513074.732899:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.24@tcp1 of length 224/224 into md 0xf97c5da9 [1] + 0 00000800:00000200:22.0:1616513074.732900:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:4.0:1616513074.732904:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774c38 00000400:00000200:4.0:1616513074.732907:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.732909:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.24@tcp1 : PUT - for me 00000100:00000200:4.0:1616513074.732912:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:16.0:1616513074.732917:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513074.732917:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774c38 00000400:00000200:22.0:1616513074.732918:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.24@tcp1 of length 224 into portal 12 MB=0x6018c3fc34a80 00000800:00000200:4.0:1616513074.732920:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.732923:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513074.732924:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.24@tcp1 of length 224/224 into md 0xf96450b9 [1] + 44368 00000800:00000200:4.0:1616513074.732925:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513074.732929:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.732932:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513074.732939:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.732942:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.732944:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.732947:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.732984:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690551737076224 00010000:00000200:20.0:1616513074.732994:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1690551737076224/t0(0) o400->8447777a-8b58-630d-f79f-652eb6219f4c@10.31.142.24@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.733007:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690551737076224, offset 192 00000400:00000200:20.0:1616513074.733012:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.24@tcp1 00000100:00000200:18.0:1616513074.733012:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690551737076352 00000400:00000200:20.0:1616513074.733017:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.24@tcp1 routed destination 00010000:00000200:18.0:1616513074.733025:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad3600 x1690551737076352/t0(0) o400->6f5d3274-a24b-714f-357e-c8c5ed130626@10.31.142.24@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513074.733030:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.24@tcp1(10.31.142.24@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.733036:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:18.0:1616513074.733040:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690551737076352, offset 192 00000800:00000200:20.0:1616513074.733041:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.733044:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.733045:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513074.733046:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.24@tcp1 00000800:00000200:20.0:1616513074.733050:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513074.733053:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.24@tcp1 routed destination 00000800:00000200:7.2:1616513074.733055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.733057:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513074.733069:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.24@tcp1(10.31.142.24@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.733076:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513074.733081:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.733084:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.733086:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.733091:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.733097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.733107:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513074.733110:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513074.733112:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.733114:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.733120:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513074.733124:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:8.0:1616513074.733124:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc330 00000400:00000200:26.0:1616513074.733127:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.733127:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.733129:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513074.733134:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.733140:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23760 00000800:00000200:26.0:1616513074.733143:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.733146:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513074.746563:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924624, portal 4 00000100:00000200:2.0:1616513074.746568:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924624, offset 0 00000400:00000200:2.0:1616513074.746574:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:2.0:1616513074.746580:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:2.0:1616513074.746593:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513074.746599:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:2.0:1616513074.746605:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513074.746608:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:2.0:1616513074.746610:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:2.0:1616513074.746615:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513074.746662:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513074.746719:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:16.0:1616513074.746726:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:16.0:1616513074.746729:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513074.746735:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:16.0:1616513074.746741:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d5a00 x1693067725924624/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513074.746753:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eeccc0 00000800:00000200:16.0:1616513074.746757:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513074.746760:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513074.746846:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513074.746856:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513074.746864:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513074.746873:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:28.0:1616513074.746878:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:8.0:1616513074.746883:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626910 00000400:00000200:8.0:1616513074.746889:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d3f91 [1] + 192 00000400:00000200:8.0:1616513074.746894:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.746898:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d5a00 x1693067725924624/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513074.746912:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513074.746915:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513074.746916:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:2.0:1616513074.746920:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d5a00 x1693067725924624/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513074.746933:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d5a00 x1693067725924624/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513074.746943:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed760 00000800:00000200:7.2:1616513074.805297:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.805356:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.805367:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.805376:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.88@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.805386:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.88@tcp1 of length 224 into portal 12 MB=0x5ffdd33e37180 00000400:00000200:2.0:1616513074.805393:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.88@tcp1 of length 224/224 into md 0xf96450b9 [1] + 44592 00000400:00000200:2.0:1616513074.805398:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.805401:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513074.805409:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.805412:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.805412:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.805417:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.805422:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513074.805492:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700406952320 00000800:00000200:16.0:1616513074.805497:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:18.0:1616513074.805505:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461208900 x1688700406952320/t0(0) o400->82d9d262-9945-f208-e72e-b6c299c49c20@10.31.138.88@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513074.805507:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513074.805516:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.88@tcp1 : PUT - for me 00000100:00000200:18.0:1616513074.805519:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700406952320, offset 192 00000400:00000200:18.0:1616513074.805525:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.88@tcp1 00000400:00000200:16.0:1616513074.805526:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.88@tcp1 of length 224 into portal 26 MB=0x5ffdd33e36d80 00000400:00000200:18.0:1616513074.805532:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.88@tcp1 routed destination 00000400:00000200:16.0:1616513074.805533:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.88@tcp1 of length 224/224 into md 0xf97c5dc9 [1] + 0 00000400:00000200:16.0:1616513074.805537:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77742a8 00000400:00000200:16.0:1616513074.805540:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513074.805543:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513074.805545:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.88@tcp1(10.31.138.88@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513074.805548:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77742a8 00000800:00000200:4.0:1616513074.805550:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513074.805551:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513074.805551:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.805554:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.805556:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513074.805557:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.805560:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513074.805562:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513074.805567:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.805572:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.805614:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700406951296 00010000:00000200:20.0:1616513074.805625:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688700406951296/t0(0) o400->a5da1624-ce45-5a54-6440-1d692b9d447d@10.31.138.88@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513074.805629:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.805637:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.805639:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:20.0:1616513074.805640:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700406951296, offset 192 00000400:00000200:26.0:1616513074.805646:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513074.805646:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.88@tcp1 00000400:00000200:26.0:1616513074.805650:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cce58 00000800:00000200:26.0:1616513074.805654:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513074.805654:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.88@tcp1 routed destination 00000800:00000200:26.0:1616513074.805657:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513074.805669:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.88@tcp1(10.31.138.88@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.805675:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.805681:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.805684:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.805686:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.805691:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.805696:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513074.805698:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513074.805750:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.805758:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.805760:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.805762:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513074.805767:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.805773:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22660 00000800:00000200:28.0:1616513074.805776:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.838280:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.838339:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.838349:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.838358:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.111@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.838368:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.111@tcp1 of length 224 into portal 26 MB=0x6001d34b83580 00000400:00000200:22.0:1616513074.838375:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.111@tcp1 of length 224/224 into md 0xf97c5de9 [1] + 0 00000400:00000200:22.0:1616513074.838380:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775430 00000400:00000200:22.0:1616513074.838385:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.838388:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513074.838392:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775430 00000800:00000200:2.0:1616513074.838394:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.838395:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.838398:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.838400:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.838446:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513074.838463:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513074.838470:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975298803072 00000800:00000200:4.0:1616513074.838473:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00010000:00000200:20.0:1616513074.838482:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688975298803072/t0(0) o400->56c335d7-cc56-f221-bf38-e649066cb69f@10.31.142.111@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:4.0:1616513074.838482:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.111@tcp1 : PUT - for me 00000400:00000200:4.0:1616513074.838492:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.111@tcp1 of length 224 into portal 12 MB=0x6001d34b83780 00000100:00000200:20.0:1616513074.838497:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688975298803072, offset 192 00000400:00000200:4.0:1616513074.838499:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.111@tcp1 of length 224/224 into md 0x865b9f31 [1] + 80376 00000400:00000200:4.0:1616513074.838505:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513074.838507:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.111@tcp1 00000100:00000200:4.0:1616513074.838510:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513074.838514:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.111@tcp1 routed destination 00000800:00000200:16.0:1616513074.838517:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.838521:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.838524:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.838525:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:20.0:1616513074.838530:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.111@tcp1(10.31.142.111@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.838537:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.838543:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.838546:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.838548:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.838553:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.838559:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.838560:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513074.838582:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975298803584 00010000:00000200:24.0:1616513074.838595:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b088480 x1688975298803584/t0(0) o400->b1396153-fb20-b899-35fb-abcedd34153d@10.31.142.111@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513074.838610:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688975298803584, offset 192 00000800:00000200:26.0:1616513074.838613:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513074.838616:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.111@tcp1 00000800:00000200:26.0:1616513074.838619:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.838622:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513074.838624:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.111@tcp1 routed destination 00000400:00000200:26.0:1616513074.838629:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.838635:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee223b8 00000800:00000200:26.0:1616513074.838638:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513074.838639:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.111@tcp1(10.31.142.111@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513074.838641:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513074.838645:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513074.838651:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.838654:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.838656:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513074.838661:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.838667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.838678:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.838684:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.838686:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.838693:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.838696:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000800:00000200:8.0:1616513074.838700:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.838703:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.840800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.840853:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.840863:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.840872:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.71@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.840880:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.71@tcp1 of length 224 into portal 26 MB=0x601ab16d447c0 00000400:00000200:2.0:1616513074.840886:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.71@tcp1 of length 224/224 into md 0xf97c5e09 [1] + 0 00000400:00000200:2.0:1616513074.840891:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77743b8 00000400:00000200:2.0:1616513074.840894:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.840897:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513074.840900:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77743b8 00000800:00000200:2.0:1616513074.840903:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.840907:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.840907:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.840911:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.840964:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690684194310080 00010000:00000200:20.0:1616513074.840975:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1690684194310080/t0(0) o400->f57cf01f-35ad-d4e1-8cda-fc90d35412a4@10.31.141.71@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513074.840977:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513074.840989:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690684194310080, offset 192 00000400:00000200:20.0:1616513074.840996:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.71@tcp1 00000400:00000200:20.0:1616513074.841003:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.71@tcp1 routed destination 00000400:00000200:20.0:1616513074.841018:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.71@tcp1(10.31.141.71@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.841025:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.841031:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513074.841031:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.841034:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.841036:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.841041:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513074.841041:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513074.841045:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.841047:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513074.841050:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.71@tcp1 : PUT - for me 00000400:00000200:16.0:1616513074.841058:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.71@tcp1 of length 224 into portal 12 MB=0x601ab16d44840 00000400:00000200:16.0:1616513074.841064:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.71@tcp1 of length 224/224 into md 0x865b9f31 [1] + 80600 00000400:00000200:16.0:1616513074.841070:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513074.841072:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513074.841079:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.841082:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513074.841085:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513074.841089:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513074.841107:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513074.841114:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.841119:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.841126:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.841131:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee234b8 00000800:00000200:26.0:1616513074.841134:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513074.841137:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513074.841146:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690684194310208 00010000:00000200:24.0:1616513074.841158:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1b00 x1690684194310208/t0(0) o400->044c6967-09ef-ee1f-c9c1-3a766762baae@10.31.141.71@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513074.841173:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690684194310208, offset 192 00000400:00000200:24.0:1616513074.841179:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.71@tcp1 00000400:00000200:24.0:1616513074.841186:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.71@tcp1 routed destination 00000400:00000200:24.0:1616513074.841211:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.71@tcp1(10.31.141.71@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.841217:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513074.841223:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513074.841225:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513074.841227:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513074.841232:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.841237:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.841246:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.841251:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.841253:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.841259:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.841262:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2110 00000800:00000200:8.0:1616513074.841265:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.841268:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513074.859118:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924640, portal 4 00000100:00000200:15.0:1616513074.859124:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924640, offset 0 00000400:00000200:15.0:1616513074.859130:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:15.0:1616513074.859136:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:15.0:1616513074.859149:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.859155:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:15.0:1616513074.859162:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.859165:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:15.0:1616513074.859167:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:15.0:1616513074.859172:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513074.859217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513074.859282:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513074.859290:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513074.859293:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513074.859300:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513074.859306:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd604380 x1693067725924640/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513074.859318:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cd48 00000800:00000200:22.0:1616513074.859322:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513074.859325:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513074.859385:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513074.859458:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513074.859467:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:26.0:1616513074.859476:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513074.859484:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626920 00000400:00000200:26.0:1616513074.859491:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c6469 [1] + 192 00000400:00000200:26.0:1616513074.859496:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513074.859501:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd604380 x1693067725924640/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513074.859517:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513074.859517:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513074.859519:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:26.0:1616513074.859521:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:15.0:1616513074.859526:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd604380 x1693067725924640/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513074.859538:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd604380 x1693067725924640/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513074.859548:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2df58 00000400:00000200:23.0:1616513074.870536:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:23.0:1616513074.870544:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:7.2:1616513074.870810:0:48890:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.870870:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.870880:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.870889:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.82@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.870899:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.82@tcp1 of length 224 into portal 12 MB=0x5ffd4dca5d580 00000400:00000200:22.0:1616513074.870906:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.82@tcp1 of length 224/224 into md 0xf96450b9 [1] + 44816 00000400:00000200:22.0:1616513074.870911:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.870914:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513074.870923:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.870924:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513074.870926:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.870929:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513074.870946:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664583558528 00010000:00000200:18.0:1616513074.870959:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad7980 x1688664583558528/t0(0) o400->ce57591e-5026-f4fe-2ec0-5722f78f9236@10.31.136.82@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513074.870973:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664583558528, offset 192 00000400:00000200:18.0:1616513074.870980:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.82@tcp1 00000800:00000200:11.2:1616513074.870981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513074.870988:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.82@tcp1 routed destination 00000400:00000200:18.0:1616513074.871004:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.82@tcp1(10.31.136.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.871010:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513074.871016:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.871019:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513074.871021:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513074.871026:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.871030:0:48890:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513074.871036:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.871045:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513074.871055:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.82@tcp1 : PUT - for me 00000400:00000200:4.0:1616513074.871063:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.82@tcp1 of length 224 into portal 26 MB=0x5ffd4dca5d440 00000400:00000200:4.0:1616513074.871070:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.82@tcp1 of length 224/224 into md 0xf97c5e19 [1] + 0 00000400:00000200:4.0:1616513074.871075:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775a90 00000400:00000200:4.0:1616513074.871078:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513074.871080:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513074.871085:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513074.871085:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775a90 00000800:00000200:4.0:1616513074.871088:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513074.871090:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513074.871091:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:4.0:1616513074.871092:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513074.871094:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513074.871095:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513074.871101:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.871105:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccb28 00000800:00000200:14.0:1616513074.871109:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.871112:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.871149:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664583558208 00010000:00000200:20.0:1616513074.871160:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688664583558208/t0(0) o400->4dd93725-b389-9170-51ba-33e79b2c9376@10.31.136.82@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.871175:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664583558208, offset 192 00000400:00000200:20.0:1616513074.871181:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.82@tcp1 00000400:00000200:20.0:1616513074.871188:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.82@tcp1 routed destination 00000400:00000200:20.0:1616513074.871204:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.82@tcp1(10.31.136.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.871210:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513074.871218:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.871220:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513074.871222:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513074.871227:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.871232:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513074.871234:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513074.871284:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.871291:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.871293:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.871300:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.871305:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22000 00000800:00000200:8.0:1616513074.871308:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.871311:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:11.0:1616513074.874558:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924656, portal 4 00000100:00000200:11.0:1616513074.874561:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924656, offset 0 00000400:00000200:11.0:1616513074.874566:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:11.0:1616513074.874571:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:11.0:1616513074.874581:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513074.874586:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:11.0:1616513074.874591:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513074.874593:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:11.0:1616513074.874594:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:11.0:1616513074.874598:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513074.874645:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513074.874666:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513074.874672:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513074.874675:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513074.874682:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513074.874689:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded2d00 x1693067725924656/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513074.874712:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3ba90 00000800:00000200:1.0:1616513074.874717:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513074.874720:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513074.874793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513074.874802:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513074.874808:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513074.874812:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513074.874816:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513074.874825:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626930 00000400:00000200:1.0:1616513074.874831:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ff9f5 [1] + 192 00000400:00000200:1.0:1616513074.874836:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513074.874841:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded2d00 x1693067725924656/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513074.874854:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513074.874856:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:11.0:1616513074.874858:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded2d00 x1693067725924656/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513074.874858:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:11.0:1616513074.874875:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded2d00 x1693067725924656/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513074.874882:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bc28 00000800:00000200:7.2:1616513074.875369:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.875424:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.875447:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.875456:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.209@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.875464:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.209@tcp1 of length 224 into portal 26 MB=0x5ffe4276d7100 00000400:00000200:2.0:1616513074.875471:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.209@tcp1 of length 224/224 into md 0xf97c5e29 [1] + 0 00000400:00000200:2.0:1616513074.875476:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77745d8 00000800:00000200:22.0:1616513074.875477:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513074.875478:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.875481:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513074.875485:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77745d8 00000800:00000200:2.0:1616513074.875488:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.875491:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.875493:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.875549:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688730262663424 00010000:00000200:20.0:1616513074.875560:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688730262663424/t0(0) o400->7acd755e-a0b1-0f42-672e-8eca3d2ced8a@10.31.128.209@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.875574:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688730262663424, offset 192 00000400:00000200:20.0:1616513074.875580:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.209@tcp1 00000400:00000200:20.0:1616513074.875588:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.209@tcp1 routed destination 00000400:00000200:20.0:1616513074.875603:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.209@tcp1(10.31.128.209@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.875609:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.875615:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513074.875618:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:7.2:1616513074.875619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.875620:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513074.875625:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:20.0:1616513074.875632:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513074.875673:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.875682:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.875691:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.209@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.875699:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.209@tcp1 of length 224 into portal 12 MB=0x5ffe4276d7480 00000400:00000200:14.0:1616513074.875706:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.209@tcp1 of length 224/224 into md 0xf96450b9 [1] + 45040 00000400:00000200:14.0:1616513074.875711:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.875713:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.875720:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513074.875723:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513074.875727:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:14.0:1616513074.875734:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513074.875735:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513074.875737:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.875738:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.875745:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.875750:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23d38 00000800:00000200:26.0:1616513074.875754:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513074.875783:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688730262664320 00010000:00000200:18.0:1616513074.875796:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e0480 x1688730262664320/t0(0) o400->bba65a1c-25cd-ee81-a956-8b5f1af2b730@10.31.128.209@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513074.875810:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688730262664320, offset 192 00000400:00000200:18.0:1616513074.875816:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.209@tcp1 00000400:00000200:18.0:1616513074.875824:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.209@tcp1 routed destination 00000400:00000200:18.0:1616513074.875839:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.209@tcp1(10.31.128.209@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.875845:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513074.875851:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.875853:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.875856:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.875860:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.875866:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.875919:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513074.875926:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513074.875929:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513074.875937:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513074.875941:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd430 00000800:00000200:4.0:1616513074.875945:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.875948:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.878372:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513074.878427:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.878447:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.878456:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.59@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.878466:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.59@tcp1 of length 224 into portal 26 MB=0x5ffd49a49a5c0 00000400:00000200:2.0:1616513074.878473:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.59@tcp1 of length 224/224 into md 0xf97c5e39 [1] + 0 00000400:00000200:2.0:1616513074.878477:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775ed0 00000400:00000200:2.0:1616513074.878480:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513074.878481:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:2.0:1616513074.878484:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513074.878488:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775ed0 00000800:00000200:2.0:1616513074.878491:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513074.878492:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513074.878494:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513074.878496:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.878499:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.878500:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.59@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.878509:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.59@tcp1 of length 224 into portal 12 MB=0x5ffd49a49a680 00000400:00000200:14.0:1616513074.878516:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.59@tcp1 of length 224/224 into md 0xf96450b9 [1] + 45264 00000400:00000200:14.0:1616513074.878521:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.878524:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.878531:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.878534:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.878536:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513074.878553:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663470220736 00010000:00000200:20.0:1616513074.878564:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688663470220736/t0(0) o400->9d8b6e30-2898-1314-a533-00895796539e@10.31.138.59@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513074.878578:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663470220736, offset 192 00000400:00000200:20.0:1616513074.878584:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.59@tcp1 00000400:00000200:20.0:1616513074.878592:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.59@tcp1 routed destination 00000100:00000200:18.0:1616513074.878593:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663470220928 00010000:00000200:18.0:1616513074.878605:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e0900 x1688663470220928/t0(0) o400->dede8388-3942-5d5d-ded7-a6a6b4bb9d4c@10.31.138.59@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513074.878607:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.59@tcp1(10.31.138.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513074.878613:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513074.878619:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:18.0:1616513074.878619:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663470220928, offset 192 00000800:00000200:20.0:1616513074.878622:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513074.878624:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513074.878626:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.59@tcp1 00000800:00000200:20.0:1616513074.878629:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513074.878633:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.59@tcp1 routed destination 00000800:00000200:7.2:1616513074.878634:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:20.0:1616513074.878641:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513074.878650:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.59@tcp1(10.31.138.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.878657:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513074.878662:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.878665:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.878667:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.878672:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.878679:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513074.878687:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513074.878691:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513074.878694:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.878696:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513074.878697:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513074.878698:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513074.878699:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513074.878703:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.878706:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513074.878709:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23b18 00000400:00000200:8.0:1616513074.878709:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cde48 00000800:00000200:26.0:1616513074.878712:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513074.878713:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.878715:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:31.0:1616513074.890470:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924672, portal 4 00000100:00000200:31.0:1616513074.890476:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924672, offset 0 00000400:00000200:31.0:1616513074.890482:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:31.0:1616513074.890488:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:31.0:1616513074.890501:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513074.890508:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:31.0:1616513074.890515:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513074.890518:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:31.0:1616513074.890520:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:31.0:1616513074.890526:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513074.890572:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513074.890643:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513074.890650:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513074.890654:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513074.890660:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513074.890667:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63b180 x1693067725924672/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513074.890679:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2b28 00000800:00000200:4.0:1616513074.890683:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513074.890686:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513074.890748:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513074.890759:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513074.890766:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513074.890775:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513074.890782:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626940 00000400:00000200:8.0:1616513074.890789:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d4021 [1] + 192 00000400:00000200:8.0:1616513074.890795:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513074.890812:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63b180 x1693067725924672/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513074.890820:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513074.890826:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513074.890829:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513074.890831:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:31.0:1616513074.890835:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63b180 x1693067725924672/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513074.890848:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63b180 x1693067725924672/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513074.890858:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000800:00000200:11.2:1616513074.893176:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513074.893230:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.893239:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513074.893248:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.175@tcp1 : PUT - for me 00000400:00000200:4.0:1616513074.893258:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.175@tcp1 of length 224 into portal 26 MB=0x5ffd41cc90b00 00000400:00000200:4.0:1616513074.893265:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.175@tcp1 of length 224/224 into md 0xf97c5e49 [1] + 0 00000400:00000200:4.0:1616513074.893270:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774a18 00000400:00000200:4.0:1616513074.893273:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513074.893276:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513074.893280:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774a18 00000800:00000200:16.0:1616513074.893283:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.893283:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513074.893286:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513074.893288:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.893291:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513074.893343:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661364640512 00000800:00000200:28.0:1616513074.893351:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:20.0:1616513074.893354:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688661364640512/t0(0) o400->2e3f7af4-cc49-e285-3a65-b4c595083314@10.31.138.175@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513074.893361:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513074.893363:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:20.0:1616513074.893368:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661364640512, offset 192 00000400:00000200:28.0:1616513074.893370:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.85@tcp1 : PUT - for me 00000800:00000200:8.0:1616513074.893370:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513074.893373:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:20.0:1616513074.893375:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.175@tcp1 00000400:00000200:8.0:1616513074.893378:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.85@tcp1 : PUT - for me 00000400:00000200:28.0:1616513074.893380:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.85@tcp1 of length 224 into portal 26 MB=0x5ffde3dc0b8c0 00000800:00000200:16.0:1616513074.893382:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513074.893383:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.175@tcp1 routed destination 00000800:00000200:4.0:1616513074.893383:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513074.893386:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.85@tcp1 of length 224/224 into md 0xf97c5e69 [1] + 0 00000400:00000200:28.0:1616513074.893391:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774dd0 00000400:00000200:28.0:1616513074.893394:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.893394:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.85@tcp1 of length 224 into portal 12 MB=0x5ffde3dc0bb80 00000400:00000200:16.0:1616513074.893396:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.175@tcp1 : PUT - for me 00000100:00000200:28.0:1616513074.893397:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513074.893399:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.175@tcp1(10.31.138.175@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513074.893400:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.85@tcp1 of length 224/224 into md 0x865b9f31 [1] + 80824 00000400:00000200:28.0:1616513074.893402:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774dd0 00000800:00000200:28.0:1616513074.893405:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:8.0:1616513074.893405:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513074.893406:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513074.893406:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.175@tcp1 of length 224 into portal 12 MB=0x5ffd41cc90bc0 00000100:00000200:8.0:1616513074.893407:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513074.893408:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513074.893410:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513074.893412:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513074.893413:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513074.893414:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513074.893415:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.175@tcp1 of length 224/224 into md 0x865b9f31 [1] + 81048 00000800:00000200:8.0:1616513074.893416:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513074.893417:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513074.893417:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513074.893421:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513074.893421:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513074.893423:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513074.893425:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704867416256 00000800:00000200:7.2:1616513074.893428:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513074.893429:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513074.893432:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:15.0:1616513074.893446:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688704867416256/t0(0) o400->a3581b9e-d306-979a-7f6e-b860e69a36d5@10.31.142.85@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513074.893449:0:6461:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513074.893453:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513074.893455:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.893457:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513074.893460:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704867416256, offset 192 00000400:00000200:15.0:1616513074.893467:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.85@tcp1 00000800:00000200:4.0:1616513074.893471:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:15.0:1616513074.893475:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.85@tcp1 routed destination 00000800:00000200:4.0:1616513074.893476:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513074.893478:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704867416960 00000800:00000200:4.0:1616513074.893478:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.893480:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513074.893483:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661364640704 00000800:00000200:22.0:1616513074.893485:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:8.0:1616513074.893489:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7080 x1688704867416960/t0(0) o400->f2db997c-5301-a7f4-28eb-a862e95a3dc3@10.31.142.85@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513074.893490:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.85@tcp1(10.31.142.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513074.893492:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.893494:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:24.0:1616513074.893496:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1f80 x1688661364640704/t0(0) o400->c286a46e-e2d1-d90c-10c4-2ab68ad6bf81@10.31.138.175@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513074.893497:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513074.893501:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:8.0:1616513074.893502:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704867416960, offset 192 00000800:00000200:15.0:1616513074.893503:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.893506:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513074.893507:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23540 00000800:00000200:15.0:1616513074.893508:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513074.893510:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513074.893511:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.85@tcp1 00000800:00000200:15.0:1616513074.893512:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513074.893513:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513074.893514:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661364640704, offset 192 00000800:00000200:11.2:1616513074.893516:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513074.893518:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.85@tcp1 routed destination 00000100:00000200:15.0:1616513074.893520:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513074.893522:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.175@tcp1 00000400:00000200:24.0:1616513074.893530:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.175@tcp1 routed destination 00000400:00000200:8.0:1616513074.893534:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.85@tcp1(10.31.142.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513074.893540:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513074.893545:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.175@tcp1(10.31.138.175@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513074.893552:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513074.893552:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513074.893555:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513074.893556:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513074.893558:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513074.893560:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513074.893561:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513074.893564:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:7.2:1616513074.893565:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513074.893569:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513074.893572:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513074.893576:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:8.0:1616513074.893578:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513074.893580:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:8.0:1616513074.893583:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:16.0:1616513074.893584:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513074.893586:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.893587:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2df58 00000800:00000200:28.0:1616513074.893589:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.893591:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513074.893592:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513074.893596:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2cc0 00000800:00000200:16.0:1616513074.893599:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513074.893620:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513074.893627:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513074.893629:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513074.893636:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513074.893640:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71298 00000800:00000200:2.0:1616513074.893644:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.893647:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.910360:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.910418:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.910428:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.910453:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.108@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.910463:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.108@tcp1 of length 224 into portal 26 MB=0x602ada2a91e80 00000400:00000200:22.0:1616513074.910470:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.108@tcp1 of length 224/224 into md 0xf97c5e79 [1] + 0 00000800:00000200:14.0:1616513074.910473:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513074.910474:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774e58 00000400:00000200:22.0:1616513074.910477:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.910480:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513074.910484:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.910485:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774e58 00000800:00000200:22.0:1616513074.910488:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513074.910491:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513074.910492:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.910493:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.108@tcp1 : PUT - for me 00000800:00000200:22.0:1616513074.910495:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.910502:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.108@tcp1 of length 224 into portal 12 MB=0x602ada2a91f00 00000100:00000200:15.0:1616513074.910508:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691794641854080 00000400:00000200:14.0:1616513074.910509:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.108@tcp1 of length 224/224 into md 0xf96450b9 [1] + 45488 00000400:00000200:14.0:1616513074.910514:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.910517:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:15.0:1616513074.910519:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1691794641854080/t0(0) o400->17b45124-cb65-c5c7-cf73-9fbfd5fdca58@10.31.141.108@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513074.910524:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.910527:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.910528:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.910534:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691794641854080, offset 192 00000400:00000200:15.0:1616513074.910540:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.108@tcp1 00000400:00000200:15.0:1616513074.910561:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.108@tcp1 routed destination 00000400:00000200:15.0:1616513074.910575:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.108@tcp1(10.31.141.108@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.910581:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.910587:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.910590:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.910592:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:18.0:1616513074.910596:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691794641854208 00000800:00000200:15.0:1616513074.910596:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.910601:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.910603:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:18.0:1616513074.910609:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e0d80 x1691794641854208/t0(0) o400->5a4acf17-8332-0d3a-25a5-ecd704a7417b@10.31.141.108@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513074.910623:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691794641854208, offset 192 00000400:00000200:18.0:1616513074.910630:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.108@tcp1 00000400:00000200:18.0:1616513074.910638:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.108@tcp1 routed destination 00000400:00000200:18.0:1616513074.910653:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.108@tcp1(10.31.141.108@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513074.910655:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513074.910660:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513074.910663:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513074.910666:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513074.910666:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513074.910666:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.910668:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.910670:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513074.910673:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513074.910675:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513074.910679:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c660 00000800:00000200:11.2:1616513074.910682:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513074.910684:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513074.910745:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513074.910751:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513074.910754:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513074.910761:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513074.910766:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd210 00000800:00000200:4.0:1616513074.910770:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.910773:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513074.928299:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513074.928310:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.928317:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513074.928325:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.179@tcp1 : PUT - for me 00000400:00000200:8.0:1616513074.928333:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.179@tcp1 of length 224 into portal 26 MB=0x60182bd8bb500 00000400:00000200:8.0:1616513074.928340:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.179@tcp1 of length 224/224 into md 0xf97c5e89 [1] + 0 00000400:00000200:8.0:1616513074.928344:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775078 00000400:00000200:8.0:1616513074.928346:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:7.2:1616513074.928348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:8.0:1616513074.928350:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513074.928354:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775078 00000800:00000200:8.0:1616513074.928357:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.928359:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.928361:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513074.928363:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513074.928404:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.928414:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:15.0:1616513074.928419:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510897689856 00000400:00000200:2.0:1616513074.928423:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.179@tcp1 : PUT - for me 00010000:00000200:15.0:1616513074.928431:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1690510897689856/t0(0) o400->5732845d-c027-747e-76da-d74c4cf29b3e@10.31.138.179@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513074.928432:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.179@tcp1 of length 224 into portal 12 MB=0x60182bd8bb5c0 00000400:00000200:2.0:1616513074.928451:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.179@tcp1 of length 224/224 into md 0xf96450b9 [1] + 45712 00000400:00000200:2.0:1616513074.928457:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:15.0:1616513074.928458:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510897689856, offset 192 00000800:00000200:14.0:1616513074.928459:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:2.0:1616513074.928459:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:15.0:1616513074.928464:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.179@tcp1 00000800:00000200:2.0:1616513074.928466:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.928469:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.928471:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513074.928472:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.179@tcp1 routed destination 00000400:00000200:15.0:1616513074.928488:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.179@tcp1(10.31.138.179@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.928499:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.928505:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.928508:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.928510:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.928515:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.928521:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.928523:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513074.928532:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510897690048 00010000:00000200:18.0:1616513074.928545:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120d100 x1690510897690048/t0(0) o400->68495f8b-de12-fc50-f722-742000ae3afc@10.31.138.179@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513074.928560:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510897690048, offset 192 00000400:00000200:18.0:1616513074.928566:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.179@tcp1 00000400:00000200:18.0:1616513074.928573:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.179@tcp1 routed destination 00000800:00000200:22.0:1616513074.928576:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.928585:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513074.928588:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.928588:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513074.928589:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.179@tcp1(10.31.138.179@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513074.928594:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513074.928595:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513074.928600:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d188 00000800:00000200:18.0:1616513074.928601:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513074.928604:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513074.928604:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.928608:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.928613:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.928618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513074.928669:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513074.928676:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513074.928678:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513074.928684:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513074.928688:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd540 00000800:00000200:4.0:1616513074.928692:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513074.928694:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:2.0:1616513074.938592:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924688, portal 4 00000100:00000200:2.0:1616513074.938598:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924688, offset 0 00000400:00000200:2.0:1616513074.938604:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:2.0:1616513074.938610:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:2.0:1616513074.938623:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513074.938629:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:2.0:1616513074.938635:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513074.938638:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:2.0:1616513074.938642:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:2.0:1616513074.938647:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513074.938694:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513074.938707:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513074.938712:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513074.938714:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.938720:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513074.938725:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56cbad5580 x1693067725924688/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513074.938736:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed540 00000800:00000200:8.0:1616513074.938740:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513074.938742:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513074.938867:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:4.0:1616513074.938922:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513074.938931:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:4.0:1616513074.938940:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:4.0:1616513074.938954:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626950 00000400:00000200:4.0:1616513074.938960:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d4061 [1] + 192 00000400:00000200:4.0:1616513074.938965:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513074.938971:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56cbad5580 x1693067725924688/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:4.0:1616513074.938985:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513074.938988:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:4.0:1616513074.938990:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513074.938993:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:2.0:1616513074.938993:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56cbad5580 x1693067725924688/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513074.939005:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56cbad5580 x1693067725924688/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513081 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513074.939015:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed760 00000800:00000200:7.2:1616513074.951423:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513074.951479:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.951489:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513074.951498:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.125@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.951506:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.125@tcp1 of length 224 into portal 26 MB=0x6018311aad240 00000400:00000200:14.0:1616513074.951513:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.125@tcp1 of length 224/224 into md 0xf97c5ea9 [1] + 0 00000400:00000200:14.0:1616513074.951518:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77755c8 00000400:00000200:14.0:1616513074.951521:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.951523:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513074.951528:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77755c8 00000800:00000200:14.0:1616513074.951538:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.951541:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513074.951543:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.951545:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.951551:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513074.951553:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.951557:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.125@tcp1 : PUT - for me 00000400:00000200:14.0:1616513074.951563:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.125@tcp1 of length 224 into portal 12 MB=0x6018311aad340 00000400:00000200:14.0:1616513074.951568:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.125@tcp1 of length 224/224 into md 0xf96450b9 [1] + 45936 00000400:00000200:14.0:1616513074.951572:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513074.951574:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513074.951585:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513074.951587:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513074.951589:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.951595:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512309015104 00010000:00000200:15.0:1616513074.951607:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1690512309015104/t0(0) o400->bf34e61d-cb14-0439-419f-e61a5e690530@10.31.136.125@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.951621:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512309015104, offset 192 00000400:00000200:15.0:1616513074.951627:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.125@tcp1 00000400:00000200:15.0:1616513074.951635:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.125@tcp1 routed destination 00000100:00000200:18.0:1616513074.951651:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512309015360 00000400:00000200:15.0:1616513074.951651:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.125@tcp1(10.31.136.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513074.951658:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:18.0:1616513074.951663:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e1200 x1690512309015360/t0(0) o400->007e8ba6-cf59-57ec-df48-d26eedbd4be7@10.31.136.125@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513074.951664:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.951667:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.951670:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.951675:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:18.0:1616513074.951678:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512309015360, offset 192 00000800:00000200:7.2:1616513074.951678:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.951683:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513074.951684:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.125@tcp1 00000400:00000200:18.0:1616513074.951692:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.125@tcp1 routed destination 00000400:00000200:18.0:1616513074.951707:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.125@tcp1(10.31.136.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.951714:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513074.951720:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.951723:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.951725:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.951730:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513074.951733:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.951741:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513074.951744:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513074.951745:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513074.951750:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513074.951757:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c110 00000800:00000200:22.0:1616513074.951760:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.951776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.951788:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.951792:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.951794:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.951799:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.951803:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc660 00000800:00000200:8.0:1616513074.951806:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.951809:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.957513:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513074.957524:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513074.957530:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513074.957538:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.135@tcp1 : PUT - for me 00000400:00000200:30.0:1616513074.957546:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.135@tcp1 of length 224 into portal 26 MB=0x602154b1bf6c0 00000400:00000200:30.0:1616513074.957552:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.135@tcp1 of length 224/224 into md 0xf97c5eb9 [1] + 0 00000400:00000200:30.0:1616513074.957556:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774f68 00000400:00000200:30.0:1616513074.957559:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513074.957561:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513074.957565:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774f68 00000800:00000200:30.0:1616513074.957568:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513074.957571:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513074.957573:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513074.957575:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513074.957580:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513074.957629:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691140337956544 00000800:00000200:2.0:1616513074.957634:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:15.0:1616513074.957640:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1691140337956544/t0(0) o400->7991c4c1-ff80-8394-c96b-24487abd10d0@10.31.140.135@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513074.957643:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.957652:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.135@tcp1 : PUT - for me 00000100:00000200:15.0:1616513074.957655:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691140337956544, offset 192 00000400:00000200:15.0:1616513074.957661:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.135@tcp1 00000400:00000200:2.0:1616513074.957661:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.135@tcp1 of length 224 into portal 12 MB=0x602154b1bf980 00000400:00000200:2.0:1616513074.957668:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.135@tcp1 of length 224/224 into md 0xf96450b9 [1] + 46160 00000400:00000200:15.0:1616513074.957669:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.135@tcp1 routed destination 00000400:00000200:2.0:1616513074.957673:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.957676:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:15.0:1616513074.957685:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.135@tcp1(10.31.140.135@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513074.957691:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513074.957692:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513074.957694:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513074.957694:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:15.0:1616513074.957698:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513074.957698:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:15.0:1616513074.957701:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.957705:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513074.957710:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.957715:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513074.957717:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513074.957727:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.957734:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513074.957736:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513074.957738:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.957744:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:18.0:1616513074.957746:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691140337957248 00000400:00000200:14.0:1616513074.957750:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c2a8 00000800:00000200:14.0:1616513074.957754:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:18.0:1616513074.957759:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120e780 x1691140337957248/t0(0) o400->610a4995-dc50-2c04-4012-7091cf7de8aa@10.31.140.135@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513074.957773:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691140337957248, offset 192 00000400:00000200:18.0:1616513074.957780:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.135@tcp1 00000400:00000200:18.0:1616513074.957787:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.135@tcp1 routed destination 00000400:00000200:18.0:1616513074.957802:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.135@tcp1(10.31.140.135@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.957809:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513074.957815:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.957818:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.957820:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.957825:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513074.957870:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513074.957927:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513074.957934:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513074.957937:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513074.957944:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513074.957948:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccee0 00000800:00000200:28.0:1616513074.957951:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513074.957955:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513074.995837:0:48903:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513074.995894:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513074.995904:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513074.995914:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.14@tcp1 : PUT - for me 00000400:00000200:22.0:1616513074.995923:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.14@tcp1 of length 224 into portal 26 MB=0x603d0aa5cc2c0 00000400:00000200:22.0:1616513074.995929:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.14@tcp1 of length 224/224 into md 0xf97c5ec9 [1] + 0 00000400:00000200:22.0:1616513074.995934:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774550 00000400:00000200:22.0:1616513074.995939:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513074.995942:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513074.995947:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774550 00000800:00000200:2.0:1616513074.995949:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.995950:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513074.995953:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513074.995957:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513074.995960:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.995960:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513074.995969:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.14@tcp1 : PUT - for me 00000400:00000200:2.0:1616513074.995977:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.14@tcp1 of length 224 into portal 12 MB=0x603d0aa5cc340 00000400:00000200:2.0:1616513074.995984:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.14@tcp1 of length 224/224 into md 0xf96450b9 [1] + 46384 00000400:00000200:2.0:1616513074.995989:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513074.995991:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513074.995998:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513074.996001:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513074.996003:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513074.996013:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693044606550720 00010000:00000200:15.0:1616513074.996025:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1693044606550720/t0(0) o400->6f43b0eb-679e-2185-f66d-20a83e8fe2ca@10.31.142.14@tcp1:31/0 lens 224/224 e 0 to 0 dl 1616513166 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513074.996039:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693044606550720, offset 192 00000400:00000200:15.0:1616513074.996045:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.14@tcp1 00000400:00000200:15.0:1616513074.996054:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.14@tcp1 routed destination 00000100:00000200:18.0:1616513074.996063:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693044606550848 00000400:00000200:15.0:1616513074.996069:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.14@tcp1(10.31.142.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:18.0:1616513074.996076:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120a880 x1693044606550848/t0(0) o400->a01d5d97-7d0c-4d9a-a0f6-ce2db3802e62@10.31.142.14@tcp1:700/0 lens 224/224 e 0 to 0 dl 1616513080 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513074.996076:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513074.996083:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513074.996086:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513074.996088:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:18.0:1616513074.996090:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693044606550848, offset 192 00000800:00000200:15.0:1616513074.996093:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513074.996097:0:48903:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513074.996098:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.14@tcp1 00000100:00000200:15.0:1616513074.996101:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513074.996118:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.14@tcp1 routed destination 00000400:00000200:18.0:1616513074.996133:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.14@tcp1(10.31.142.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513074.996139:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513074.996145:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513074.996147:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513074.996149:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513074.996154:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513074.996158:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513074.996166:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513074.996169:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513074.996169:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513074.996176:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513074.996182:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2ccc0 00000800:00000200:14.0:1616513074.996186:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513074.996199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.996213:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.996219:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513074.996222:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513074.996228:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513074.996232:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd188 00000800:00000200:8.0:1616513074.996235:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.996238:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513074.996244:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513074.996246:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513074.996248:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513074.996249:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513074.996252:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513075.018584:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924704, portal 4 00000100:00000200:26.0:1616513075.018590:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924704, offset 0 00000400:00000200:26.0:1616513075.018596:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:26.0:1616513075.018602:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:26.0:1616513075.018616:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513075.018622:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:26.0:1616513075.018628:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513075.018631:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:26.0:1616513075.018634:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:26.0:1616513075.018639:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513075.018686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513075.018706:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513075.018712:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513075.018715:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513075.018722:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513075.018729:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d2400 x1693067725924704/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513075.018741:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af430 00000800:00000200:27.0:1616513075.018745:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513075.018748:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513075.018822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513075.018832:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513075.018839:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513075.018847:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513075.018857:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626960 00000400:00000200:27.0:1616513075.018864:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37165bbd [1] + 192 00000400:00000200:27.0:1616513075.018870:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513075.018874:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d2400 x1693067725924704/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513075.018884:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513075.018889:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513075.018892:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513075.018893:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:26.0:1616513075.018898:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d2400 x1693067725924704/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513075.018911:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d2400 x1693067725924704/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513075.018921:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:11.2:1616513075.020556:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513075.020572:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513075.020581:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513075.020581:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513075.020588:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513075.020589:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.94@tcp1 : PUT - for me 00000400:00000200:8.0:1616513075.020596:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.94@tcp1 : PUT - for me 00000400:00000200:4.0:1616513075.020598:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.94@tcp1 of length 224 into portal 26 MB=0x60194e4da4580 00000400:00000200:8.0:1616513075.020604:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.94@tcp1 of length 224 into portal 12 MB=0x60194e4da4780 00000400:00000200:4.0:1616513075.020605:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.94@tcp1 of length 224/224 into md 0xf97c5ed9 [1] + 0 00000400:00000200:4.0:1616513075.020609:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774990 00000400:00000200:8.0:1616513075.020610:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.94@tcp1 of length 224/224 into md 0x865b9f31 [1] + 81272 00000400:00000200:4.0:1616513075.020612:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513075.020614:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513075.020615:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513075.020618:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:4.0:1616513075.020619:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774990 00000800:00000200:4.0:1616513075.020622:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513075.020624:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513075.020627:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:4.0:1616513075.020627:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:8.0:1616513075.020629:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513075.020630:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.020632:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.020684:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588866561408 00000100:00000200:24.0:1616513075.020691:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588866561920 00010000:00000200:15.0:1616513075.020696:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1690588866561408/t0(0) o400->08aeb1e7-7bea-0ccf-acc8-fbe3b4710c5c@10.31.140.94@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:24.0:1616513075.020704:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5e80 x1690588866561920/t0(0) o400->893df218-5f71-a998-8684-819fcf103fb9@10.31.140.94@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513075.020710:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588866561408, offset 192 00000400:00000200:15.0:1616513075.020716:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.94@tcp1 00000100:00000200:24.0:1616513075.020718:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588866561920, offset 192 00000400:00000200:24.0:1616513075.020725:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.94@tcp1 00000400:00000200:15.0:1616513075.020725:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.94@tcp1 routed destination 00000400:00000200:24.0:1616513075.020733:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.94@tcp1 routed destination 00000400:00000200:15.0:1616513075.020741:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.94@tcp1(10.31.140.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.020749:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513075.020750:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.94@tcp1(10.31.140.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.020756:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.020758:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.020759:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.020761:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.020764:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.020766:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.020766:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513075.020769:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513075.020772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.020773:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513075.020773:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513075.020778:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513075.020826:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.020834:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.020837:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.020844:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513075.020846:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513075.020850:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c198 00000800:00000200:28.0:1616513075.020853:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513075.020854:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513075.020856:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513075.020857:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513075.020857:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.020865:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.020869:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf22a8 00000800:00000200:28.0:1616513075.020872:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.034475:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513075.034488:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513075.034496:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513075.034505:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.36@tcp1 : PUT - for me 00000400:00000200:8.0:1616513075.034514:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.36@tcp1 of length 224 into portal 26 MB=0x5ffdce5a1f280 00000400:00000200:8.0:1616513075.034520:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.36@tcp1 of length 224/224 into md 0xf97c5ee9 [1] + 0 00000400:00000200:8.0:1616513075.034525:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775650 00000400:00000200:8.0:1616513075.034527:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513075.034530:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513075.034534:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775650 00000800:00000200:8.0:1616513075.034537:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.034539:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.034541:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.034545:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.034598:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699094037120 00010000:00000200:15.0:1616513075.034610:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688699094037120/t0(0) o400->21befdc4-b491-e9fc-9068-228e4d260e50@10.31.140.36@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513075.034614:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513075.034625:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.034626:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699094037120, offset 192 00000800:00000200:30.0:1616513075.034633:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:15.0:1616513075.034633:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.36@tcp1 00000400:00000200:30.0:1616513075.034641:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.36@tcp1 : PUT - for me 00000400:00000200:15.0:1616513075.034641:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.36@tcp1 routed destination 00000400:00000200:15.0:1616513075.034658:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.36@tcp1(10.31.140.36@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513075.034665:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.36@tcp1 of length 224 into portal 12 MB=0x5ffdce5a1f300 00000800:00000200:15.0:1616513075.034665:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:30.0:1616513075.034670:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.36@tcp1 of length 224/224 into md 0xf96450b9 [1] + 46608 00000800:00000200:15.0:1616513075.034672:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513075.034675:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:15.0:1616513075.034675:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:30.0:1616513075.034677:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:15.0:1616513075.034677:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513075.034679:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513075.034683:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:15.0:1616513075.034683:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513075.034686:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:7.2:1616513075.034687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (23)++ 00000100:00000200:15.0:1616513075.034690:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513075.034691:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513075.034693:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513075.034697:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513075.034699:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513075.034704:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513075.034708:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2ce58 00000800:00000200:30.0:1616513075.034710:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513075.034712:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.034747:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699094037248 00010000:00000200:18.0:1616513075.034760:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7d580 x1688699094037248/t0(0) o400->c09e4994-4089-bc2e-c832-311584cbad69@10.31.140.36@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.034774:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699094037248, offset 192 00000400:00000200:18.0:1616513075.034781:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.36@tcp1 00000400:00000200:18.0:1616513075.034789:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.36@tcp1 routed destination 00000400:00000200:18.0:1616513075.034804:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.36@tcp1(10.31.140.36@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.034811:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.034817:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.034819:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.034822:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.034826:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.034832:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513075.034883:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513075.034889:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513075.034892:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513075.034897:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513075.034901:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd078 00000800:00000200:4.0:1616513075.034904:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513075.034907:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.041630:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513075.041686:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.041696:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513075.041698:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.041705:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.178@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.041715:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.178@tcp1 of length 224 into portal 26 MB=0x6033c0e8f7180 00000400:00000200:16.0:1616513075.041722:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.178@tcp1 of length 224/224 into md 0xf97c5ef9 [1] + 0 00000400:00000200:16.0:1616513075.041726:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77744c8 00000400:00000200:16.0:1616513075.041729:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.041732:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513075.041737:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77744c8 00000800:00000200:16.0:1616513075.041740:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.041743:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.041744:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.041798:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692406337466752 00010000:00000200:15.0:1616513075.041809:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1692406337466752/t0(0) o400->3e6e2986-43d8-56bb-6f66-09a529392c2f@10.31.136.178@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.041822:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692406337466752, offset 192 00000400:00000200:15.0:1616513075.041828:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.178@tcp1 00000400:00000200:15.0:1616513075.041836:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.178@tcp1 routed destination 00000400:00000200:15.0:1616513075.041851:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.178@tcp1(10.31.136.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.041857:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.041863:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.041865:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.041867:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.041872:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513075.041878:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.041878:0:48908:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.041932:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.041940:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513075.041943:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513075.041943:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513075.041950:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.041956:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c000 00000800:00000200:14.0:1616513075.041960:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.042256:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513075.042306:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513075.042314:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513075.042321:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.178@tcp1 : PUT - for me 00000400:00000200:4.0:1616513075.042329:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.178@tcp1 of length 224 into portal 12 MB=0x6033c0e8f72c0 00000400:00000200:4.0:1616513075.042335:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.178@tcp1 of length 224/224 into md 0x865b9f31 [1] + 81496 00000400:00000200:4.0:1616513075.042339:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513075.042342:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513075.042348:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513075.042351:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513075.042352:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513075.042355:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.042413:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692406337467072 00010000:00000200:24.0:1616513075.042425:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08d100 x1692406337467072/t0(0) o400->0d905ba0-d215-e6f2-b5d4-5e8140e9abd8@10.31.136.178@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.042453:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692406337467072, offset 192 00000400:00000200:24.0:1616513075.042459:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.178@tcp1 00000400:00000200:24.0:1616513075.042467:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.178@tcp1 routed destination 00000400:00000200:24.0:1616513075.042482:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.178@tcp1(10.31.136.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.042489:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.042495:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.042499:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.042501:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.042506:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.042511:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513075.042520:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.042524:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513075.042526:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513075.042529:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513075.042531:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513075.042535:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:8.0:1616513075.042538:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.050338:0:48909:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.050393:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.050403:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.050412:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.57@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.050421:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.57@tcp1 of length 224 into portal 12 MB=0x5ffd59d0f43c0 00000400:00000200:2.0:1616513075.050429:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.57@tcp1 of length 224/224 into md 0xf96450b9 [1] + 46832 00000800:00000200:22.0:1616513075.050447:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513075.050449:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.050451:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.050458:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.050461:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.050463:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513075.050512:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924720, portal 4 00000100:00000200:22.0:1616513075.050517:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924720, offset 0 00000100:00000200:18.0:1616513075.050522:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667811693504 00000400:00000200:22.0:1616513075.050523:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513075.050529:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00010000:00000200:18.0:1616513075.050535:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120f500 x1688667811693504/t0(0) o400->e9284cfc-b778-8512-3b05-e63f91ce44d1@10.31.129.57@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:22.0:1616513075.050542:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513075.050548:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000100:00000200:18.0:1616513075.050549:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667811693504, offset 192 00000800:00000200:22.0:1616513075.050555:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513075.050556:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.57@tcp1 00000800:00000200:22.0:1616513075.050558:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513075.050560:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513075.050564:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513075.050564:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.57@tcp1 routed destination 00000400:00000200:18.0:1616513075.050578:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.57@tcp1(10.31.129.57@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.050585:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.050591:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.050595:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.050597:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.050602:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.050608:0:48909:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:25.2:1616513075.050613:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513075.050622:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513075.050628:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513075.050630:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513075.050632:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000400:00000200:30.0:1616513075.050637:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513075.050638:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:30.0:1616513075.050640:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc088 00000400:00000200:22.0:1616513075.050641:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.050644:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513075.050646:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:22.0:1616513075.050646:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:2.0:1616513075.050648:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000100:00000200:22.0:1616513075.050651:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9641f80 x1693067725924720/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513075.050663:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a7f8 00000800:00000200:22.0:1616513075.050666:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:11.2:1616513075.050745:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:25.2:1616513075.050777:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513075.050785:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513075.050791:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:2.0:1616513075.050793:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:30.0:1616513075.050797:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:4.0:1616513075.050798:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513075.050803:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626970 00000800:00000200:4.0:1616513075.050806:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:30.0:1616513075.050808:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c65f9 [1] + 192 00000400:00000200:30.0:1616513075.050813:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:4.0:1616513075.050813:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.57@tcp1 : PUT - for me 00000100:00000200:30.0:1616513075.050817:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9641f80 x1693067725924720/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513075.050821:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.57@tcp1 of length 224 into portal 26 MB=0x5ffd59d0f4080 00000800:00000200:30.0:1616513075.050828:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513075.050830:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000400:00000200:4.0:1616513075.050830:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.57@tcp1 of length 224/224 into md 0xf97c5f09 [1] + 0 00000800:00000200:30.0:1616513075.050832:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000400:00000200:4.0:1616513075.050834:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775760 00000400:00000200:4.0:1616513075.050836:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.050839:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9641f80 x1693067725924720/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513075.050851:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513075.050851:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9641f80 x1693067725924720/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513075.050859:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000100:00000200:4.0:1616513075.050862:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513075.050866:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775760 00000800:00000200:4.0:1616513075.050868:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513075.050870:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513075.050872:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.050928:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667811692672 00010000:00000200:15.0:1616513075.050938:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688667811692672/t0(0) o400->134dcdf4-e4ee-9423-23eb-ca3172b8cd5b@10.31.129.57@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.050951:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667811692672, offset 192 00000400:00000200:15.0:1616513075.050957:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.57@tcp1 00000400:00000200:15.0:1616513075.050965:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.57@tcp1 routed destination 00000400:00000200:15.0:1616513075.050981:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.57@tcp1(10.31.129.57@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.050988:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.050994:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.050996:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.050999:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.051003:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.051008:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.051010:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513075.051061:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.051069:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.051072:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513075.051072:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.051079:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.051086:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2dba0 00000800:00000200:16.0:1616513075.051089:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.058673:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.058729:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.058740:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513075.058741:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.058748:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.058749:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.203@tcp1 : PUT - for me 00000400:00000200:8.0:1616513075.058755:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.203@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.058758:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.203@tcp1 of length 224 into portal 26 MB=0x5ffe0c52f1580 00000400:00000200:8.0:1616513075.058763:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.203@tcp1 of length 224 into portal 12 MB=0x5ffe0c52f1880 00000400:00000200:28.0:1616513075.058765:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.203@tcp1 of length 224/224 into md 0xf97c5f19 [1] + 0 00000400:00000200:8.0:1616513075.058769:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.203@tcp1 of length 224/224 into md 0x865b9f31 [1] + 81720 00000400:00000200:28.0:1616513075.058770:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774880 00000400:00000200:28.0:1616513075.058773:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513075.058773:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513075.058775:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:28.0:1616513075.058776:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.058781:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774880 00000800:00000200:8.0:1616513075.058781:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513075.058784:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513075.058785:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:8.0:1616513075.058785:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.058787:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513075.058787:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.058789:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.058841:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688715729507712 00000100:00000200:24.0:1616513075.058847:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688715729508480 00010000:00000200:15.0:1616513075.058851:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688715729507712/t0(0) o400->fabf1fe6-0d44-ff49-f669-5bd7f2ca2b33@10.31.142.203@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:24.0:1616513075.058859:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4c80 x1688715729508480/t0(0) o400->43ec5197-f756-9ec8-e588-ae5b5e33a231@10.31.142.203@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513075.058863:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688715729507712, offset 192 00000400:00000200:15.0:1616513075.058869:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.203@tcp1 00000100:00000200:24.0:1616513075.058874:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688715729508480, offset 192 00000400:00000200:15.0:1616513075.058876:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.203@tcp1 routed destination 00000400:00000200:24.0:1616513075.058880:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.203@tcp1 00000400:00000200:24.0:1616513075.058889:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.203@tcp1 routed destination 00000400:00000200:15.0:1616513075.058890:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.203@tcp1(10.31.142.203@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.058896:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.058901:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.058904:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513075.058905:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.203@tcp1(10.31.142.203@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.058906:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.058910:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513075.058911:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513075.058915:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.058916:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513075.058917:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.058920:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.058922:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.058927:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.058932:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513075.058968:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.058975:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.058979:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.058985:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.058991:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c990 00000800:00000200:22.0:1616513075.058995:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.058998:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513075.059000:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.059006:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513075.059008:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.059009:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513075.059015:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.059019:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3650 00000800:00000200:16.0:1616513075.059022:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.059989:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.060044:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513075.060054:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.060054:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.060063:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.231@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.060072:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.231@tcp1 of length 224 into portal 12 MB=0x5ffed998b3e40 00000400:00000200:2.0:1616513075.060078:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.231@tcp1 of length 224/224 into md 0xf96450b9 [1] + 47056 00000400:00000200:2.0:1616513075.060083:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.060086:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.060092:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.060095:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.060097:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.060157:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688770831924800 00010000:00000200:18.0:1616513075.060169:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120b180 x1688770831924800/t0(0) o400->c1b766dd-8309-c1d8-40d1-67e1f397db77@10.31.140.231@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.060183:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688770831924800, offset 192 00000400:00000200:18.0:1616513075.060189:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.231@tcp1 00000400:00000200:18.0:1616513075.060197:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.231@tcp1 routed destination 00000400:00000200:18.0:1616513075.060212:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.231@tcp1(10.31.140.231@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.060218:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513075.060222:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.060224:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.060227:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.060229:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.060234:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.060238:0:48910:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513075.060278:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.060287:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.060295:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.231@tcp1 : PUT - for me 00000800:00000200:22.0:1616513075.060296:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.060303:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513075.060304:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.231@tcp1 of length 224 into portal 26 MB=0x5ffed998b3d80 00000400:00000200:22.0:1616513075.060306:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.060311:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.231@tcp1 of length 224/224 into md 0xf97c5f29 [1] + 0 00000400:00000200:22.0:1616513075.060312:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.060316:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774ee0 00000400:00000200:28.0:1616513075.060319:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.060319:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cded0 00000100:00000200:28.0:1616513075.060322:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513075.060323:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.060325:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513075.060327:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774ee0 00000800:00000200:28.0:1616513075.060330:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.060330:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.060333:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.060335:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.060387:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688770831924608 00010000:00000200:15.0:1616513075.060397:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688770831924608/t0(0) o400->a2233e78-aeee-cdfe-4591-7548970fdb8b@10.31.140.231@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.060409:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688770831924608, offset 192 00000400:00000200:15.0:1616513075.060414:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.231@tcp1 00000400:00000200:15.0:1616513075.060421:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.231@tcp1 routed destination 00000400:00000200:15.0:1616513075.060447:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.231@tcp1(10.31.140.231@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.060453:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.060458:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.060462:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.060464:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.060469:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.060474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.060475:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513075.060482:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.060485:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513075.060487:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513075.060492:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513075.060497:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c088 00000800:00000200:8.0:1616513075.060499:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.060501:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.075375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.075430:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.075455:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.075464:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.39@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.075474:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.39@tcp1 of length 224 into portal 26 MB=0x5ffd51b023880 00000400:00000200:12.0:1616513075.075480:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.39@tcp1 of length 224/224 into md 0xf97c5f39 [1] + 0 00000800:00000200:28.0:1616513075.075484:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513075.075485:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774088 00000400:00000200:12.0:1616513075.075488:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.075491:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513075.075494:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513075.075495:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.075496:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774088 00000800:00000200:12.0:1616513075.075499:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.075502:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.075503:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.39@tcp1 : PUT - for me 00000800:00000200:12.0:1616513075.075504:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.075512:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.39@tcp1 of length 224 into portal 12 MB=0x5ffd51b023900 00000400:00000200:28.0:1616513075.075518:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.39@tcp1 of length 224/224 into md 0x865b9f31 [1] + 81944 00000400:00000200:28.0:1616513075.075526:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.075529:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.075535:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.075538:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.075540:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.075567:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665629800704 00000100:00000200:15.0:1616513075.075568:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665629800576 00010000:00000200:15.0:1616513075.075578:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688665629800576/t0(0) o400->30fd0063-d3a7-1703-6b51-6a86f8d7639b@10.31.138.39@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:24.0:1616513075.075580:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf86c00 x1688665629800704/t0(0) o400->e2986215-a4b8-31a4-30e5-da6902e8c7d3@10.31.138.39@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513075.075590:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665629800576, offset 192 00000100:00000200:24.0:1616513075.075595:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665629800704, offset 192 00000400:00000200:15.0:1616513075.075596:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.39@tcp1 00000400:00000200:24.0:1616513075.075602:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.39@tcp1 00000400:00000200:15.0:1616513075.075604:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.39@tcp1 routed destination 00000400:00000200:24.0:1616513075.075611:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.39@tcp1 routed destination 00000400:00000200:15.0:1616513075.075618:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.39@tcp1(10.31.138.39@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.075624:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513075.075628:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.39@tcp1(10.31.138.39@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.075630:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.075633:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.075634:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.075635:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.075639:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513075.075641:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.075643:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513075.075644:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.075645:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513075.075646:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.075650:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513075.075655:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513075.075655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513075.075663:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513075.075666:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513075.075672:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513075.075677:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c550 00000800:00000200:30.0:1616513075.075680:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513075.075682:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513075.075712:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.075719:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.075721:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513075.075721:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.075727:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.075731:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:16.0:1616513075.075735:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.088959:0:48913:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513075.089014:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.089024:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.089033:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.170@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.089041:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.170@tcp1 of length 224 into portal 26 MB=0x5ffde3356bb00 00000400:00000200:22.0:1616513075.089048:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.170@tcp1 of length 224/224 into md 0xf97c5f59 [1] + 0 00000400:00000200:22.0:1616513075.089053:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775f58 00000400:00000200:22.0:1616513075.089056:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.089059:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513075.089064:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775f58 00000800:00000200:22.0:1616513075.089067:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.089068:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.089070:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.089072:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.089123:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704692697856 00010000:00000200:15.0:1616513075.089132:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688704692697856/t0(0) o400->4b257fad-2303-6285-7566-7f7b1ec8d13a@10.31.132.170@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.089143:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704692697856, offset 192 00000400:00000200:15.0:1616513075.089149:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.170@tcp1 00000400:00000200:15.0:1616513075.089155:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.170@tcp1 routed destination 00000400:00000200:15.0:1616513075.089168:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.170@tcp1(10.31.132.170@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.089173:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.089179:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.089181:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.089183:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.089187:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.089191:0:48913:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.089193:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513075.089200:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513075.089205:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513075.089207:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513075.089212:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513075.089217:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cb28 00000800:00000200:30.0:1616513075.089219:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513075.089222:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513075.089227:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513075.089230:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513075.089234:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.170@tcp1 : PUT - for me 00000400:00000200:30.0:1616513075.089240:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.170@tcp1 of length 224 into portal 12 MB=0x5ffde3356bc00 00000400:00000200:30.0:1616513075.089246:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.170@tcp1 of length 224/224 into md 0xf96450b9 [1] + 47280 00000400:00000200:30.0:1616513075.089251:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513075.089253:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513075.089258:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513075.089260:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513075.089262:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.089321:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704692698112 00010000:00000200:18.0:1616513075.089333:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7c800 x1688704692698112/t0(0) o400->af1480a9-a2af-17f8-b9d9-3d3ac769b41a@10.31.132.170@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.089347:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704692698112, offset 192 00000400:00000200:18.0:1616513075.089353:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.170@tcp1 00000400:00000200:18.0:1616513075.089360:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.170@tcp1 routed destination 00000400:00000200:18.0:1616513075.089374:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.170@tcp1(10.31.132.170@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.089380:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.089386:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.089389:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.089391:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.089395:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.089402:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.089455:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.089462:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.089465:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513075.089471:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.089475:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdf58 00000800:00000200:12.0:1616513075.089479:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.089482:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.090853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513075.090863:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513075.090870:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513075.090878:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.88@tcp1 : PUT - for me 00000400:00000200:8.0:1616513075.090887:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.88@tcp1 of length 224 into portal 26 MB=0x5ffdf1e7f5f40 00000400:00000200:8.0:1616513075.090893:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.88@tcp1 of length 224/224 into md 0xf97c5f79 [1] + 0 00000400:00000200:8.0:1616513075.090898:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77758f8 00000400:00000200:8.0:1616513075.090900:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513075.090903:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513075.090907:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77758f8 00000800:00000200:8.0:1616513075.090910:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.090912:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513075.090914:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.090916:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513075.090920:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513075.090926:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.88@tcp1 : PUT - for me 00000400:00000200:8.0:1616513075.090931:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.88@tcp1 of length 224 into portal 12 MB=0x5ffdf1e7f6200 00000400:00000200:8.0:1616513075.090937:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.88@tcp1 of length 224/224 into md 0x865b9f31 [1] + 82168 00000400:00000200:8.0:1616513075.090941:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513075.090943:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513075.090947:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.090949:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513075.090950:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.090953:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.090972:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708638007104 00010000:00000200:15.0:1616513075.090983:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688708638007104/t0(0) o400->d2766df1-1e17-8d19-7974-29c836ea7bfb@10.31.141.88@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.090997:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688708638007104, offset 192 00000400:00000200:15.0:1616513075.091003:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.88@tcp1 00000400:00000200:15.0:1616513075.091011:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.88@tcp1 routed destination 00000100:00000200:24.0:1616513075.091014:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708638007808 00010000:00000200:24.0:1616513075.091027:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0900 x1688708638007808/t0(0) o400->3c996321-e97d-c96e-30a7-b108427b2bea@10.31.141.88@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513075.091027:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.88@tcp1(10.31.141.88@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.091034:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.091040:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.091043:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.091045:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.091051:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513075.091053:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688708638007808, offset 192 00000800:00000200:7.2:1616513075.091056:0:48913:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.091058:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513075.091059:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.88@tcp1 00000400:00000200:24.0:1616513075.091067:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.88@tcp1 routed destination 00000400:00000200:24.0:1616513075.091082:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.88@tcp1(10.31.141.88@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.091088:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.091094:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.091097:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.091099:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.091104:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.091109:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513075.091111:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.091118:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.091122:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.091130:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.091136:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2ddc0 00000800:00000200:22.0:1616513075.091140:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.091143:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513075.091161:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.091168:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.091171:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513075.091173:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.091177:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.091181:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3760 00000800:00000200:28.0:1616513075.091185:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.093011:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.093065:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.093075:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513075.093076:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513075.093083:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.093084:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.95@tcp1 : PUT - for me 00000400:00000200:30.0:1616513075.093090:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.95@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.093093:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.95@tcp1 of length 224 into portal 26 MB=0x5ffd50b4db280 00000400:00000200:30.0:1616513075.093098:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.95@tcp1 of length 224 into portal 12 MB=0x5ffd50b4db440 00000400:00000200:2.0:1616513075.093100:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.95@tcp1 of length 224/224 into md 0xf97c5f89 [1] + 0 00000400:00000200:2.0:1616513075.093105:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775540 00000400:00000200:2.0:1616513075.093108:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513075.093109:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.95@tcp1 of length 224/224 into md 0xf96450b9 [1] + 47504 00000400:00000200:30.0:1616513075.093114:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.093114:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513075.093118:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775540 00000100:00000200:30.0:1616513075.093121:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.093122:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513075.093124:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513075.093126:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513075.093126:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513075.093129:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.093129:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513075.093133:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.093182:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665366311552 00000100:00000200:18.0:1616513075.093191:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665366312000 00010000:00000200:15.0:1616513075.093193:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688665366311552/t0(0) o400->e20c45e7-260d-64eb-d06f-690f4726a226@10.31.131.95@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:18.0:1616513075.093204:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7c380 x1688665366312000/t0(0) o400->2d32e117-9141-229c-801f-ac74f3a5075d@10.31.131.95@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513075.093208:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665366311552, offset 192 00000400:00000200:15.0:1616513075.093214:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.95@tcp1 00000400:00000200:15.0:1616513075.093223:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.95@tcp1 routed destination 00000100:00000200:18.0:1616513075.093230:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665366312000, offset 192 00000400:00000200:18.0:1616513075.093237:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.95@tcp1 00000400:00000200:15.0:1616513075.093238:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.95@tcp1(10.31.131.95@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.093245:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513075.093247:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.95@tcp1 routed destination 00000800:00000200:15.0:1616513075.093251:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.093254:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.093256:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.093261:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513075.093263:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.95@tcp1(10.31.131.95@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513075.093265:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.093269:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513075.093270:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.093276:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.093278:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.093280:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.093285:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.093290:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513075.093299:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.093303:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513075.093305:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513075.093310:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513075.093314:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdf58 00000800:00000200:14.0:1616513075.093317:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513075.093317:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.093319:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513075.093325:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.093328:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.093329:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513075.093335:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.093341:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2da90 00000800:00000200:14.0:1616513075.093345:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513075.114539:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924736, portal 4 00000100:00000200:26.0:1616513075.114545:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924736, offset 0 00000400:00000200:26.0:1616513075.114552:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:26.0:1616513075.114558:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:26.0:1616513075.114571:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513075.114578:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:26.0:1616513075.114584:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513075.114587:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513075.114590:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513075.114595:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513075.114642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513075.114655:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:30.0:1616513075.114660:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:30.0:1616513075.114662:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513075.114666:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:30.0:1616513075.114668:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:30.0:1616513075.114673:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a7f500 x1693067725924736/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513075.114684:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aebb0 00000800:00000200:30.0:1616513075.114688:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513075.114759:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513075.114815:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513075.114824:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:2.0:1616513075.114833:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513075.114841:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626980 00000400:00000200:2.0:1616513075.114848:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c66c9 [1] + 192 00000400:00000200:2.0:1616513075.114853:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.114858:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a7f500 x1693067725924736/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513075.114869:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513075.114874:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513075.114877:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513075.114879:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:26.0:1616513075.114880:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a7f500 x1693067725924736/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513075.114893:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a7f500 x1693067725924736/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513075.114903:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:7.2:1616513075.122968:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513075.123025:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.123034:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513075.123036:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513075.123043:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.34@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.123053:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.34@tcp1 of length 224 into portal 26 MB=0x60183f9c42e80 00000400:00000200:22.0:1616513075.123059:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.34@tcp1 of length 224/224 into md 0xf97c5fa9 [1] + 0 00000400:00000200:22.0:1616513075.123065:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774ff0 00000400:00000200:22.0:1616513075.123067:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.123070:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513075.123075:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774ff0 00000800:00000200:22.0:1616513075.123078:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.123081:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.123083:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.123140:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516202991232 00010000:00000200:15.0:1616513075.123151:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1690516202991232/t0(0) o400->76297a0e-5388-0201-8871-d55a8b705b46@10.31.132.34@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.123165:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690516202991232, offset 192 00000400:00000200:15.0:1616513075.123173:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.34@tcp1 00000400:00000200:15.0:1616513075.123181:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.34@tcp1 routed destination 00000400:00000200:15.0:1616513075.123196:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.34@tcp1(10.31.132.34@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.123203:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.123210:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.123212:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.123215:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.123220:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.123224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.123227:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513075.123277:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.123285:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.123288:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513075.123295:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.123301:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c7f8 00000800:00000200:14.0:1616513075.123305:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.123308:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.123314:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513075.123315:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513075.123322:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.34@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.123330:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.34@tcp1 of length 224 into portal 12 MB=0x60183f9c42fc0 00000400:00000200:14.0:1616513075.123337:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.34@tcp1 of length 224/224 into md 0xf96450b9 [1] + 47728 00000400:00000200:14.0:1616513075.123342:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.123344:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.123351:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.123354:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.123356:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.123375:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516202991552 00010000:00000200:18.0:1616513075.123388:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e1680 x1690516202991552/t0(0) o400->5c0f4ddd-32ab-4449-d129-5455590cf8c9@10.31.132.34@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.123403:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690516202991552, offset 192 00000400:00000200:18.0:1616513075.123409:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.34@tcp1 00000400:00000200:18.0:1616513075.123417:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.34@tcp1 routed destination 00000400:00000200:18.0:1616513075.123445:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.34@tcp1(10.31.132.34@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.123452:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.123458:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.123461:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.123463:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.123468:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.123474:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.123529:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.123536:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.123539:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513075.123546:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.123550:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cded0 00000800:00000200:16.0:1616513075.123554:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.123557:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.124231:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513075.124242:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513075.124249:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513075.124256:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.200@tcp1 : PUT - for me 00000400:00000200:8.0:1616513075.124265:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.200@tcp1 of length 224 into portal 26 MB=0x5ffd4a9e8c240 00000400:00000200:8.0:1616513075.124271:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.200@tcp1 of length 224/224 into md 0xf97c5fe9 [1] + 0 00000400:00000200:8.0:1616513075.124275:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774770 00000400:00000200:8.0:1616513075.124277:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513075.124279:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513075.124283:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774770 00000800:00000200:8.0:1616513075.124286:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.124288:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513075.124289:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.124291:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.124296:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.124296:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:15.0:1616513075.124299:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663732306496 00000400:00000200:8.0:1616513075.124301:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.200@tcp1 : PUT - for me 00010000:00000200:15.0:1616513075.124305:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688663732306496/t0(0) o400->bda26bab-021f-b72d-95e3-e212a1458d4b@10.31.136.200@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513075.124306:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.200@tcp1 of length 224 into portal 12 MB=0x5ffd4a9e8c2c0 00000400:00000200:8.0:1616513075.124311:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.200@tcp1 of length 224/224 into md 0x865b9f31 [1] + 82392 00000400:00000200:8.0:1616513075.124314:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:15.0:1616513075.124317:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663732306496, offset 192 00000100:00000200:8.0:1616513075.124317:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513075.124321:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513075.124322:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.200@tcp1 00000800:00000200:8.0:1616513075.124323:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.124324:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:15.0:1616513075.124329:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.200@tcp1 routed destination 00000100:00000200:24.0:1616513075.124343:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663732306624 00000400:00000200:15.0:1616513075.124343:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.200@tcp1(10.31.136.200@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.124349:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:24.0:1616513075.124354:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5580 x1688663732306624/t0(0) o400->3532d29d-a24d-c7f1-c0c5-76391112a138@10.31.136.200@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.124354:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.124357:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.124359:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.124363:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513075.124367:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663732306624, offset 192 00000800:00000200:7.2:1616513075.124367:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.124368:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513075.124372:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.200@tcp1 00000400:00000200:24.0:1616513075.124380:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.200@tcp1 routed destination 00000400:00000200:24.0:1616513075.124393:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.200@tcp1(10.31.136.200@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.124399:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.124405:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.124407:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.124409:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.124413:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.124418:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513075.124419:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.124426:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513075.124428:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513075.124430:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513075.124430:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.124442:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513075.124450:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513075.124451:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.124454:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.124456:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2dc28 00000800:00000200:22.0:1616513075.124460:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513075.124461:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.124465:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf34b8 00000800:00000200:28.0:1616513075.124469:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.144813:0:48922:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.144871:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.144881:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.144890:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.54@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.144900:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.54@tcp1 of length 224 into portal 26 MB=0x603e28b635400 00000400:00000200:2.0:1616513075.144910:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.54@tcp1 of length 224/224 into md 0xf97c5ff9 [1] + 0 00000400:00000200:2.0:1616513075.144915:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774908 00000400:00000200:2.0:1616513075.144918:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.144921:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513075.144926:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513075.144926:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774908 00000800:00000200:2.0:1616513075.144930:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513075.144932:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513075.144934:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.144937:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.144938:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.144947:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.54@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.144956:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.54@tcp1 of length 224 into portal 12 MB=0x603e28b635480 00000400:00000200:14.0:1616513075.144963:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.54@tcp1 of length 224/224 into md 0xf96450b9 [1] + 47952 00000400:00000200:14.0:1616513075.144968:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.144971:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.144978:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.144981:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.144983:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.144994:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693121396298752 00010000:00000200:15.0:1616513075.145006:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1693121396298752/t0(0) o400->40694ddb-5ec8-48cd-a27f-7e71b2b23a0b@10.31.136.54@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.145021:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693121396298752, offset 192 00000400:00000200:15.0:1616513075.145027:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.54@tcp1 00000400:00000200:15.0:1616513075.145035:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.54@tcp1 routed destination 00000100:00000200:18.0:1616513075.145046:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693121396298880 00000400:00000200:15.0:1616513075.145051:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.54@tcp1(10.31.136.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.145058:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:18.0:1616513075.145059:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e1b00 x1693121396298880/t0(0) o400->a5a9096e-21a7-0a46-320a-ae1c09f3b743@10.31.136.54@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.145067:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.145070:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.145072:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:18.0:1616513075.145074:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693121396298880, offset 192 00000800:00000200:15.0:1616513075.145077:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513075.145081:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.54@tcp1 00000800:00000200:7.2:1616513075.145081:0:48922:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.145085:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513075.145089:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.54@tcp1 routed destination 00000400:00000200:18.0:1616513075.145105:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.54@tcp1(10.31.136.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.145113:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.145120:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.145122:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.145124:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.145129:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.145136:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513075.145138:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.145146:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513075.145148:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.145150:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.145156:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.145162:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cff0 00000800:00000200:22.0:1616513075.145166:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513075.145191:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.145198:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.145201:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513075.145201:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.145208:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.145212:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc088 00000800:00000200:16.0:1616513075.145216:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000080:5.0:1616513075.151089:0:48922:0:(module.c:90:libcfs_ioctl()) libcfs ioctl cmd 3233310049 00000400:00000080:5.0:1616513075.151134:0:48922:0:(module.c:90:libcfs_ioctl()) libcfs ioctl cmd 3233310049 00000400:00000080:5.0:1616513075.151146:0:48922:0:(module.c:90:libcfs_ioctl()) libcfs ioctl cmd 3233310049 00000800:00000200:11.2:1616513075.159554:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.159610:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.159620:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.159631:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.67@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.159641:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.67@tcp1 of length 224 into portal 26 MB=0x5ffd3374a2980 00000400:00000200:12.0:1616513075.159647:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.67@tcp1 of length 224/224 into md 0xf97c6009 [1] + 0 00000400:00000200:12.0:1616513075.159653:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774330 00000400:00000200:12.0:1616513075.159656:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.159658:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513075.159663:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774330 00000800:00000200:28.0:1616513075.159664:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.159666:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.159669:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.159671:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.159726:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657514342784 00010000:00000200:15.0:1616513075.159737:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688657514342784/t0(0) o400->d5f908db-7c6a-2b60-9cc9-108938f29706@10.31.133.67@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.159750:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657514342784, offset 192 00000400:00000200:15.0:1616513075.159756:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.67@tcp1 00000400:00000200:15.0:1616513075.159763:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.67@tcp1 routed destination 00000400:00000200:15.0:1616513075.159777:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.67@tcp1(10.31.133.67@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.159784:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.159789:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.159792:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.159794:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.159799:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513075.159805:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.159838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.159901:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.159909:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.159912:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.159919:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.159926:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d320 00000800:00000200:2.0:1616513075.159930:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.159933:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.160305:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513075.160357:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513075.160365:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513075.160373:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.67@tcp1 : PUT - for me 00000400:00000200:8.0:1616513075.160380:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.67@tcp1 of length 224 into portal 12 MB=0x5ffd3374a2a00 00000400:00000200:8.0:1616513075.160386:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.67@tcp1 of length 224/224 into md 0x865b9f31 [1] + 82616 00000400:00000200:8.0:1616513075.160390:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513075.160392:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513075.160398:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.160400:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513075.160402:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.160404:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.160414:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657514342912 00010000:00000200:24.0:1616513075.160423:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7500 x1688657514342912/t0(0) o400->80c97c66-6215-c23b-1157-90d6d3086aba@10.31.133.67@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.160434:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657514342912, offset 192 00000400:00000200:24.0:1616513075.160444:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.67@tcp1 00000400:00000200:24.0:1616513075.160450:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.67@tcp1 routed destination 00000400:00000200:24.0:1616513075.160461:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.67@tcp1(10.31.133.67@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.160467:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.160471:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.160474:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.160475:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.160479:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.160484:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.160495:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.160501:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.160503:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.160505:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.160510:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.160514:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf26e8 00000800:00000200:28.0:1616513075.160518:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:11.0:1616513075.162538:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924752, portal 4 00000100:00000200:11.0:1616513075.162542:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924752, offset 0 00000400:00000200:11.0:1616513075.162547:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:11.0:1616513075.162552:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:11.0:1616513075.162561:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513075.162566:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:11.0:1616513075.162571:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513075.162573:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:11.0:1616513075.162575:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:11.0:1616513075.162579:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513075.162586:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:27.0:1616513075.162623:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924768, portal 4 00000100:00000200:27.0:1616513075.162628:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924768, offset 0 00000400:00000200:27.0:1616513075.162635:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:8.0:1616513075.162638:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:27.0:1616513075.162640:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:8.0:1616513075.162644:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513075.162646:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513075.162652:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:27.0:1616513075.162654:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000100:00000200:8.0:1616513075.162659:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd604380 x1693067725924752/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513075.162661:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513075.162667:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:8.0:1616513075.162668:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bed0 00000800:00000200:27.0:1616513075.162670:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513075.162672:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:8.0:1616513075.162672:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513075.162675:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.0:1616513075.162678:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513075.162685:0:6509:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513075.162698:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513075.162705:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:31.0:1616513075.162708:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513075.162711:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:31.0:1616513075.162715:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:31.0:1616513075.162721:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc564c80 x1693067725924768/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513075.162733:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249430 00000800:00000200:31.0:1616513075.162737:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:5.2:1616513075.162804:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:27.2:1616513075.162855:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:12.0:1616513075.162857:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513075.162865:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:12.0:1616513075.162867:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:27.0:1616513075.162871:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513075.162873:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:12.0:1616513075.162876:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513075.162879:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513075.162885:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626990 00000400:00000200:27.0:1616513075.162887:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6269a0 00000400:00000200:27.0:1616513075.162893:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37165cbd [1] + 192 00000400:00000200:12.0:1616513075.162893:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d40e1 [1] + 192 00000400:00000200:12.0:1616513075.162898:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:27.0:1616513075.162899:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513075.162904:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc564c80 x1693067725924768/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:12.0:1616513075.162905:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd604380 x1693067725924752/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513075.162910:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:27.0:1616513075.162919:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:12.0:1616513075.162920:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513075.162922:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513075.162924:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:12.0:1616513075.162924:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513075.162925:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:31.0:1616513075.162926:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc564c80 x1693067725924768/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513075.162927:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd604380 x1693067725924752/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513075.162935:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd604380 x1693067725924752/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513075.162938:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc564c80 x1693067725924768/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513075.162943:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bc28 00000400:00000200:31.0:1616513075.162948:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248e58 00000800:00000200:25.2:1616513075.166762:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74dd8ea400] (20)++ 00000800:00000200:6.0:1616513075.166780:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74dd8ea400] (21)++ 00000800:00000200:6.0:1616513075.166789:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.148@o2ib1 00000400:00000200:6.0:1616513075.166798:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.148@o2ib1 : PUT - for me 00000400:00000200:6.0:1616513075.166806:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.148@o2ib1 of length 224 into portal 26 MB=0x604a72eb8c300 00000400:00000200:6.0:1616513075.166813:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.148@o2ib1 of length 224/224 into md 0xf97c6019 [1] + 0 00000400:00000200:6.0:1616513075.166818:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775ba0 00000400:00000200:6.0:1616513075.166820:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513075.166823:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:6.0:1616513075.166828:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775ba0 00000800:00000200:6.0:1616513075.166831:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74dd8ea400] (22)++ 00000800:00000200:22.0:1616513075.166834:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74dd8ea400] (23)++ 00000800:00000200:6.0:1616513075.166834:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74dd8ea400] (23)-- 00000800:00000200:6.0:1616513075.166841:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74dd8ea400] (23)-- 00000800:00000200:6.0:1616513075.166844:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74dd8ea400] (22)-- 00000800:00000200:22.0:1616513075.166845:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.148@o2ib1 00000400:00000200:22.0:1616513075.166853:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.148@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513075.166861:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.148@o2ib1 of length 224 into portal 12 MB=0x604a72eb8c380 00000400:00000200:22.0:1616513075.166867:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.148@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 48176 00000400:00000200:22.0:1616513075.166875:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.166877:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513075.166885:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74dd8ea400] (21)++ 00000800:00000200:22.0:1616513075.166888:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74dd8ea400] (22)-- 00000800:00000200:22.0:1616513075.166890:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74dd8ea400] (21)-- 00000100:00000200:15.0:1616513075.166893:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693965950173952 00010000:00000200:15.0:1616513075.166904:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1693965950173952/t0(0) o400->b4b6f891-995d-0a33-5676-0ef963e9112a@10.242.46.148@o2ib1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513075.166906:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693965950174080 00000100:00000200:15.0:1616513075.166918:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693965950173952, offset 192 00010000:00000200:18.0:1616513075.166919:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad2d00 x1693965950174080/t0(0) o400->739b9bf3-f5d0-cf58-89eb-790a3388c1c4@10.242.46.148@o2ib1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513075.166924:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.148@o2ib1 00000400:00000200:15.0:1616513075.166932:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.148@o2ib1 local destination 00000100:00000200:18.0:1616513075.166933:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693965950174080, offset 192 00000400:00000200:18.0:1616513075.166940:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.148@o2ib1 00000400:00000200:15.0:1616513075.166946:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.148@o2ib1(10.242.46.148@o2ib1:10.242.46.148@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.166953:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.148@o2ib1 00000400:00000200:18.0:1616513075.166956:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.148@o2ib1 local destination 00000800:00000200:15.0:1616513075.166959:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5dc1ac8780] -> 10.242.46.148@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.166962:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74dd8ea400] (20)++ 00000800:00000200:15.0:1616513075.166964:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74dd8ea400] (21)++ 00000800:00000200:15.0:1616513075.166969:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74dd8ea400] (22)-- 00000400:00000200:18.0:1616513075.166971:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.148@o2ib1(10.242.46.148@o2ib1:10.242.46.148@o2ib1) : PUT try# 0 00000800:00000200:25.2:1616513075.166975:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74dd8ea400] (21)++ 00000800:00000200:18.0:1616513075.166977:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.148@o2ib1 00000100:00000200:15.0:1616513075.166978:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513075.166984:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5dc1ac8780] -> 10.242.46.148@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.166987:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74dd8ea400] (22)++ 00000800:00000200:18.0:1616513075.166989:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74dd8ea400] (23)++ 00000800:00000200:18.0:1616513075.166994:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74dd8ea400] (24)-- 00000800:00000200:14.0:1616513075.167028:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74dd8ea400] (23)++ 00000800:00000200:14.0:1616513075.167036:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74dd8ea400] (24)-- 00000400:00000200:14.0:1616513075.167039:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:6.0:1616513075.167039:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74dd8ea400] (23)++ 00000800:00000200:6.0:1616513075.167043:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74dd8ea400] (24)-- 00000400:00000200:6.0:1616513075.167045:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513075.167046:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.148@o2ib1: PUT: OK 00000800:00000200:2.0:1616513075.167048:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74dd8ea400] (23)-- 00000400:00000200:6.0:1616513075.167051:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.148@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.167052:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c770 00000800:00000200:14.0:1616513075.167055:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74dd8ea400] (22)-- 00000400:00000200:6.0:1616513075.167055:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd078 00000800:00000200:6.0:1616513075.167058:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74dd8ea400] (21)-- 00000800:00000200:11.2:1616513075.169162:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513075.169176:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513075.169185:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513075.169194:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.68@tcp1 : PUT - for me 00000400:00000200:8.0:1616513075.169203:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.68@tcp1 of length 224 into portal 26 MB=0x601960696c940 00000400:00000200:8.0:1616513075.169210:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.68@tcp1 of length 224/224 into md 0xf97c6049 [1] + 0 00000400:00000200:8.0:1616513075.169214:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774d48 00000400:00000200:8.0:1616513075.169217:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513075.169220:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513075.169237:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774d48 00000800:00000200:16.0:1616513075.169238:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.169239:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.169241:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513075.169243:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.169300:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690593727531328 00010000:00000200:15.0:1616513075.169312:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1690593727531328/t0(0) o400->68e47114-e461-923d-e174-da77ae480de0@10.31.142.68@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.169326:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690593727531328, offset 192 00000400:00000200:15.0:1616513075.169333:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.68@tcp1 00000400:00000200:15.0:1616513075.169341:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.68@tcp1 routed destination 00000400:00000200:15.0:1616513075.169356:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.68@tcp1(10.31.142.68@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.169363:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.169369:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.169372:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.169374:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.169379:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.169385:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.169386:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513075.169451:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.169458:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.169461:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.169468:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.169474:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2dcb0 00000800:00000200:22.0:1616513075.169478:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.169481:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.170461:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.170513:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.170524:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.170533:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.68@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.170542:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.68@tcp1 of length 224 into portal 12 MB=0x601960696d840 00000400:00000200:14.0:1616513075.170548:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.68@tcp1 of length 224/224 into md 0xf96450b9 [1] + 48400 00000400:00000200:14.0:1616513075.170554:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.170556:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.170563:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.170566:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.170566:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.170570:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.170626:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690593727535168 00010000:00000200:18.0:1616513075.170638:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e1f80 x1690593727535168/t0(0) o400->015e398a-085f-9efd-6d09-7036431f8ca1@10.31.142.68@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.170651:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690593727535168, offset 192 00000400:00000200:18.0:1616513075.170657:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.68@tcp1 00000400:00000200:18.0:1616513075.170664:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.68@tcp1 routed destination 00000400:00000200:18.0:1616513075.170679:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.68@tcp1(10.31.142.68@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.170685:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.170690:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.170693:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.170695:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.170699:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.170706:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.170760:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.170767:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.170770:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.170777:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.170780:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd188 00000800:00000200:28.0:1616513075.170784:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.170787:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513075.178595:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924784, portal 4 00000100:00000200:10.0:1616513075.178600:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924784, offset 0 00000400:00000200:10.0:1616513075.178606:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:10.0:1616513075.178611:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:10.0:1616513075.178622:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513075.178628:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:10.0:1616513075.178633:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513075.178636:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513075.178638:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513075.178642:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513075.178701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513075.178758:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513075.178765:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513075.178768:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.178771:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513075.178775:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513075.178781:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d746120c800 x1693067725924784/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513075.178793:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83a90 00000800:00000200:22.0:1616513075.178797:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513075.178852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513075.178863:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513075.178869:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:14.0:1616513075.178873:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:10.0:1616513075.178876:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:10.0:1616513075.178881:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6269b0 00000400:00000200:10.0:1616513075.178886:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c6789 [1] + 192 00000400:00000200:10.0:1616513075.178890:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.178897:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d746120c800 x1693067725924784/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:10.0:1616513075.178910:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513075.178913:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513075.178914:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:14.0:1616513075.178920:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d746120c800 x1693067725924784/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513075.178933:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d746120c800 x1693067725924784/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513075.178942:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83b18 00000800:00000200:31.2:1616513075.190403:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:2.0:1616513075.190463:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:2.0:1616513075.190472:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.234@o2ib1 00000400:00000200:2.0:1616513075.190481:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513075.190490:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 224 into portal 26 MB=0x603db0154bdb0 00000400:00000200:2.0:1616513075.190497:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.234@o2ib1 of length 224/224 into md 0xf97c6059 [1] + 0 00000400:00000200:2.0:1616513075.190502:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775188 00000400:00000200:2.0:1616513075.190505:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.190507:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513075.190512:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775188 00000800:00000200:2.0:1616513075.190516:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:22.0:1616513075.190517:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:2.0:1616513075.190519:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:2.0:1616513075.190520:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:15.0:1616513075.190535:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693089015315888 00010000:00000200:15.0:1616513075.190547:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1693089015315888/t0(0) o400->2a20e44c-6e9f-2a77-f91e-22f6e67c3d0e@10.242.46.234@o2ib1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.190561:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693089015315888, offset 192 00000400:00000200:15.0:1616513075.190567:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:15.0:1616513075.190575:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:15.0:1616513075.190589:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.190595:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:15.0:1616513075.190601:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.190604:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:15.0:1616513075.190606:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:15.0:1616513075.190611:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513075.190616:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:15.0:1616513075.190618:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513075.190669:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:14.0:1616513075.190675:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:14.0:1616513075.190678:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513075.190685:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.190691:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c440 00000800:00000200:14.0:1616513075.190694:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:14.0:1616513075.190697:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:7.2:1616513075.199625:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513075.199682:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.199691:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.199700:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.250@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.199709:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.250@tcp1 of length 224 into portal 26 MB=0x60194e7026d40 00000400:00000200:22.0:1616513075.199716:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.250@tcp1 of length 224/224 into md 0xf97c6079 [1] + 0 00000400:00000200:22.0:1616513075.199721:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775a08 00000400:00000200:22.0:1616513075.199723:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.199726:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513075.199731:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775a08 00000800:00000200:22.0:1616513075.199734:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.199735:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.199737:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.199739:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.199795:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588902747456 00000800:00000200:7.2:1616513075.199801:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:15.0:1616513075.199806:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1690588902747456/t0(0) o400->18e3a8fc-4829-74e3-72e0-d9fbad75d4f2@10.31.140.250@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.199820:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588902747456, offset 192 00000400:00000200:15.0:1616513075.199828:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.250@tcp1 00000400:00000200:15.0:1616513075.199836:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.250@tcp1 routed destination 00000400:00000200:15.0:1616513075.199851:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.250@tcp1(10.31.140.250@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.199858:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513075.199862:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.199864:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.199867:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513075.199869:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513075.199871:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.199875:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513075.199880:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.250@tcp1 : PUT - for me 00000100:00000200:15.0:1616513075.199882:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513075.199889:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.250@tcp1 of length 224 into portal 12 MB=0x60194e70272c0 00000400:00000200:14.0:1616513075.199897:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.250@tcp1 of length 224/224 into md 0xf96450b9 [1] + 48624 00000400:00000200:14.0:1616513075.199903:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.199906:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.199913:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:10.0:1616513075.199915:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513075.199916:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513075.199923:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:10.0:1616513075.199924:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513075.199926:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.199927:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.199934:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.199940:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2caa0 00000800:00000200:10.0:1616513075.199943:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.199977:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588902748864 00010000:00000200:18.0:1616513075.199989:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e2880 x1690588902748864/t0(0) o400->22c948b0-9a4e-4e34-d599-a9d1d9dfaa31@10.31.140.250@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.200003:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588902748864, offset 192 00000400:00000200:18.0:1616513075.200009:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.250@tcp1 00000400:00000200:18.0:1616513075.200016:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.250@tcp1 routed destination 00000400:00000200:18.0:1616513075.200030:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.250@tcp1(10.31.140.250@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.200036:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.200042:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.200045:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.200046:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.200051:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.200059:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.200113:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.200123:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.200125:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513075.200126:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.200132:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.200136:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccee0 00000800:00000200:16.0:1616513075.200140:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513075.210594:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924800, portal 4 00000100:00000200:14.0:1616513075.210600:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924800, offset 0 00000400:00000200:14.0:1616513075.210607:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:14.0:1616513075.210613:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:14.0:1616513075.210626:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513075.210633:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:14.0:1616513075.210639:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513075.210642:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:14.0:1616513075.210645:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:14.0:1616513075.210650:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513075.210702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.210722:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513075.210728:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513075.210731:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513075.210739:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513075.210746:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d746120a400 x1693067725924800/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513075.210759:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757210 00000800:00000200:1.0:1616513075.210764:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513075.210766:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513075.210893:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513075.210903:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.210910:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513075.210913:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513075.210918:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513075.210926:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6269c0 00000400:00000200:1.0:1616513075.210933:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffa05 [1] + 192 00000400:00000200:1.0:1616513075.210938:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513075.210943:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d746120a400 x1693067725924800/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513075.210968:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.210971:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513075.210973:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:14.0:1616513075.210979:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d746120a400 x1693067725924800/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513075.211004:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d746120a400 x1693067725924800/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513075.211013:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757e48 00000800:00000200:11.2:1616513075.213488:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.213552:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.213562:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.213571:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.206@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.213582:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.206@tcp1 of length 224 into portal 12 MB=0x5ffdee19efac0 00000400:00000200:28.0:1616513075.213588:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.206@tcp1 of length 224/224 into md 0x865b9f31 [1] + 82840 00000400:00000200:28.0:1616513075.213594:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.213596:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.213603:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.213606:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.213608:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.213610:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.213616:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513075.213619:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.213622:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.206@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.213628:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.206@tcp1 of length 224 into portal 26 MB=0x5ffdee19ef740 00000400:00000200:28.0:1616513075.213634:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.206@tcp1 of length 224/224 into md 0xf97c6089 [1] + 0 00000400:00000200:28.0:1616513075.213638:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775298 00000400:00000200:28.0:1616513075.213640:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.213642:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.213646:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775298 00000800:00000200:28.0:1616513075.213649:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.213651:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.213652:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.213670:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707616667456 00000100:00000200:24.0:1616513075.213671:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707616668352 00010000:00000200:15.0:1616513075.213681:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688707616667456/t0(0) o400->c7e314cc-ebdd-c5c1-56d7-778e5b48b817@10.31.128.206@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:24.0:1616513075.213685:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf84c80 x1688707616668352/t0(0) o400->bda0e936-71df-9666-8f68-db837259800e@10.31.128.206@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513075.213696:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688707616667456, offset 192 00000100:00000200:24.0:1616513075.213699:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688707616668352, offset 192 00000400:00000200:15.0:1616513075.213702:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.206@tcp1 00000400:00000200:24.0:1616513075.213706:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.206@tcp1 00000400:00000200:15.0:1616513075.213711:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.206@tcp1 routed destination 00000400:00000200:24.0:1616513075.213714:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.206@tcp1 routed destination 00000400:00000200:15.0:1616513075.213728:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.206@tcp1(10.31.128.206@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513075.213732:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.206@tcp1(10.31.128.206@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.213735:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513075.213739:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.213742:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.213745:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513075.213746:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.213747:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.213748:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.213751:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.213752:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513075.213755:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513075.213759:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513075.213760:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513075.213760:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513075.213814:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.213821:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.213824:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513075.213824:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.213831:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.213835:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3f58 00000800:00000200:22.0:1616513075.213836:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513075.213839:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513075.213843:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.213846:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.213853:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.213859:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d3a8 00000800:00000200:22.0:1616513075.213863:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.213866:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.226965:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.227022:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.227031:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.227040:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.132@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.227051:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.132@tcp1 of length 224 into portal 12 MB=0x5ffd406021140 00000400:00000200:2.0:1616513075.227057:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.132@tcp1 of length 224/224 into md 0xf96450b9 [1] + 48848 00000400:00000200:2.0:1616513075.227062:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.227065:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.227072:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.227074:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.227075:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.227079:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.227138:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660982501696 00010000:00000200:18.0:1616513075.227150:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461209680 x1688660982501696/t0(0) o400->f78e13b0-1666-45eb-7f0b-dde0167ef487@10.31.132.132@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.227165:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660982501696, offset 192 00000400:00000200:18.0:1616513075.227171:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.132@tcp1 00000400:00000200:18.0:1616513075.227178:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.132@tcp1 routed destination 00000400:00000200:18.0:1616513075.227193:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.132@tcp1(10.31.132.132@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.227200:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.227206:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.227209:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.227211:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.227216:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.227221:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.227276:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.227283:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.227285:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.227292:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.227296:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc660 00000800:00000200:22.0:1616513075.227300:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.227303:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.228133:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.228189:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.228199:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.228208:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.132@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.228217:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.132@tcp1 of length 224 into portal 26 MB=0x5ffd406021040 00000400:00000200:12.0:1616513075.228224:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.132@tcp1 of length 224/224 into md 0xf97c6099 [1] + 0 00000400:00000200:12.0:1616513075.228229:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7774bb0 00000400:00000200:12.0:1616513075.228232:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.228234:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513075.228239:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7774bb0 00000800:00000200:28.0:1616513075.228243:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.228250:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.228253:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.228255:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.228303:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660982501440 00010000:00000200:15.0:1616513075.228314:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688660982501440/t0(0) o400->d1eefac4-5aba-3f2e-a314-075d07f3f379@10.31.132.132@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.228329:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660982501440, offset 192 00000400:00000200:15.0:1616513075.228336:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.132@tcp1 00000400:00000200:15.0:1616513075.228344:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.132@tcp1 routed destination 00000400:00000200:15.0:1616513075.228360:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.132@tcp1(10.31.132.132@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.228371:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.228377:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.228380:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.228382:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.228387:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.228392:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.228394:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513075.228403:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.228409:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513075.228412:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513075.228431:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513075.228442:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d3a8 00000800:00000200:8.0:1616513075.228446:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.228448:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.238711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.238766:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.238775:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.238784:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.128.30@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.238793:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.128.30@tcp1 of length 224 into portal 26 MB=0x5f9ca215209c0 00000400:00000200:28.0:1616513075.238800:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.128.30@tcp1 of length 224/224 into md 0xf97c60b9 [1] + 0 00000400:00000200:28.0:1616513075.238805:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77746e8 00000400:00000200:28.0:1616513075.238808:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.238811:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.238816:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77746e8 00000800:00000200:28.0:1616513075.238819:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.238819:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.238822:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.238826:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.238829:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513075.238831:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.238838:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.128.30@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.238846:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.128.30@tcp1 of length 224 into portal 12 MB=0x5f9ca21520a80 00000400:00000200:12.0:1616513075.238853:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.128.30@tcp1 of length 224/224 into md 0x865b9f31 [1] + 83064 00000400:00000200:12.0:1616513075.238858:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.238860:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.238867:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.238870:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.238872:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.238878:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682021421287872 00010000:00000200:15.0:1616513075.238888:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1682021421287872/t0(0) o400->43639d51-cbc9-4f19-8425-c03c140bd0bf@10.242.128.30@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.238902:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1682021421287872, offset 192 00000400:00000200:15.0:1616513075.238908:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.128.30@tcp1 00000400:00000200:15.0:1616513075.238915:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.128.30@tcp1 routed destination 00000400:00000200:15.0:1616513075.238929:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.128.30@tcp1(10.242.128.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.238935:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.238940:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:24.0:1616513075.238941:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682021421288064 00000800:00000200:15.0:1616513075.238943:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.238945:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.238950:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:24.0:1616513075.238954:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57f500 x1682021421288064/t0(0) o400->37ac4789-30e8-b5e1-08b5-947127b62457@10.242.128.30@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513075.238955:0:48932:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.238956:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513075.238968:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1682021421288064, offset 192 00000400:00000200:24.0:1616513075.238974:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.128.30@tcp1 00000400:00000200:24.0:1616513075.238982:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.128.30@tcp1 routed destination 00000400:00000200:24.0:1616513075.238998:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.128.30@tcp1(10.242.128.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.239004:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513075.239009:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513075.239010:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.239013:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.239016:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513075.239016:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513075.239020:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513075.239020:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513075.239025:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513075.239026:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.239032:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c440 00000800:00000200:10.0:1616513075.239036:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.239039:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513075.239083:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.239090:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.239093:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513075.239093:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.239099:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.239103:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3cb0 00000800:00000200:16.0:1616513075.239107:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.247109:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513075.247142:0:48933:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513075.247163:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.247173:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.247182:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.47@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.247191:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.47@tcp1 of length 224 into portal 26 MB=0x5ffd35f0e9b80 00000400:00000200:28.0:1616513075.247198:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.47@tcp1 of length 224/224 into md 0xf97c60c9 [1] + 0 00000800:00000200:22.0:1616513075.247198:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513075.247203:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd77747f8 00000400:00000200:28.0:1616513075.247206:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513075.247208:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000100:00000200:28.0:1616513075.247209:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.247213:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd77747f8 00000800:00000200:28.0:1616513075.247216:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513075.247217:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.47@tcp1 : PUT - for me 00000800:00000200:12.0:1616513075.247218:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.247219:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.247222:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513075.247226:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.47@tcp1 of length 224 into portal 12 MB=0x5ffd35f0e9d40 00000400:00000200:22.0:1616513075.247233:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.47@tcp1 of length 224/224 into md 0xf96450b9 [1] + 49072 00000400:00000200:22.0:1616513075.247238:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.247241:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513075.247247:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.247251:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.247253:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.247257:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.247275:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658181528448 00010000:00000200:15.0:1616513075.247285:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688658181528448/t0(0) o400->f1fceed5-f078-279b-d2ec-7b4c1d6a90e4@10.31.138.47@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.247298:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658181528448, offset 192 00000400:00000200:15.0:1616513075.247304:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.47@tcp1 00000400:00000200:15.0:1616513075.247312:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.47@tcp1 routed destination 00000100:00000200:18.0:1616513075.247314:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658181528896 00010000:00000200:18.0:1616513075.247327:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad4800 x1688658181528896/t0(0) o400->ddc91805-d35a-0594-9dfb-3a32e12a3748@10.31.138.47@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513075.247327:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.47@tcp1(10.31.138.47@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.247333:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.247339:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:18.0:1616513075.247342:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658181528896, offset 192 00000800:00000200:15.0:1616513075.247342:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.247345:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513075.247348:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.47@tcp1 00000800:00000200:15.0:1616513075.247350:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.247354:0:48933:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513075.247356:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.47@tcp1 routed destination 00000100:00000200:15.0:1616513075.247357:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513075.247374:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.47@tcp1(10.31.138.47@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.247381:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.247387:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.247389:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.247392:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.247397:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.247402:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513075.247408:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513075.247412:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513075.247416:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:8.0:1616513075.247416:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513075.247418:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.247419:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.247421:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513075.247424:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.247426:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:8.0:1616513075.247429:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd540 00000400:00000200:10.0:1616513075.247432:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c770 00000800:00000200:8.0:1616513075.247432:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513075.247435:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513075.247448:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.255379:0:48934:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.255456:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.255466:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.255475:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.148@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.255483:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.148@tcp1 of length 224 into portal 26 MB=0x5ffe835e524c0 00000400:00000200:14.0:1616513075.255490:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.148@tcp1 of length 224/224 into md 0xf97c60e9 [1] + 0 00000400:00000200:14.0:1616513075.255495:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775870 00000400:00000200:14.0:1616513075.255498:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.255501:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513075.255508:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775870 00000800:00000200:22.0:1616513075.255510:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.255511:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.255514:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513075.255517:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.255520:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.255521:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.255529:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.148@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.255538:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.148@tcp1 of length 224 into portal 12 MB=0x5ffe835e527c0 00000400:00000200:22.0:1616513075.255544:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.148@tcp1 of length 224/224 into md 0xf96450b9 [1] + 49296 00000400:00000200:22.0:1616513075.255550:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.255552:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513075.255560:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.255563:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.255567:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.255570:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688747685258432 00010000:00000200:15.0:1616513075.255581:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688747685258432/t0(0) o400->3995ae7c-8c01-a85c-8f46-5deecbd22ab5@10.31.141.148@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.255594:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688747685258432, offset 192 00000400:00000200:15.0:1616513075.255600:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.148@tcp1 00000400:00000200:15.0:1616513075.255607:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.148@tcp1 routed destination 00000400:00000200:15.0:1616513075.255621:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.148@tcp1(10.31.141.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.255628:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.255634:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.255637:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.255639:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:18.0:1616513075.255642:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688747685259200 00000800:00000200:15.0:1616513075.255643:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.255647:0:48934:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.255650:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:18.0:1616513075.255654:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5580 x1688747685259200/t0(0) o400->aa6f99dc-d3ea-9f75-27d4-cf0122e30726@10.31.141.148@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.255668:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688747685259200, offset 192 00000400:00000200:18.0:1616513075.255674:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.148@tcp1 00000400:00000200:18.0:1616513075.255681:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.148@tcp1 routed destination 00000400:00000200:18.0:1616513075.255696:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.148@tcp1(10.31.141.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513075.255700:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513075.255702:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513075.255707:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513075.255708:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.255710:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:10.0:1616513075.255710:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513075.255711:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513075.255712:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.255717:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513075.255717:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.255723:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cff0 00000800:00000200:11.2:1616513075.255724:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513075.255727:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513075.255779:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.255786:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.255789:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513075.255795:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.255799:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd210 00000800:00000200:12.0:1616513075.255803:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.255806:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.257245:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513075.257255:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513075.257261:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513075.257269:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.96@tcp1 : PUT - for me 00000400:00000200:8.0:1616513075.257279:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.96@tcp1 of length 224 into portal 26 MB=0x6001edabd7a00 00000400:00000200:8.0:1616513075.257286:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.96@tcp1 of length 224/224 into md 0xf97c6109 [1] + 0 00000400:00000200:8.0:1616513075.257290:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775b18 00000400:00000200:8.0:1616513075.257293:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513075.257296:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513075.257300:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775b18 00000800:00000200:8.0:1616513075.257303:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513075.257305:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.257307:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513075.257311:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.257362:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688982379133440 00010000:00000200:15.0:1616513075.257372:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688982379133440/t0(0) o400->dab07249-3aa2-72f2-631b-7ce1f3d8f2a6@10.31.140.96@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.257385:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688982379133440, offset 192 00000400:00000200:15.0:1616513075.257391:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.96@tcp1 00000400:00000200:15.0:1616513075.257399:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.96@tcp1 routed destination 00000400:00000200:15.0:1616513075.257413:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.96@tcp1(10.31.140.96@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.257419:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.257425:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.257428:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.257430:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.257448:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.257453:0:48934:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.257455:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513075.257509:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.257516:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.257519:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.257526:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.257532:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c7f8 00000800:00000200:2.0:1616513075.257536:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.257538:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.258115:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.258168:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.258177:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.258185:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.96@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.258194:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.96@tcp1 of length 224 into portal 12 MB=0x6001edabd8640 00000400:00000200:12.0:1616513075.258201:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.96@tcp1 of length 224/224 into md 0x865b9f31 [1] + 83288 00000400:00000200:12.0:1616513075.258206:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.258209:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.258215:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.258218:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.258220:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.258223:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.258279:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688982379136576 00010000:00000200:24.0:1616513075.258292:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57d100 x1688982379136576/t0(0) o400->142df80d-ca61-af95-4ca9-41058db6c1d3@10.31.140.96@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.258306:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688982379136576, offset 192 00000400:00000200:24.0:1616513075.258312:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.96@tcp1 00000400:00000200:24.0:1616513075.258320:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.96@tcp1 routed destination 00000400:00000200:24.0:1616513075.258335:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.96@tcp1(10.31.140.96@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.258342:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.258348:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.258351:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.258353:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.258358:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.258363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.258374:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.258380:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.258383:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.258385:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.258390:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.258394:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3210 00000800:00000200:16.0:1616513075.258398:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513075.258475:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924816, portal 4 00000100:00000200:11.0:1616513075.258476:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924832, portal 4 00000100:00000200:24.0:1616513075.258478:0:6480:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924848, portal 4 00000100:00000200:22.0:1616513075.258481:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924816, offset 0 00000100:00000200:11.0:1616513075.258481:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924832, offset 0 00000100:00000200:24.0:1616513075.258483:0:6480:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924848, offset 0 00000400:00000200:24.0:1616513075.258488:0:6480:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:22.0:1616513075.258488:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:11.0:1616513075.258488:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:24.0:1616513075.258494:0:6480:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:11.0:1616513075.258494:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:22.0:1616513075.258495:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:24.0:1616513075.258509:0:6480:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000400:00000200:11.0:1616513075.258510:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.258515:0:6480:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:11.0:1616513075.258517:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:24.0:1616513075.258521:0:6480:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513075.258522:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.258523:0:6480:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:11.0:1616513075.258524:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.258525:0:6480:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:11.0:1616513075.258527:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:24.0:1616513075.258529:0:6480:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:22.0:1616513075.258529:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:11.0:1616513075.258529:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:11.0:1616513075.258533:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:22.0:1616513075.258535:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513075.258538:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:22.0:1616513075.258540:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:22.0:1616513075.258544:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:5.2:1616513075.258595:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:1.2:1616513075.258596:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:12.0:1616513075.258609:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513075.258614:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:1.0:1616513075.258614:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000400:00000200:12.0:1616513075.258617:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.258618:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:1.0:1616513075.258620:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (24)-- 00000400:00000200:12.0:1616513075.258623:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:1.0:1616513075.258623:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513075.258626:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000100:00000200:12.0:1616513075.258629:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be41680 x1693067725924832/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513075.258630:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:5.0:1616513075.258634:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (24)-- 00000400:00000200:5.0:1616513075.258636:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:1.0:1616513075.258638:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc7e00d80 x1693067725924848/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513075.258641:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bcb0 00000400:00000200:5.0:1616513075.258643:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:12.0:1616513075.258645:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:5.0:1616513075.258650:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a79f80 x1693067725924816/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513075.258650:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2440 00000800:00000200:1.0:1616513075.258655:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513075.258658:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:5.0:1616513075.258662:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a7f8 00000800:00000200:5.0:1616513075.258666:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:5.2:1616513075.258758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:1.2:1616513075.258758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:12.0:1616513075.258767:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:1.0:1616513075.258767:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:12.0:1616513075.258773:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:1.0:1616513075.258774:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.232@o2ib1 00000800:00000200:16.0:1616513075.258776:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.0:1616513075.258776:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:12.0:1616513075.258782:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:5.0:1616513075.258783:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513075.258783:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:9.0:1616513075.258786:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:12.0:1616513075.258789:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6269e0 00000400:00000200:5.0:1616513075.258791:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513075.258792:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6269f0 00000400:00000200:12.0:1616513075.258795:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d4121 [1] + 192 00000400:00000200:5.0:1616513075.258799:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6269d0 00000400:00000200:12.0:1616513075.258800:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:1.0:1616513075.258800:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffa25 [1] + 192 00000100:00000200:12.0:1616513075.258804:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be41680 x1693067725924832/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513075.258805:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:5.0:1616513075.258807:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffa15 [1] + 192 00000400:00000200:5.0:1616513075.258812:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513075.258814:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc7e00d80 x1693067725924848/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513075.258817:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513075.258820:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513075.258821:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:11.0:1616513075.258825:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be41680 x1693067725924832/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513075.258829:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000100:00000200:5.0:1616513075.258830:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a79f80 x1693067725924816/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513075.258831:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513075.258833:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:5.0:1616513075.258844:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513075.258846:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:24.0:1616513075.258847:0:6480:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc7e00d80 x1693067725924848/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513075.258847:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be41680 x1693067725924832/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513075.258850:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000400:00000200:11.0:1616513075.258856:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bc28 00000100:00000200:24.0:1616513075.258860:0:6480:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc7e00d80 x1693067725924848/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513075.258870:0:6480:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000100:00000200:22.0:1616513075.258876:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a79f80 x1693067725924816/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513075.258888:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a79f80 x1693067725924816/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513075.258898:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:11.2:1616513075.267838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.267894:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.267903:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.267912:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.240@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.267921:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.240@tcp1 of length 224 into portal 26 MB=0x5fff0e4077100 00000400:00000200:28.0:1616513075.267928:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.240@tcp1 of length 224/224 into md 0xf97c6129 [1] + 0 00000400:00000200:28.0:1616513075.267934:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775320 00000400:00000200:28.0:1616513075.267936:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.267939:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.267944:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775320 00000800:00000200:28.0:1616513075.267947:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.267947:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.267950:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.267953:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.267956:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.267957:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.267966:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.240@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.267975:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.240@tcp1 of length 224 into portal 12 MB=0x5fff0e4077440 00000400:00000200:12.0:1616513075.267981:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.240@tcp1 of length 224/224 into md 0x865b9f31 [1] + 83512 00000400:00000200:12.0:1616513075.267986:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.267989:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.267996:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.267999:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.268001:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.268009:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688784966480128 00010000:00000200:15.0:1616513075.268021:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688784966480128/t0(0) o400->0b1da6af-66e8-56da-6681-a27cf90b2eeb@10.31.141.240@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.268037:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688784966480128, offset 192 00000400:00000200:15.0:1616513075.268043:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.240@tcp1 00000400:00000200:15.0:1616513075.268052:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.240@tcp1 routed destination 00000100:00000200:24.0:1616513075.268061:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688784966480960 00000400:00000200:15.0:1616513075.268067:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.240@tcp1(10.31.141.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:24.0:1616513075.268074:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c664c80 x1688784966480960/t0(0) o400->a0c801ad-6178-1674-a3e3-23de11b5667f@10.31.141.240@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.268074:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.268081:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.268084:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.268086:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.268091:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513075.268098:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.268098:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513075.268099:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688784966480960, offset 192 00000400:00000200:24.0:1616513075.268105:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.240@tcp1 00000400:00000200:24.0:1616513075.268120:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.240@tcp1 routed destination 00000400:00000200:24.0:1616513075.268135:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.240@tcp1(10.31.141.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.268141:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.268147:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.268150:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.268152:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513075.268153:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513075.268157:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513075.268160:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513075.268162:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513075.268163:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513075.268168:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513075.268170:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513075.268171:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513075.268175:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2ddc0 00000800:00000200:12.0:1616513075.268175:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.268177:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.268179:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513075.268179:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513075.268183:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.268188:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:12.0:1616513075.268191:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.269176:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.269186:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.269190:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.269198:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.211@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.269204:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.211@tcp1 of length 224 into portal 26 MB=0x5ffd4f46ed000 00000400:00000200:12.0:1616513075.269211:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.211@tcp1 of length 224/224 into md 0xf97c6149 [1] + 0 00000400:00000200:12.0:1616513075.269215:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775dc0 00000400:00000200:12.0:1616513075.269216:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.269219:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513075.269223:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775dc0 00000800:00000200:12.0:1616513075.269225:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.269228:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.269229:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.269232:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.269238:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664982605824 00010000:00000200:15.0:1616513075.269244:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688664982605824/t0(0) o400->84915e45-0bd7-b2ff-c277-10bf3bd0c1b7@10.31.132.211@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.269255:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664982605824, offset 192 00000400:00000200:15.0:1616513075.269260:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.211@tcp1 00000400:00000200:15.0:1616513075.269267:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.211@tcp1 routed destination 00000400:00000200:15.0:1616513075.269281:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.211@tcp1(10.31.132.211@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.269286:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.269292:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.269295:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.269296:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.269300:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513075.269305:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.269305:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.269369:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.269377:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.269381:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.269388:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.269394:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c550 00000800:00000200:2.0:1616513075.269398:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.269401:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.269407:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513075.269409:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.269411:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.269413:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.269415:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.269488:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.269551:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.269571:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513075.269572:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.269579:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.211@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.269587:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.211@tcp1 of length 224 into portal 12 MB=0x5ffd4f46ed0c0 00000400:00000200:28.0:1616513075.269594:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.211@tcp1 of length 224/224 into md 0x865b9f31 [1] + 83736 00000400:00000200:28.0:1616513075.269599:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.269602:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.269608:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.269611:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.269613:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.269671:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664982606016 00010000:00000200:24.0:1616513075.269684:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf87980 x1688664982606016/t0(0) o400->32670036-3bb4-6b5c-8f62-9135acb35229@10.31.132.211@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.269698:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664982606016, offset 192 00000400:00000200:24.0:1616513075.269704:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.211@tcp1 00000400:00000200:24.0:1616513075.269712:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.211@tcp1 routed destination 00000400:00000200:24.0:1616513075.269727:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.211@tcp1(10.31.132.211@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.269733:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.269739:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.269742:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.269744:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.269749:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.269753:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.269762:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.269765:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.269767:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513075.269771:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:16.0:1616513075.269772:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.269774:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2330 00000800:00000200:12.0:1616513075.269776:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.273114:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513075.273120:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513075.273167:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.273175:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.273183:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.31@tcp1 : PUT - for me 00000800:00000200:14.0:1616513075.273188:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513075.273192:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.31@tcp1 of length 224 into portal 26 MB=0x5ffe9d054bb80 00000800:00000200:14.0:1616513075.273197:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513075.273199:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.31@tcp1 of length 224/224 into md 0xf97c6159 [1] + 0 00000400:00000200:12.0:1616513075.273203:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7775e48 00000400:00000200:14.0:1616513075.273206:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.31@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.273206:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.273208:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513075.273213:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7775e48 00000400:00000200:14.0:1616513075.273215:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.31@tcp1 of length 224 into portal 12 MB=0x5ffe9d054bc00 00000800:00000200:12.0:1616513075.273216:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.273219:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.273219:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513075.273221:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.31@tcp1 of length 224/224 into md 0xf96450b9 [1] + 49520 00000800:00000200:12.0:1616513075.273223:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513075.273226:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.273229:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.273236:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.273239:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.273241:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.273244:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.273277:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688754571230080 00010000:00000200:15.0:1616513075.273288:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688754571230080/t0(0) o400->26c818ec-caf0-e8cf-22d6-1e0cd38d58b8@10.31.140.31@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513075.273300:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688754571230208 00000100:00000200:15.0:1616513075.273302:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688754571230080, offset 192 00000400:00000200:15.0:1616513075.273309:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.31@tcp1 00010000:00000200:18.0:1616513075.273312:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e2d00 x1688754571230208/t0(0) o400->ab697dd3-028f-7045-bc3d-3ec5f08e7fed@10.31.140.31@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513075.273317:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.31@tcp1 routed destination 00000100:00000200:18.0:1616513075.273327:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688754571230208, offset 192 00000400:00000200:18.0:1616513075.273333:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.31@tcp1 00000400:00000200:15.0:1616513075.273334:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.31@tcp1(10.31.140.31@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.273342:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513075.273344:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.31@tcp1 routed destination 00000800:00000200:15.0:1616513075.273348:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.273351:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.273353:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513075.273358:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.31@tcp1(10.31.140.31@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.273359:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.273364:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.273365:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:15.0:1616513075.273365:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513075.273370:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.273373:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.273376:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.273381:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.273386:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513075.273416:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.273423:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.273426:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.273432:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.273451:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c990 00000800:00000200:22.0:1616513075.273454:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.273457:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513075.273458:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.273465:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.273468:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.273469:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.273474:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.273478:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cde48 00000800:00000200:16.0:1616513075.273483:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.287425:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.287481:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.287491:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.287500:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.87@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.287509:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.87@tcp1 of length 224 into portal 26 MB=0x6018382748c00 00000400:00000200:2.0:1616513075.287516:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.87@tcp1 of length 224/224 into md 0xf97c6199 [1] + 0 00000400:00000200:2.0:1616513075.287520:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d540 00000400:00000200:2.0:1616513075.287523:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.287526:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513075.287531:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d540 00000800:00000200:2.0:1616513075.287534:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.287537:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.287538:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.287541:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.287593:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514201283584 00010000:00000200:15.0:1616513075.287604:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1690514201283584/t0(0) o400->dd6d5b97-ee51-aae4-1bca-38ac2f81453e@10.31.131.87@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.287617:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514201283584, offset 192 00000400:00000200:15.0:1616513075.287623:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.87@tcp1 00000800:00000200:11.2:1616513075.287628:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:15.0:1616513075.287631:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.87@tcp1 routed destination 00000400:00000200:15.0:1616513075.287645:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.87@tcp1(10.31.131.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.287652:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.287657:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.287660:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.287662:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.287667:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.287671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.287673:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513075.287682:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.287691:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.287699:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.87@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.287708:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.87@tcp1 of length 224 into portal 12 MB=0x6018382748c40 00000400:00000200:28.0:1616513075.287714:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.87@tcp1 of length 224/224 into md 0x865b9f31 [1] + 83960 00000400:00000200:28.0:1616513075.287719:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.287721:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513075.287722:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513075.287727:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513075.287728:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513075.287730:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513075.287731:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.287732:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.287734:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513075.287737:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.287743:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c000 00000800:00000200:22.0:1616513075.287746:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.287749:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513075.287790:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514201283648 00010000:00000200:24.0:1616513075.287802:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf86300 x1690514201283648/t0(0) o400->11c55ef4-10ef-0b9e-8e16-2dacbfd6c3f1@10.31.131.87@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.287815:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514201283648, offset 192 00000400:00000200:24.0:1616513075.287822:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.87@tcp1 00000400:00000200:24.0:1616513075.287829:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.87@tcp1 routed destination 00000400:00000200:24.0:1616513075.287843:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.87@tcp1(10.31.131.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.287851:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.287856:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.287859:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.287861:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.287865:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.287870:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.287881:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.287888:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.287890:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513075.287897:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.287900:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2f68 00000800:00000200:12.0:1616513075.287904:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.287907:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.298370:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.298427:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.298450:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.298460:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.85@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.298469:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.85@tcp1 of length 224 into portal 26 MB=0x60007d405b380 00000400:00000200:28.0:1616513075.298477:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.85@tcp1 of length 224/224 into md 0xf97c61a9 [1] + 0 00000800:00000200:12.0:1616513075.298480:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.298482:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c6e8 00000400:00000200:28.0:1616513075.298484:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.298487:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.298492:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c6e8 00000800:00000200:28.0:1616513075.298495:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.298498:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.298500:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.298554:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688883482178432 00010000:00000200:15.0:1616513075.298564:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688883482178432/t0(0) o400->565ff600-f504-f249-c439-05eff60c1081@10.31.138.85@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.298577:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688883482178432, offset 192 00000400:00000200:15.0:1616513075.298583:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.85@tcp1 00000400:00000200:15.0:1616513075.298591:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.85@tcp1 routed destination 00000400:00000200:15.0:1616513075.298605:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.85@tcp1(10.31.138.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.298611:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.298617:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.298621:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.298623:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.298628:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513075.298634:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.298634:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.298700:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.298707:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.298710:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.298717:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.298723:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c198 00000800:00000200:2.0:1616513075.298727:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.298730:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.298764:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513075.298818:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.298827:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.298837:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.85@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.298848:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.85@tcp1 of length 224 into portal 12 MB=0x60007d405b580 00000400:00000200:22.0:1616513075.298855:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.85@tcp1 of length 224/224 into md 0xf96450b9 [1] + 49744 00000400:00000200:22.0:1616513075.298860:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.298863:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513075.298871:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.298871:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.298874:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.298876:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.298934:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688883482178944 00010000:00000200:18.0:1616513075.298946:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2ec00 x1688883482178944/t0(0) o400->5f8aae9a-de25-f89f-3502-b98b3dd704a3@10.31.138.85@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.298960:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688883482178944, offset 192 00000400:00000200:18.0:1616513075.298966:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.85@tcp1 00000400:00000200:18.0:1616513075.298973:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.85@tcp1 routed destination 00000400:00000200:18.0:1616513075.298988:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.85@tcp1(10.31.138.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.298994:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.298999:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.299002:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.299004:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.299008:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.299014:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.299025:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.299030:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.299033:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513075.299040:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.299044:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd430 00000800:00000200:12.0:1616513075.299048:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.299050:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513075.306613:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924864, portal 4 00000100:00000200:28.0:1616513075.306619:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924864, offset 0 00000400:00000200:28.0:1616513075.306627:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:28.0:1616513075.306633:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:28.0:1616513075.306646:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513075.306652:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:28.0:1616513075.306658:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513075.306661:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:28.0:1616513075.306663:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:28.0:1616513075.306668:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513075.306714:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.306732:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513075.306738:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513075.306741:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513075.306748:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513075.306755:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affc3f00 x1693067725924864/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513075.306767:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc550 00000800:00000200:1.0:1616513075.306772:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513075.306775:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513075.306883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513075.306892:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.306898:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513075.306906:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513075.306914:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626a00 00000400:00000200:1.0:1616513075.306920:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffa35 [1] + 192 00000400:00000200:1.0:1616513075.306925:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513075.306930:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affc3f00 x1693067725924864/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513075.306943:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513075.306944:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513075.306947:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513075.306948:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:28.0:1616513075.306953:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affc3f00 x1693067725924864/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513075.306966:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affc3f00 x1693067725924864/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513075.306977:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd8f8 00000800:00000200:7.2:1616513075.307629:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.307686:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.307695:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.307704:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.55@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.307715:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.55@tcp1 of length 224 into portal 26 MB=0x5ffdb1f4a0240 00000400:00000200:2.0:1616513075.307722:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.55@tcp1 of length 224/224 into md 0xf97c61d9 [1] + 0 00000400:00000200:2.0:1616513075.307727:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2cee0 00000400:00000200:2.0:1616513075.307730:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.307732:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513075.307737:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cee0 00000800:00000200:14.0:1616513075.307739:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.307740:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.307743:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.307745:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.307798:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691471417920 00010000:00000200:15.0:1616513075.307808:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688691471417920/t0(0) o400->503e2c81-69ac-7207-de8b-3242ead0cf37@10.31.141.55@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.307820:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688691471417920, offset 192 00000400:00000200:15.0:1616513075.307825:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.55@tcp1 00000400:00000200:15.0:1616513075.307832:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.55@tcp1 routed destination 00000400:00000200:15.0:1616513075.307846:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.55@tcp1(10.31.141.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.307851:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.307857:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.307859:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.307861:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.307866:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.307870:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.307871:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513075.307923:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.307931:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.307935:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.307942:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.307949:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c2a8 00000800:00000200:10.0:1616513075.307953:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.307955:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.307961:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513075.307964:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.307969:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.55@tcp1 : PUT - for me 00000400:00000200:10.0:1616513075.307977:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.55@tcp1 of length 224 into portal 12 MB=0x5ffdb1f4a0380 00000400:00000200:10.0:1616513075.307983:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.55@tcp1 of length 224/224 into md 0xf96450b9 [1] + 49968 00000400:00000200:10.0:1616513075.307988:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.307990:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513075.307996:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.307999:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.308001:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.308061:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691471418240 00010000:00000200:18.0:1616513075.308073:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bda880 x1688691471418240/t0(0) o400->3db675f0-b059-1e04-2ac7-bfb16737b436@10.31.141.55@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.308088:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688691471418240, offset 192 00000400:00000200:18.0:1616513075.308095:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.55@tcp1 00000400:00000200:18.0:1616513075.308102:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.55@tcp1 routed destination 00000400:00000200:18.0:1616513075.308117:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.55@tcp1(10.31.141.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.308124:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.308138:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.308140:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.308142:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.308147:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.308155:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.308209:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.308216:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.308218:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513075.308225:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.308229:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccb28 00000800:00000200:12.0:1616513075.308233:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.308236:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.312012:0:48940:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.312066:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.312075:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.312084:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.39@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.312093:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.39@tcp1 of length 224 into portal 26 MB=0x5ffdce8e4ea80 00000400:00000200:2.0:1616513075.312100:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.39@tcp1 of length 224/224 into md 0xf97c6209 [1] + 0 00000400:00000200:2.0:1616513075.312104:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d430 00000400:00000200:2.0:1616513075.312107:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.312110:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513075.312114:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d430 00000800:00000200:2.0:1616513075.312117:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.312119:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.312120:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.312126:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.312129:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.312130:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.312138:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.39@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.312147:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.39@tcp1 of length 224 into portal 12 MB=0x5ffdce8e4eb00 00000400:00000200:14.0:1616513075.312154:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.39@tcp1 of length 224/224 into md 0xf96450b9 [1] + 50192 00000400:00000200:14.0:1616513075.312159:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.312161:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.312168:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.312173:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513075.312174:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699148757632 00000800:00000200:14.0:1616513075.312175:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:15.0:1616513075.312183:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688699148757632/t0(0) o400->f81b1cc7-4773-b354-7957-d3580af7fba4@10.31.142.39@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.312196:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699148757632, offset 192 00000400:00000200:15.0:1616513075.312202:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.39@tcp1 00000400:00000200:15.0:1616513075.312209:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.39@tcp1 routed destination 00000400:00000200:15.0:1616513075.312222:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.39@tcp1(10.31.142.39@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.312228:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:18.0:1616513075.312232:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699148757760 00000800:00000200:15.0:1616513075.312233:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.312236:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.312238:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.312242:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:18.0:1616513075.312245:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e3180 x1688699148757760/t0(0) o400->ff97c159-2c10-3a0c-1089-b0eabdb61400@10.31.142.39@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513075.312246:0:48940:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.312248:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513075.312260:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699148757760, offset 192 00000400:00000200:18.0:1616513075.312269:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.39@tcp1 00000400:00000200:18.0:1616513075.312277:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.39@tcp1 routed destination 00000400:00000200:18.0:1616513075.312292:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.39@tcp1(10.31.142.39@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.312298:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513075.312299:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513075.312304:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.312307:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513075.312307:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513075.312309:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513075.312310:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.312310:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513075.312314:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513075.312316:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513075.312319:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513075.312323:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d188 00000800:00000200:10.0:1616513075.312326:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513075.312389:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.312396:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.312399:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513075.312405:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.312409:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cce58 00000800:00000200:12.0:1616513075.312413:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.312416:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.313924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513075.313985:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.313994:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.314003:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.105@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.314012:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.105@tcp1 of length 224 into portal 26 MB=0x604a2eba5bc80 00000400:00000200:22.0:1616513075.314019:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.105@tcp1 of length 224/224 into md 0xf97c6219 [1] + 0 00000400:00000200:22.0:1616513075.314024:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22a18 00000400:00000200:22.0:1616513075.314027:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.314030:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513075.314034:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22a18 00000800:00000200:14.0:1616513075.314037:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.314038:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.314040:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.314042:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.314100:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947644984448 00010000:00000200:15.0:1616513075.314111:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1693947644984448/t0(0) o400->3b0c7386-6cb1-1045-3db3-937a9d7c0523@10.31.140.105@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.314126:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693947644984448, offset 192 00000400:00000200:15.0:1616513075.314132:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.105@tcp1 00000400:00000200:15.0:1616513075.314141:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.105@tcp1 routed destination 00000400:00000200:15.0:1616513075.314156:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.105@tcp1(10.31.140.105@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.314163:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.314182:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.314184:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.314186:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.314190:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.314194:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.314196:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513075.314235:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513075.314246:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.314253:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.314256:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.314262:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.314268:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2df58 00000800:00000200:2.0:1616513075.314271:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.314274:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513075.314286:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.314295:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513075.314301:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.314303:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.105@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.314312:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.105@tcp1 of length 224 into portal 12 MB=0x604a2eba5bf80 00000400:00000200:12.0:1616513075.314319:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.105@tcp1 of length 224/224 into md 0x865b9f31 [1] + 84184 00000400:00000200:12.0:1616513075.314324:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.314326:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.314332:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.314335:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.314337:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.314397:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947644985216 00010000:00000200:24.0:1616513075.314410:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c665580 x1693947644985216/t0(0) o400->ef287ea9-3946-beed-e54e-8bddd374cab5@10.31.140.105@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.314424:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693947644985216, offset 192 00000400:00000200:24.0:1616513075.314431:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.105@tcp1 00000400:00000200:24.0:1616513075.314452:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.105@tcp1 routed destination 00000400:00000200:24.0:1616513075.314467:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.105@tcp1(10.31.140.105@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.314474:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.314479:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.314482:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.314484:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.314489:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.314514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.314523:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.314526:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.314528:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513075.314544:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.314547:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf35c8 00000800:00000200:12.0:1616513075.314550:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.314553:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.315073:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.315085:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.315093:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513075.315093:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.315102:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.254@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.315111:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.254@tcp1 of length 224 into portal 26 MB=0x5fff984877440 00000400:00000200:28.0:1616513075.315118:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.254@tcp1 of length 224/224 into md 0xf97c6229 [1] + 0 00000400:00000200:28.0:1616513075.315123:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22908 00000400:00000200:28.0:1616513075.315126:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.315128:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.315133:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22908 00000800:00000200:28.0:1616513075.315136:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.315139:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.315141:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.315146:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688822018962496 00010000:00000200:15.0:1616513075.315152:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688822018962496/t0(0) o400->79314cab-cfb6-2e96-19b9-b440c4ae8c24@10.31.141.254@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.315162:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688822018962496, offset 192 00000400:00000200:15.0:1616513075.315166:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.254@tcp1 00000800:00000200:11.2:1616513075.315166:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:15.0:1616513075.315172:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.254@tcp1 routed destination 00000800:00000200:12.0:1616513075.315175:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.315180:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513075.315184:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:15.0:1616513075.315185:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.254@tcp1(10.31.141.254@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513075.315188:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.254@tcp1 : PUT - for me 00000800:00000200:15.0:1616513075.315190:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.315195:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513075.315195:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.254@tcp1 of length 224 into portal 12 MB=0x5fff984877640 00000800:00000200:15.0:1616513075.315197:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.315199:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513075.315200:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.254@tcp1 of length 224/224 into md 0x865b9f31 [1] + 84408 00000800:00000200:15.0:1616513075.315202:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513075.315204:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:15.0:1616513075.315206:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513075.315206:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513075.315206:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513075.315211:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.315213:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.315215:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.315225:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688822018963008 00010000:00000200:24.0:1616513075.315231:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c662400 x1688822018963008/t0(0) o400->738167e3-7f79-b8d1-4bf1-a2c3d310052f@10.31.141.254@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.315243:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688822018963008, offset 192 00000400:00000200:24.0:1616513075.315249:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.254@tcp1 00000400:00000200:24.0:1616513075.315254:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.254@tcp1 routed destination 00000800:00000200:14.0:1616513075.315259:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.315266:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:24.0:1616513075.315268:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.254@tcp1(10.31.141.254@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513075.315270:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513075.315273:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513075.315276:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513075.315278:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.315281:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513075.315282:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d4b8 00000800:00000200:24.0:1616513075.315283:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.315285:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513075.315286:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.315289:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.315290:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.315298:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.315302:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.315304:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.315306:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.315309:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.315311:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ee0 00000800:00000200:12.0:1616513075.315314:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.341608:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513075.341667:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.341677:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.341693:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.51@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.341702:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.51@tcp1 of length 224 into portal 26 MB=0x5f9095ecd1100 00000400:00000200:16.0:1616513075.341709:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.51@tcp1 of length 224/224 into md 0xf97c6249 [1] + 0 00000400:00000200:16.0:1616513075.341714:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23188 00000400:00000200:16.0:1616513075.341717:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.341720:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513075.341721:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513075.341726:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23188 00000800:00000200:16.0:1616513075.341729:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513075.341732:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513075.341732:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513075.341734:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.341737:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.341740:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.51@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.341751:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.51@tcp1 of length 224 into portal 12 MB=0x5f9095ecd1180 00000400:00000200:12.0:1616513075.341757:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.51@tcp1 of length 224/224 into md 0x865b9f31 [1] + 84632 00000400:00000200:12.0:1616513075.341763:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.341765:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.341772:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.341775:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.341777:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.341787:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681193524072704 00010000:00000200:15.0:1616513075.341796:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1681193524072704/t0(0) o400->3fac018b-43fe-504e-91d4-961288920bce@10.31.131.51@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.341808:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681193524072704, offset 192 00000400:00000200:15.0:1616513075.341822:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.51@tcp1 00000400:00000200:15.0:1616513075.341829:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.51@tcp1 routed destination 00000100:00000200:24.0:1616513075.341838:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681193524072832 00000400:00000200:15.0:1616513075.341842:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.51@tcp1(10.31.131.51@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.341848:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:24.0:1616513075.341851:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663600 x1681193524072832/t0(0) o400->741ebdb6-9d53-37fa-e73f-321732d08a1e@10.31.131.51@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.341854:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.341857:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.341858:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.341863:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513075.341866:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681193524072832, offset 192 00000100:00000200:15.0:1616513075.341869:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.341869:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513075.341872:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.51@tcp1 00000400:00000200:24.0:1616513075.341880:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.51@tcp1 routed destination 00000400:00000200:24.0:1616513075.341895:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.51@tcp1(10.31.131.51@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.341902:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.341908:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.341910:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.341913:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.341918:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.341922:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513075.341926:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513075.341931:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513075.341933:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513075.341935:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513075.341937:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513075.341938:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.341941:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.341944:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.341945:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.341947:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3980 00000800:00000200:12.0:1616513075.341950:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513075.341951:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d650 00000800:00000200:2.0:1616513075.341955:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.341958:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.348543:0:48944:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.348600:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.348610:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.348619:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.248@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.348627:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.248@tcp1 of length 224 into portal 26 MB=0x5fff7c48c08c0 00000400:00000200:14.0:1616513075.348634:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.248@tcp1 of length 224/224 into md 0xf97c6269 [1] + 0 00000400:00000200:14.0:1616513075.348639:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee222a8 00000400:00000200:14.0:1616513075.348642:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.348645:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513075.348650:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee222a8 00000800:00000200:14.0:1616513075.348653:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.348655:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.348657:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513075.348665:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.348668:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.348669:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.348678:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.248@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.348687:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.248@tcp1 of length 224 into portal 12 MB=0x5fff7c48c0940 00000400:00000200:22.0:1616513075.348694:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.248@tcp1 of length 224/224 into md 0xf96450b9 [1] + 50416 00000400:00000200:22.0:1616513075.348699:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.348702:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513075.348709:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.348709:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688814503069888 00000800:00000200:22.0:1616513075.348712:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.348714:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:15.0:1616513075.348719:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688814503069888/t0(0) o400->7c001225-d810-024c-0b86-53a299c9381f@10.31.141.248@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.348732:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688814503069888, offset 192 00000400:00000200:15.0:1616513075.348737:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.248@tcp1 00000400:00000200:15.0:1616513075.348744:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.248@tcp1 routed destination 00000400:00000200:15.0:1616513075.348757:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.248@tcp1(10.31.141.248@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.348763:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.348769:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.348772:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.348773:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.348778:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.348782:0:48944:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.348785:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513075.348796:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688814503070016 00010000:00000200:18.0:1616513075.348808:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a28480 x1688814503070016/t0(0) o400->a5c3b487-9447-0a63-cd18-5abc6d6e6f31@10.31.141.248@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.348823:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688814503070016, offset 192 00000400:00000200:18.0:1616513075.348830:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.248@tcp1 00000800:00000200:10.0:1616513075.348836:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513075.348837:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.248@tcp1 routed destination 00000800:00000200:10.0:1616513075.348844:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.348848:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.348848:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513075.348852:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.248@tcp1(10.31.141.248@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513075.348854:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513075.348859:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:10.0:1616513075.348860:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2df58 00000800:00000200:10.0:1616513075.348864:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513075.348865:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.348868:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.348870:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.348875:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.348882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.348937:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.348944:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.348947:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513075.348953:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.348957:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc330 00000800:00000200:16.0:1616513075.348961:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.348964:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.355643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.355700:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.355709:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.355718:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.32@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.355727:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.32@tcp1 of length 224 into portal 26 MB=0x5ffdb67412100 00000400:00000200:28.0:1616513075.355736:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.32@tcp1 of length 224/224 into md 0xf97c6289 [1] + 0 00000400:00000200:28.0:1616513075.355741:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22110 00000400:00000200:28.0:1616513075.355743:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.355746:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.355751:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22110 00000800:00000200:12.0:1616513075.355753:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.355754:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.355757:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513075.355759:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.355762:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.355762:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.355770:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.32@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.355779:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.32@tcp1 of length 224 into portal 12 MB=0x5ffdb674121c0 00000400:00000200:12.0:1616513075.355784:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.32@tcp1 of length 224/224 into md 0x865b9f31 [1] + 84856 00000400:00000200:12.0:1616513075.355789:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.355791:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.355798:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.355801:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.355802:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.355816:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692678795520 00010000:00000200:15.0:1616513075.355827:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688692678795520/t0(0) o400->f2a3b1c3-4661-12bd-a41b-5188fdc4075d@10.31.141.32@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.355841:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692678795520, offset 192 00000400:00000200:15.0:1616513075.355857:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.32@tcp1 00000400:00000200:15.0:1616513075.355864:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.32@tcp1 routed destination 00000100:00000200:24.0:1616513075.355868:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692678795712 00000400:00000200:15.0:1616513075.355877:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.32@tcp1(10.31.141.32@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:24.0:1616513075.355880:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663f00 x1688692678795712/t0(0) o400->f8c215e1-bb0a-17de-4c6d-96fd05404bb2@10.31.141.32@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.355883:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.355890:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.355893:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513075.355895:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692678795712, offset 192 00000800:00000200:15.0:1616513075.355895:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.355899:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513075.355901:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.32@tcp1 00000100:00000200:15.0:1616513075.355905:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513075.355909:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.32@tcp1 routed destination 00000400:00000200:24.0:1616513075.355924:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.32@tcp1(10.31.141.32@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.355931:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.355936:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.355939:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.355941:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513075.355941:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.355946:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.355951:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.355960:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.355964:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.355966:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.355968:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.355972:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.355975:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000800:00000200:12.0:1616513075.355979:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513075.355996:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.356003:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.356006:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.356013:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.356019:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d430 00000800:00000200:2.0:1616513075.356023:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.356026:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.356534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.356588:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.356597:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.356606:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.126@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.356615:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.126@tcp1 of length 224 into portal 26 MB=0x5ffdb251a58c0 00000400:00000200:14.0:1616513075.356622:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.126@tcp1 of length 224/224 into md 0xf97c6299 [1] + 0 00000400:00000200:14.0:1616513075.356627:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23298 00000400:00000200:14.0:1616513075.356629:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.356632:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513075.356637:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23298 00000800:00000200:14.0:1616513075.356643:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.356645:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.356646:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:15.0:1616513075.356650:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691568957632 00000800:00000200:14.0:1616513075.356650:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:15.0:1616513075.356656:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688691568957632/t0(0) o400->e7316964-d220-abe4-5b44-f609ab24b016@10.31.140.126@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.356666:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688691568957632, offset 192 00000400:00000200:15.0:1616513075.356670:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.126@tcp1 00000400:00000200:15.0:1616513075.356676:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.126@tcp1 routed destination 00000400:00000200:15.0:1616513075.356688:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.126@tcp1(10.31.140.126@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.356693:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.356697:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.356700:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.356701:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.356704:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513075.356708:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.356708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.356717:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.356722:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.356724:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513075.356727:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513075.356729:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.356734:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cee0 00000800:00000200:2.0:1616513075.356737:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.356807:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.356816:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.356821:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513075.356825:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.356828:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.126@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.356835:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.126@tcp1 of length 224 into portal 12 MB=0x5ffdb251a5c40 00000400:00000200:12.0:1616513075.356841:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.126@tcp1 of length 224/224 into md 0x865b9f31 [1] + 85080 00000400:00000200:12.0:1616513075.356846:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.356848:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.356853:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.356855:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.356856:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.356866:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691568958528 00010000:00000200:24.0:1616513075.356872:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c660000 x1688691568958528/t0(0) o400->e14e134b-9b54-1ad7-e663-65a4f63a34a8@10.31.140.126@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.356884:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688691568958528, offset 192 00000400:00000200:24.0:1616513075.356889:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.126@tcp1 00000400:00000200:24.0:1616513075.356895:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.126@tcp1 routed destination 00000400:00000200:24.0:1616513075.356910:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.126@tcp1(10.31.140.126@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.356916:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.356921:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.356923:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.356925:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.356928:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.356933:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.356942:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.356946:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.356948:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.356950:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.356954:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.356957:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000800:00000200:28.0:1616513075.356960:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.363121:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513075.363137:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513075.363175:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.363184:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.363193:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.57@tcp1 : PUT - for me 00000800:00000200:10.0:1616513075.363194:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513075.363202:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.57@tcp1 of length 224 into portal 26 MB=0x5ffdc8588ce00 00000800:00000200:10.0:1616513075.363203:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513075.363210:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.57@tcp1 of length 224/224 into md 0xf97c62b9 [1] + 0 00000400:00000200:10.0:1616513075.363212:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.57@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.363215:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22198 00000400:00000200:16.0:1616513075.363217:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.363220:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513075.363221:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.57@tcp1 of length 224 into portal 12 MB=0x5ffdc8588ce80 00000400:00000200:16.0:1616513075.363225:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22198 00000800:00000200:12.0:1616513075.363227:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.363228:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:10.0:1616513075.363230:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.57@tcp1 of length 224/224 into md 0xf96450b9 [1] + 50640 00000800:00000200:16.0:1616513075.363231:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.363233:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513075.363235:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.363240:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.363249:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.363250:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.363252:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.363255:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.363285:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697481776640 00010000:00000200:15.0:1616513075.363295:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688697481776640/t0(0) o400->f4d7e6ab-a807-c621-4da8-175e563941f7@10.31.140.57@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.363308:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697481776640, offset 192 00000400:00000200:15.0:1616513075.363313:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.57@tcp1 00000100:00000200:18.0:1616513075.363315:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697481776768 00000400:00000200:15.0:1616513075.363320:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.57@tcp1 routed destination 00010000:00000200:18.0:1616513075.363328:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdf980 x1688697481776768/t0(0) o400->86a77b4d-c9ec-e4f7-a104-9e79c84b1446@10.31.140.57@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513075.363333:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.57@tcp1(10.31.140.57@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.363339:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:18.0:1616513075.363342:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697481776768, offset 192 00000800:00000200:15.0:1616513075.363345:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.363347:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:18.0:1616513075.363349:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.57@tcp1 00000800:00000200:15.0:1616513075.363349:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.363353:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513075.363357:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.57@tcp1 routed destination 00000800:00000200:7.2:1616513075.363358:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.363362:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513075.363372:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.57@tcp1(10.31.140.57@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.363379:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.363385:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.363389:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.363391:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.363396:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.363401:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513075.363411:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513075.363411:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.363416:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513075.363418:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:12.0:1616513075.363418:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.363423:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.363423:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513075.363425:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.363429:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd980 00000400:00000200:22.0:1616513075.363430:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513075.363447:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513075.363448:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c6e8 00000800:00000200:12.0:1616513075.363450:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513075.363452:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.381206:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.381261:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.381271:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.381280:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.78@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.381288:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.78@tcp1 of length 224 into portal 26 MB=0x601847a396980 00000400:00000200:14.0:1616513075.381295:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.78@tcp1 of length 224/224 into md 0xf97c62d9 [1] + 0 00000400:00000200:14.0:1616513075.381300:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23cb0 00000400:00000200:14.0:1616513075.381302:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.381305:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513075.381310:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23cb0 00000800:00000200:14.0:1616513075.381313:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.381315:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.381316:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.381319:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.381372:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690518358157696 00010000:00000200:15.0:1616513075.381383:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1690518358157696/t0(0) o400->3a428e2a-3563-d8fd-a25e-81edba6af106@10.31.140.78@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.381395:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690518358157696, offset 192 00000400:00000200:15.0:1616513075.381401:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.78@tcp1 00000400:00000200:15.0:1616513075.381407:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.78@tcp1 routed destination 00000400:00000200:15.0:1616513075.381421:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.78@tcp1(10.31.140.78@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.381426:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.381432:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.381435:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.381451:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.381456:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.381460:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.381463:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513075.381523:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.381531:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.381534:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.381541:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.381548:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d540 00000800:00000200:2.0:1616513075.381552:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.381555:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.381680:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.381737:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.381747:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.381756:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.78@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.381764:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.78@tcp1 of length 224 into portal 12 MB=0x601847a396d40 00000400:00000200:12.0:1616513075.381770:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.78@tcp1 of length 224/224 into md 0x865b9f31 [1] + 85304 00000400:00000200:12.0:1616513075.381776:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.381778:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.381784:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.381787:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.381789:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.381792:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.381850:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690518358158656 00010000:00000200:24.0:1616513075.381862:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c662880 x1690518358158656/t0(0) o400->c2b9d88c-bf50-0fb1-bddd-48325d6c555b@10.31.140.78@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.381876:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690518358158656, offset 192 00000400:00000200:24.0:1616513075.381882:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.78@tcp1 00000400:00000200:24.0:1616513075.381890:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.78@tcp1 routed destination 00000400:00000200:24.0:1616513075.381905:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.78@tcp1(10.31.140.78@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.381911:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.381917:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.381920:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.381922:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.381927:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.381941:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.381950:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.381954:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.381956:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513075.381962:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.381965:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3980 00000800:00000200:12.0:1616513075.381969:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.381971:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.386108:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513075.386162:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.386171:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.386182:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.48@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.386192:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.48@tcp1 of length 224 into portal 26 MB=0x5ffd504456fc0 00000400:00000200:16.0:1616513075.386199:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.48@tcp1 of length 224/224 into md 0xf97c62e9 [1] + 0 00000400:00000200:16.0:1616513075.386203:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23078 00000400:00000200:16.0:1616513075.386206:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.386209:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513075.386213:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513075.386214:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23078 00000800:00000200:16.0:1616513075.386217:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513075.386220:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513075.386221:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.386224:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.386224:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.386232:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.48@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.386241:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.48@tcp1 of length 224 into portal 12 MB=0x5ffd504457040 00000400:00000200:12.0:1616513075.386246:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.48@tcp1 of length 224/224 into md 0x865b9f31 [1] + 85528 00000400:00000200:12.0:1616513075.386251:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.386254:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.386260:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.386263:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.386265:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.386281:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665248329664 00010000:00000200:15.0:1616513075.386292:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688665248329664/t0(0) o400->a485be25-6fb0-22d5-f679-958e49fe181d@10.31.131.48@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.386307:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665248329664, offset 192 00000400:00000200:15.0:1616513075.386313:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.48@tcp1 00000400:00000200:15.0:1616513075.386322:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.48@tcp1 routed destination 00000100:00000200:24.0:1616513075.386323:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665248329792 00010000:00000200:24.0:1616513075.386336:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663180 x1688665248329792/t0(0) o400->0e984337-e83f-cb98-7112-510f0a538f7d@10.31.131.48@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513075.386338:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.48@tcp1(10.31.131.48@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.386344:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:24.0:1616513075.386350:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665248329792, offset 192 00000800:00000200:15.0:1616513075.386351:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.386353:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.386356:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513075.386357:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.48@tcp1 00000800:00000200:15.0:1616513075.386360:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513075.386365:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.48@tcp1 routed destination 00000800:00000200:7.2:1616513075.386366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.386368:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513075.386380:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.48@tcp1(10.31.131.48@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.386387:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.386392:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.386395:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.386399:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.386404:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.386409:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513075.386418:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.386426:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.386429:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.386455:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.386461:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c550 00000800:00000200:28.0:1616513075.386464:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513075.386464:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.386468:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513075.386471:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.386474:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513075.386475:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.386480:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.386484:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ee0 00000800:00000200:28.0:1616513075.386488:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.395077:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.395133:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.395143:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.395152:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.101@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.395161:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.101@tcp1 of length 224 into portal 26 MB=0x60183fe168bc0 00000400:00000200:2.0:1616513075.395168:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.101@tcp1 of length 224/224 into md 0xf97c62f9 [1] + 0 00000400:00000200:2.0:1616513075.395173:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22d48 00000400:00000200:2.0:1616513075.395176:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.395178:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513075.395183:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22d48 00000800:00000200:22.0:1616513075.395187:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.395188:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.395191:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.395193:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.395258:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516275497920 00010000:00000200:15.0:1616513075.395268:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1690516275497920/t0(0) o400->d289aa4a-fde2-2400-1c3a-fcd3bbf96289@10.31.142.101@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.395279:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690516275497920, offset 192 00000400:00000200:15.0:1616513075.395286:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.101@tcp1 00000400:00000200:15.0:1616513075.395293:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.101@tcp1 routed destination 00000400:00000200:15.0:1616513075.395305:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.101@tcp1(10.31.142.101@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.395311:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.395316:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.395318:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.395320:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.395324:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.395329:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.395330:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513075.395338:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.395344:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.395347:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.395353:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.395358:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c7f8 00000800:00000200:10.0:1616513075.395362:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.395364:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.395432:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513075.395447:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.395455:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513075.395457:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513075.395464:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.101@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.395473:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.101@tcp1 of length 224 into portal 12 MB=0x60183fe168e40 00000400:00000200:22.0:1616513075.395480:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.101@tcp1 of length 224/224 into md 0xf96450b9 [1] + 50864 00000400:00000200:22.0:1616513075.395485:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.395488:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513075.395495:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.395498:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.395500:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.395557:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516275498560 00010000:00000200:18.0:1616513075.395568:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2f500 x1690516275498560/t0(0) o400->d29be47b-684b-d3c7-329b-b21635c3b17b@10.31.142.101@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.395582:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690516275498560, offset 192 00000400:00000200:18.0:1616513075.395588:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.101@tcp1 00000400:00000200:18.0:1616513075.395595:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.101@tcp1 routed destination 00000400:00000200:18.0:1616513075.395610:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.101@tcp1(10.31.142.101@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.395616:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.395622:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.395624:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.395626:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.395630:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.395635:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.395689:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.395695:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.395698:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513075.395701:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.395704:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.395707:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccd48 00000800:00000200:12.0:1616513075.395711:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.412753:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513075.412765:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.412771:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513075.412776:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:10.0:1616513075.412778:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.133@tcp1 : PUT - for me 00000400:00000200:10.0:1616513075.412800:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.133@tcp1 of length 224 into portal 26 MB=0x60184031e2c40 00000400:00000200:10.0:1616513075.412806:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.133@tcp1 of length 224/224 into md 0xf97c6329 [1] + 0 00000400:00000200:10.0:1616513075.412810:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22e58 00000400:00000200:10.0:1616513075.412812:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.412815:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513075.412819:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22e58 00000800:00000200:10.0:1616513075.412821:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.412823:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.412825:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.412827:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513075.412837:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.412846:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.412855:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.133@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.412865:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.133@tcp1 of length 224 into portal 12 MB=0x60184031e3040 00000400:00000200:16.0:1616513075.412871:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.133@tcp1 of length 224/224 into md 0x865b9f31 [1] + 85752 00000400:00000200:16.0:1616513075.412877:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.412879:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513075.412883:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516359883840 00000800:00000200:16.0:1616513075.412886:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.412889:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.412890:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:15.0:1616513075.412893:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1690516359883840/t0(0) o400->31dc60d2-8c2f-11e9-e51e-1129bc2e265f@10.31.132.133@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513075.412894:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.412907:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690516359883840, offset 192 00000400:00000200:15.0:1616513075.412913:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.133@tcp1 00000400:00000200:15.0:1616513075.412921:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.133@tcp1 routed destination 00000400:00000200:15.0:1616513075.412935:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.133@tcp1(10.31.132.133@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.412941:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.412947:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.412950:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.412952:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.412957:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513075.412961:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516359884864 00000800:00000200:7.2:1616513075.412961:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.412963:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:24.0:1616513075.412973:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd7500 x1690516359884864/t0(0) o400->71ad17dd-d60a-530d-05c6-11a3559f6c1c@10.31.132.133@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.412986:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690516359884864, offset 192 00000400:00000200:24.0:1616513075.412991:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.133@tcp1 00000400:00000200:24.0:1616513075.412999:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.133@tcp1 routed destination 00000400:00000200:24.0:1616513075.413013:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.133@tcp1(10.31.132.133@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.413019:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.413025:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.413027:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513075.413027:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513075.413029:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.413034:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513075.413034:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.413038:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513075.413039:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513075.413044:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.413050:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c770 00000800:00000200:20.0:1616513075.413051:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513075.413054:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.413056:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513075.413058:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.413060:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.413062:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513075.413067:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.413071:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf35c8 00000800:00000200:20.0:1616513075.413075:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.419626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513075.419667:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.419677:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513075.419688:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.228@tcp1 : PUT - for me 00000400:00000200:10.0:1616513075.419697:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.228@tcp1 of length 224 into portal 26 MB=0x5ffd3d3ce14c0 00000400:00000200:10.0:1616513075.419703:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.228@tcp1 of length 224/224 into md 0xf97c6339 [1] + 0 00000400:00000200:10.0:1616513075.419708:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23f58 00000400:00000200:10.0:1616513075.419711:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.419714:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513075.419718:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23f58 00000800:00000200:14.0:1616513075.419721:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.419722:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.419724:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.419726:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.419782:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660140233920 00010000:00000200:15.0:1616513075.419793:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688660140233920/t0(0) o400->d160cae1-e190-edd6-975b-b5e03b5b9bd6@10.31.132.228@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513075.419805:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513075.419808:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660140233920, offset 192 00000400:00000200:15.0:1616513075.419814:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.228@tcp1 00000400:00000200:15.0:1616513075.419822:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.228@tcp1 routed destination 00000400:00000200:15.0:1616513075.419838:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.228@tcp1(10.31.132.228@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.419849:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.419855:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.419858:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.419860:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513075.419864:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.419865:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513075.419872:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.419876:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513075.419881:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.419890:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.228@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.419908:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.228@tcp1 of length 224 into portal 12 MB=0x5ffd3d3ce1540 00000400:00000200:28.0:1616513075.419915:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.228@tcp1 of length 224/224 into md 0x865b9f31 [1] + 85976 00000400:00000200:28.0:1616513075.419920:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.419923:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.419929:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.419932:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.419934:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.419939:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513075.419944:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.419951:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.419954:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.419961:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.419967:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d3a8 00000800:00000200:2.0:1616513075.419971:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.419974:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513075.419991:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660140234048 00010000:00000200:24.0:1616513075.420002:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf85e80 x1688660140234048/t0(0) o400->5378320f-16c2-3272-d08c-0107eca7c671@10.31.132.228@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.420015:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660140234048, offset 192 00000400:00000200:24.0:1616513075.420021:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.228@tcp1 00000400:00000200:24.0:1616513075.420028:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.228@tcp1 routed destination 00000400:00000200:24.0:1616513075.420042:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.228@tcp1(10.31.132.228@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.420048:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.420054:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.420056:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.420058:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.420063:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.420068:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.420079:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.420086:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.420089:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513075.420092:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.420095:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.420099:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2f68 00000800:00000200:12.0:1616513075.420103:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513075.434627:0:6482:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924880, portal 4 00000100:00000200:16.0:1616513075.434632:0:6482:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924880, offset 0 00000400:00000200:16.0:1616513075.434639:0:6482:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:16.0:1616513075.434645:0:6482:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:16.0:1616513075.434658:0:6482:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513075.434664:0:6482:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:16.0:1616513075.434671:0:6482:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513075.434674:0:6482:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:16.0:1616513075.434676:0:6482:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:16.0:1616513075.434683:0:6482:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513075.434731:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.434750:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513075.434756:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513075.434759:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513075.434766:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513075.434773:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb23de80 x1693067725924880/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513075.434785:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43cb0 00000800:00000200:1.0:1616513075.434790:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513075.434793:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513075.434910:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513075.434919:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.434926:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513075.434934:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513075.434942:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626a10 00000400:00000200:1.0:1616513075.434949:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffa45 [1] + 192 00000400:00000200:1.0:1616513075.434954:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513075.434959:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb23de80 x1693067725924880/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513075.434970:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513075.434973:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.434976:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513075.434978:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:16.0:1616513075.434983:0:6482:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb23de80 x1693067725924880/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513075.434996:0:6482:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb23de80 x1693067725924880/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513075.435007:0:6482:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ee0 00000800:00000200:11.2:1616513075.447595:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513075.447597:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513075.447622:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513075.447623:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513075.447631:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513075.447633:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513075.447640:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.210@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.447642:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.210@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.447650:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.210@tcp1 of length 224 into portal 26 MB=0x5ffe6401c8100 00000400:00000200:14.0:1616513075.447650:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.210@tcp1 of length 224 into portal 12 MB=0x5ffe6401c81c0 00000400:00000200:28.0:1616513075.447656:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.210@tcp1 of length 224/224 into md 0xf97c6349 [1] + 0 00000400:00000200:28.0:1616513075.447661:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22ff0 00000400:00000200:28.0:1616513075.447664:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513075.447666:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.210@tcp1 of length 224/224 into md 0xf96450b9 [1] + 51088 00000100:00000200:28.0:1616513075.447671:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513075.447671:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513075.447675:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.447676:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ff0 00000100:00000200:14.0:1616513075.447678:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.447679:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.447682:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.447684:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513075.447686:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.447689:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.447691:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.447693:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.447743:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739266724096 00010000:00000200:15.0:1616513075.447754:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688739266724096/t0(0) o400->fb97cef5-b4be-06d0-c799-50f266940e81@10.31.141.210@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513075.447757:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739266724288 00000100:00000200:15.0:1616513075.447767:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688739266724096, offset 192 00010000:00000200:18.0:1616513075.447770:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e3600 x1688739266724288/t0(0) o400->c0605844-8bbb-55d1-8fd3-f1a6ff0ad310@10.31.141.210@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513075.447773:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.210@tcp1 00000400:00000200:15.0:1616513075.447780:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.210@tcp1 routed destination 00000100:00000200:18.0:1616513075.447785:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688739266724288, offset 192 00000400:00000200:18.0:1616513075.447792:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.210@tcp1 00000400:00000200:15.0:1616513075.447795:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.210@tcp1(10.31.141.210@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.447801:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513075.447804:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.210@tcp1 routed destination 00000800:00000200:15.0:1616513075.447808:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.447811:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.447813:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.447817:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513075.447819:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.210@tcp1(10.31.141.210@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513075.447822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.447824:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513075.447826:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.447832:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513075.447833:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513075.447835:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.447837:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513075.447841:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513075.447842:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513075.447844:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513075.447847:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513075.447851:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.447857:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c7f8 00000800:00000200:20.0:1616513075.447858:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513075.447861:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.447864:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513075.447865:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.447868:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513075.447874:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.447879:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc908 00000800:00000200:20.0:1616513075.447883:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.447886:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:2.0:1616513075.466517:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924896, portal 4 00000100:00000200:2.0:1616513075.466529:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924896, offset 0 00000400:00000200:2.0:1616513075.466536:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:2.0:1616513075.466542:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:2.0:1616513075.466555:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513075.466561:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:2.0:1616513075.466567:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513075.466570:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:2.0:1616513075.466572:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:2.0:1616513075.466579:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513075.466626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513075.466683:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:16.0:1616513075.466690:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:16.0:1616513075.466693:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513075.466699:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:16.0:1616513075.466706:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a7f980 x1693067725924896/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513075.466718:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec6e8 00000800:00000200:16.0:1616513075.466722:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513075.466725:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513075.466809:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513075.466863:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513075.466873:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513075.466882:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513075.466890:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626a20 00000400:00000200:20.0:1616513075.466898:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d4221 [1] + 192 00000400:00000200:20.0:1616513075.466904:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513075.466909:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a7f980 x1693067725924896/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513075.466918:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513075.466924:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513075.466927:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513075.466929:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:2.0:1616513075.466933:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a7f980 x1693067725924896/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513075.466946:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a7f980 x1693067725924896/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513075.466956:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed760 00000100:00000200:2.0:1616513075.482607:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924912, portal 4 00000100:00000200:2.0:1616513075.482612:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924912, offset 0 00000400:00000200:2.0:1616513075.482619:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:2.0:1616513075.482625:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:2.0:1616513075.482638:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513075.482644:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:2.0:1616513075.482650:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513075.482654:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513075.482656:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513075.482661:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513075.482709:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513075.482767:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:10.0:1616513075.482783:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:10.0:1616513075.482786:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.482792:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:10.0:1616513075.482799:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c8a2c380 x1693067725924912/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513075.482810:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec4c8 00000800:00000200:10.0:1616513075.482814:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513075.482817:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513075.482878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513075.482890:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513075.482897:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513075.482901:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513075.482905:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513075.482911:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626a30 00000400:00000200:2.0:1616513075.482917:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c6a59 [1] + 192 00000400:00000200:2.0:1616513075.482922:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.482926:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c8a2c380 x1693067725924912/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513075.482941:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513075.482944:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513075.482946:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:6.0:1616513075.482949:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c8a2c380 x1693067725924912/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:6.0:1616513075.482959:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c8a2c380 x1693067725924912/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513075.482967:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed760 00000800:00000200:7.2:1616513075.531946:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513075.532003:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.532014:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513075.532023:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.87@tcp1 : PUT - for me 00000400:00000200:10.0:1616513075.532034:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.87@tcp1 of length 224 into portal 12 MB=0x5ffd523eda180 00000400:00000200:10.0:1616513075.532040:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.87@tcp1 of length 224/224 into md 0xf96450b9 [1] + 51312 00000400:00000200:10.0:1616513075.532046:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.532049:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513075.532057:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.532059:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.532060:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.532063:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.532125:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665779446144 00010000:00000200:18.0:1616513075.532139:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdb600 x1688665779446144/t0(0) o400->e140f336-e34c-e243-158d-4c8ff5c9ce10@10.31.136.87@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513075.532149:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513075.532166:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665779446144, offset 192 00000400:00000200:18.0:1616513075.532172:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.87@tcp1 00000400:00000200:18.0:1616513075.532180:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.87@tcp1 routed destination 00000400:00000200:18.0:1616513075.532196:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.87@tcp1(10.31.136.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.532203:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.532209:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513075.532210:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.532212:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.532214:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513075.532219:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.532220:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.532225:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513075.532229:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.87@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.532239:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.87@tcp1 of length 224 into portal 26 MB=0x5ffd523eda100 00000400:00000200:16.0:1616513075.532246:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.87@tcp1 of length 224/224 into md 0xf97c6359 [1] + 0 00000400:00000200:16.0:1616513075.532250:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22cc0 00000400:00000200:16.0:1616513075.532253:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.532256:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513075.532262:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22cc0 00000800:00000200:28.0:1616513075.532265:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.532265:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.532268:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.532270:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513075.532295:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.532302:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.532305:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.532307:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513075.532311:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.532315:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdb18 00000800:00000200:22.0:1616513075.532319:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.532344:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665779446016 00010000:00000200:15.0:1616513075.532355:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688665779446016/t0(0) o400->88c9a298-3db2-8b32-b157-320b3591d668@10.31.136.87@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.532369:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665779446016, offset 192 00000400:00000200:15.0:1616513075.532375:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.87@tcp1 00000400:00000200:15.0:1616513075.532384:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.87@tcp1 routed destination 00000400:00000200:15.0:1616513075.532400:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.87@tcp1(10.31.136.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.532407:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.532413:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.532416:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.532420:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.532425:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.532430:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.532432:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513075.532487:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.532495:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.532498:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513075.532500:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.532505:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.532511:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d540 00000800:00000200:12.0:1616513075.532515:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.539597:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.539652:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.539661:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.539670:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.127@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.539679:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.127@tcp1 of length 224 into portal 26 MB=0x5fff36ef23b40 00000400:00000200:14.0:1616513075.539688:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.127@tcp1 of length 224/224 into md 0xf97c6369 [1] + 0 00000400:00000200:14.0:1616513075.539693:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23650 00000400:00000200:14.0:1616513075.539696:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.539699:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513075.539704:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23650 00000800:00000200:10.0:1616513075.539705:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.539707:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.539710:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.539712:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.539769:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795887057728 00010000:00000200:15.0:1616513075.539780:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688795887057728/t0(0) o400->7654731f-8b37-22b5-38a8-8ba60c3ce29c@10.31.132.127@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.539794:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688795887057728, offset 192 00000400:00000200:15.0:1616513075.539800:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.127@tcp1 00000400:00000200:15.0:1616513075.539808:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.127@tcp1 routed destination 00000400:00000200:15.0:1616513075.539823:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.127@tcp1(10.31.132.127@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.539830:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.539836:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.539839:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.539841:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.539846:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.539850:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.539852:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513075.539904:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.539912:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.539915:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.539924:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.539930:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cee0 00000800:00000200:2.0:1616513075.539933:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.539936:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.540048:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513075.540058:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.540066:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513075.540069:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.540075:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.127@tcp1 : PUT - for me 00000400:00000200:10.0:1616513075.540086:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.127@tcp1 of length 224 into portal 12 MB=0x5fff36ef23d40 00000400:00000200:10.0:1616513075.540092:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.127@tcp1 of length 224/224 into md 0xf96450b9 [1] + 51536 00000400:00000200:10.0:1616513075.540097:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.540100:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513075.540106:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.540109:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.540111:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.540172:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795887058240 00010000:00000200:18.0:1616513075.540185:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdda00 x1688795887058240/t0(0) o400->67338913-aa66-3a6e-689c-809a8fe25081@10.31.132.127@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.540200:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688795887058240, offset 192 00000400:00000200:18.0:1616513075.540206:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.127@tcp1 00000400:00000200:18.0:1616513075.540214:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.127@tcp1 routed destination 00000400:00000200:18.0:1616513075.540229:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.127@tcp1(10.31.132.127@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.540235:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.540242:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.540244:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.540246:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.540251:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.540258:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.540313:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.540319:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.540322:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.540329:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.540333:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc550 00000800:00000200:28.0:1616513075.540336:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.540339:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.541147:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513075.541201:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.541211:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513075.541220:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.212@tcp1 : PUT - for me 00000400:00000200:20.0:1616513075.541229:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.212@tcp1 of length 224 into portal 12 MB=0x60182889202c0 00000400:00000200:20.0:1616513075.541235:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.212@tcp1 of length 224/224 into md 0x865b9f31 [1] + 86200 00000400:00000200:20.0:1616513075.541241:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513075.541243:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513075.541250:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.541253:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513075.541255:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.541258:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.541334:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510008910528 00010000:00000200:24.0:1616513075.541347:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07500 x1690510008910528/t0(0) o400->c9438597-703a-8619-a89a-3f835475a770@10.31.128.212@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.541361:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510008910528, offset 192 00000400:00000200:24.0:1616513075.541369:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.212@tcp1 00000400:00000200:24.0:1616513075.541377:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.212@tcp1 routed destination 00000800:00000200:11.2:1616513075.541377:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.541386:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.541392:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513075.541392:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.212@tcp1(10.31.128.212@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.541399:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513075.541400:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.212@tcp1 : PUT - for me 00000800:00000200:16.0:1616513075.541404:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513075.541405:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513075.541408:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.212@tcp1 of length 224 into portal 26 MB=0x601828891fe40 00000800:00000200:24.0:1616513075.541408:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513075.541410:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.541414:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513075.541415:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.212@tcp1 of length 224/224 into md 0xf97c6389 [1] + 0 00000400:00000200:28.0:1616513075.541419:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee226e8 00000400:00000200:28.0:1616513075.541422:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.541424:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:7.2:1616513075.541426:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513075.541429:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee226e8 00000800:00000200:28.0:1616513075.541431:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.541450:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.541453:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513075.541481:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.541488:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.541491:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:15.0:1616513075.541493:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510008909376 00000400:00000200:2.0:1616513075.541497:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.541501:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2330 00010000:00000200:15.0:1616513075.541505:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1690510008909376/t0(0) o400->2741235e-5c80-a4dc-0c3e-79a6c410cda6@10.31.128.212@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513075.541505:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.541508:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.541519:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510008909376, offset 192 00000400:00000200:15.0:1616513075.541525:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.212@tcp1 00000400:00000200:15.0:1616513075.541533:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.212@tcp1 routed destination 00000400:00000200:15.0:1616513075.541548:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.212@tcp1(10.31.128.212@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.541555:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.541561:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.541564:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.541566:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.541571:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.541576:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.541578:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513075.541585:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.541589:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.541592:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.541594:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513075.541598:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.541603:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2df58 00000800:00000200:20.0:1616513075.541607:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.546231:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.546248:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.546257:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.546266:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.546275:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 224 into portal 26 MB=0x5fa07dd43acb4 00000400:00000200:14.0:1616513075.546282:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.110.165@tcp1 of length 224/224 into md 0xf97c63b9 [1] + 0 00000400:00000200:14.0:1616513075.546287:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22f68 00000400:00000200:14.0:1616513075.546290:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.546293:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513075.546298:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22f68 00000800:00000200:14.0:1616513075.546301:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.546304:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.546305:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.546308:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.546365:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567468212 00010000:00000200:15.0:1616513075.546376:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1682286567468212/t0(0) o400->cdc34efc-6553-1602-3933-a65bc0104d4b@10.242.110.165@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.546393:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1682286567468212, offset 192 00000400:00000200:15.0:1616513075.546400:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:15.0:1616513075.546408:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:15.0:1616513075.546424:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.546430:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.546450:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.546453:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.546455:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.546460:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.546464:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.546467:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513075.546517:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.546524:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.546527:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.546534:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.546540:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2df58 00000800:00000200:2.0:1616513075.546544:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.546547:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:5.0:1616513075.548754:0:6494:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924928, portal 4 00000100:00000200:5.0:1616513075.548760:0:6494:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924928, offset 0 00000400:00000200:5.0:1616513075.548767:0:6494:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:5.0:1616513075.548772:0:6494:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:5.0:1616513075.548785:0:6494:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:5.0:1616513075.548791:0:6494:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513075.548797:0:6494:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513075.548800:0:6494:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:5.0:1616513075.548802:0:6494:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513075.548807:0:6494:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513075.548853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.548871:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513075.548879:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513075.548881:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513075.548888:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513075.548895:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8302323f00 x1693067725924928/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513075.548907:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55e110 00000800:00000200:1.0:1616513075.548911:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513075.548914:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513075.548957:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513075.548967:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.548974:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513075.548976:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513075.548982:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513075.548989:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626a40 00000400:00000200:1.0:1616513075.548995:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffa55 [1] + 192 00000400:00000200:1.0:1616513075.549001:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513075.549005:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8302323f00 x1693067725924928/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513075.549019:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513075.549022:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513075.549023:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:5.0:1616513075.549025:0:6494:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8302323f00 x1693067725924928/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:5.0:1616513075.549037:0:6494:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8302323f00 x1693067725924928/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513075.549047:0:6494:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55e7f8 00000800:00000200:11.2:1616513075.556329:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513075.556385:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.556394:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.556404:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.71@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.556413:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.71@tcp1 of length 224 into portal 26 MB=0x5ffd475e1ef80 00000400:00000200:16.0:1616513075.556422:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.71@tcp1 of length 224/224 into md 0xf97c63c9 [1] + 0 00000400:00000200:16.0:1616513075.556426:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23e48 00000400:00000200:16.0:1616513075.556429:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.556432:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513075.556452:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.556452:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23e48 00000800:00000200:16.0:1616513075.556456:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.556459:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.556461:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.556536:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662859444096 00010000:00000200:15.0:1616513075.556546:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688662859444096/t0(0) o400->2eabb0c7-370f-1903-b355-5eb1b4a640b4@10.31.138.71@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.556559:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662859444096, offset 192 00000400:00000200:15.0:1616513075.556565:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.71@tcp1 00000400:00000200:15.0:1616513075.556572:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.71@tcp1 routed destination 00000400:00000200:15.0:1616513075.556587:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.71@tcp1(10.31.138.71@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.556593:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.556599:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.556601:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.556603:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.556608:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.556612:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.556614:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513075.556665:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.556672:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.556675:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513075.556677:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.556684:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.556690:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d540 00000800:00000200:12.0:1616513075.556694:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.556727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.556738:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.556746:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513075.556748:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.556755:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.71@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.556764:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.71@tcp1 of length 224 into portal 12 MB=0x5ffd475e1f140 00000400:00000200:28.0:1616513075.556770:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.71@tcp1 of length 224/224 into md 0x865b9f31 [1] + 86424 00000400:00000200:28.0:1616513075.556775:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.556778:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.556785:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.556788:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.556790:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.556868:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662859444544 00010000:00000200:24.0:1616513075.556881:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf81680 x1688662859444544/t0(0) o400->8888b4e1-e33f-80f3-901a-3650329174f4@10.31.138.71@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.556896:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662859444544, offset 192 00000400:00000200:24.0:1616513075.556902:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.71@tcp1 00000400:00000200:24.0:1616513075.556910:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.71@tcp1 routed destination 00000400:00000200:24.0:1616513075.556925:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.71@tcp1(10.31.138.71@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.556932:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513075.556939:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.556941:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513075.556944:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.556949:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.556957:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.557019:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.557025:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.557028:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513075.557034:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.557038:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:14.0:1616513075.557041:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.557044:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.579836:0:48969:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.579915:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.579935:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.579944:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.202@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.579954:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.202@tcp1 of length 224 into portal 26 MB=0x5ffd42bc8de00 00000400:00000200:2.0:1616513075.579960:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.202@tcp1 of length 224/224 into md 0xf97c63e9 [1] + 0 00000400:00000200:2.0:1616513075.579964:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22990 00000400:00000200:2.0:1616513075.579967:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.579970:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513075.579975:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22990 00000800:00000200:2.0:1616513075.579978:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.579980:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.579980:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.579985:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.580038:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661616287232 00010000:00000200:15.0:1616513075.580048:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688661616287232/t0(0) o400->3c3f4f5a-18f7-cce1-9c3a-a958f749ea46@10.31.128.202@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.580061:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661616287232, offset 192 00000400:00000200:15.0:1616513075.580067:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.202@tcp1 00000400:00000200:15.0:1616513075.580075:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.202@tcp1 routed destination 00000400:00000200:15.0:1616513075.580091:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.202@tcp1(10.31.128.202@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.580097:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.580103:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.580105:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.580107:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.580112:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513075.580118:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513075.580119:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.580132:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.580139:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.580142:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513075.580149:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.580155:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d3a8 00000800:00000200:20.0:1616513075.580159:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.580162:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.580591:0:48969:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.580647:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.580656:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.580665:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.202@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.580675:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.202@tcp1 of length 224 into portal 12 MB=0x5ffd42bc8e280 00000400:00000200:14.0:1616513075.580689:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.202@tcp1 of length 224/224 into md 0xf96450b9 [1] + 51760 00000400:00000200:14.0:1616513075.580695:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.580698:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513075.580701:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.580705:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.580708:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.580710:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.580772:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661616288384 00010000:00000200:18.0:1616513075.580785:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e3a80 x1688661616288384/t0(0) o400->91bc81b1-fcfb-116a-91b9-d2ab8bd224cf@10.31.128.202@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.580800:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661616288384, offset 192 00000400:00000200:18.0:1616513075.580807:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.202@tcp1 00000400:00000200:18.0:1616513075.580815:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.202@tcp1 routed destination 00000400:00000200:18.0:1616513075.580830:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.202@tcp1(10.31.128.202@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.580838:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.580845:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.580847:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.580850:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.580855:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.580859:0:48969:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.580870:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.580876:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.580879:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.580880:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513075.580885:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.580889:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc220 00000800:00000200:22.0:1616513075.580893:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.585567:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513075.585623:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.585632:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513075.585641:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.198@tcp1 : PUT - for me 00000400:00000200:10.0:1616513075.585650:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.198@tcp1 of length 224 into portal 12 MB=0x5ffd4b3471d80 00000400:00000200:10.0:1616513075.585656:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.198@tcp1 of length 224/224 into md 0xf96450b9 [1] + 51984 00000400:00000200:10.0:1616513075.585662:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.585664:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513075.585670:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.585673:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.585675:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.585677:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.585732:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513075.585733:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663889485184 00010000:00000200:18.0:1616513075.585745:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdde80 x1688663889485184/t0(0) o400->1510141a-7c98-9034-68b6-f6d72416a1cf@10.31.136.198@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.585759:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663889485184, offset 192 00000400:00000200:18.0:1616513075.585765:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.198@tcp1 00000400:00000200:18.0:1616513075.585773:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.198@tcp1 routed destination 00000400:00000200:18.0:1616513075.585788:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.198@tcp1(10.31.136.198@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.585795:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.585801:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513075.585801:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.585803:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.585805:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:18.0:1616513075.585810:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513075.585811:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.585820:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.198@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.585829:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.198@tcp1 of length 224 into portal 26 MB=0x5ffd4b3471d00 00000400:00000200:16.0:1616513075.585836:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.198@tcp1 of length 224/224 into md 0xf97c6409 [1] + 0 00000400:00000200:16.0:1616513075.585841:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22dd0 00000400:00000200:16.0:1616513075.585843:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.585846:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513075.585851:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22dd0 00000800:00000200:16.0:1616513075.585854:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513075.585856:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513075.585857:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513075.585862:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513075.585863:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513075.585865:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.585866:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.585873:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.585877:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc6e8 00000800:00000200:28.0:1616513075.585881:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.585930:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663889485056 00010000:00000200:15.0:1616513075.585941:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688663889485056/t0(0) o400->836e154d-b437-3b45-48c9-9940044f3b17@10.31.136.198@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.585955:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663889485056, offset 192 00000400:00000200:15.0:1616513075.585961:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.198@tcp1 00000400:00000200:15.0:1616513075.585970:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.198@tcp1 routed destination 00000400:00000200:15.0:1616513075.585985:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.198@tcp1(10.31.136.198@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.585992:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.585998:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.586001:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.586003:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.586008:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.586013:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.586015:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513075.586024:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.586031:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.586035:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.586044:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.586050:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2df58 00000800:00000200:2.0:1616513075.586054:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.586057:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.597825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.597880:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.597890:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.597899:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.39@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.597908:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.39@tcp1 of length 224 into portal 12 MB=0x6018c03795380 00000400:00000200:12.0:1616513075.597914:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.39@tcp1 of length 224/224 into md 0x865b9f31 [1] + 86648 00000400:00000200:12.0:1616513075.597920:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.597923:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.597930:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.597933:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.597934:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513075.597941:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.597944:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513075.597944:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.597953:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.39@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.597962:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.39@tcp1 of length 224 into portal 26 MB=0x6018c037950c0 00000400:00000200:16.0:1616513075.597969:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.39@tcp1 of length 224/224 into md 0xf97c6439 [1] + 0 00000400:00000200:16.0:1616513075.597974:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22c38 00000400:00000200:16.0:1616513075.597977:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.597979:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513075.597984:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22c38 00000800:00000200:16.0:1616513075.597987:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.597992:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.597994:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.597995:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690550725596032 00010000:00000200:24.0:1616513075.598008:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663a80 x1690550725596032/t0(0) o400->75bb25e1-d59b-e055-f664-b654e57f21ac@10.31.132.39@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.598022:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690550725596032, offset 192 00000400:00000200:24.0:1616513075.598029:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.39@tcp1 00000400:00000200:24.0:1616513075.598036:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.39@tcp1 routed destination 00000100:00000200:15.0:1616513075.598048:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690550725595328 00000400:00000200:24.0:1616513075.598051:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.39@tcp1(10.31.132.39@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.598058:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:15.0:1616513075.598059:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1690550725595328/t0(0) o400->6aa7a68e-b812-b804-0ccf-7e6e17330289@10.31.132.39@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513075.598064:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.598069:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.598072:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.598074:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690550725595328, offset 192 00000800:00000200:24.0:1616513075.598077:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:15.0:1616513075.598080:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.39@tcp1 00000800:00000200:11.2:1616513075.598082:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513075.598088:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.39@tcp1 routed destination 00000800:00000200:20.0:1616513075.598097:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.598104:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:15.0:1616513075.598104:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.39@tcp1(10.31.132.39@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513075.598107:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.598108:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:15.0:1616513075.598110:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513075.598114:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:15.0:1616513075.598117:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513075.598118:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2440 00000800:00000200:20.0:1616513075.598122:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:15.0:1616513075.598122:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.598124:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.598129:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513075.598136:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.598136:0:48971:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.598151:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.598158:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.598161:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.598168:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.598174:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2df58 00000800:00000200:10.0:1616513075.598179:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.598182:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.607149:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.607203:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.607212:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.607221:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.71@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.607231:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.71@tcp1 of length 224 into portal 26 MB=0x5ffd4df7e6a40 00000400:00000200:28.0:1616513075.607238:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.71@tcp1 of length 224/224 into md 0xf97c6449 [1] + 0 00000400:00000200:28.0:1616513075.607243:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22bb0 00000400:00000200:28.0:1616513075.607246:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.607249:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.607253:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22bb0 00000800:00000200:28.0:1616513075.607257:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.607257:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.607259:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.607261:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.607301:0:48972:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513075.607315:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664631306816 00010000:00000200:15.0:1616513075.607325:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688664631306816/t0(0) o400->c4d38827-857c-8f8b-bc53-b4dcd4baf006@10.31.131.71@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.607338:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664631306816, offset 192 00000400:00000200:15.0:1616513075.607344:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.71@tcp1 00000400:00000200:15.0:1616513075.607352:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.71@tcp1 routed destination 00000800:00000200:2.0:1616513075.607358:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:15.0:1616513075.607367:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.71@tcp1(10.31.131.71@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513075.607368:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.607373:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513075.607377:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.71@tcp1 : PUT - for me 00000800:00000200:15.0:1616513075.607379:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.607382:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.607384:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513075.607387:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.71@tcp1 of length 224 into portal 12 MB=0x5ffd4df7e6ac0 00000800:00000200:15.0:1616513075.607389:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.607393:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513075.607393:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.71@tcp1 of length 224/224 into md 0xf96450b9 [1] + 52208 00000100:00000200:15.0:1616513075.607395:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513075.607398:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.607401:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.607403:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513075.607408:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513075.607410:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513075.607411:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.607412:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:12.0:1616513075.607413:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.607417:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513075.607420:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.607426:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7ed0 00000800:00000200:12.0:1616513075.607430:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.607433:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513075.607475:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664631306944 00010000:00000200:18.0:1616513075.607498:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120ec00 x1688664631306944/t0(0) o400->97aa1a42-0786-8fbb-9eb5-00696cbff75c@10.31.131.71@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.607511:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664631306944, offset 192 00000400:00000200:18.0:1616513075.607517:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.71@tcp1 00000400:00000200:18.0:1616513075.607524:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.71@tcp1 routed destination 00000400:00000200:18.0:1616513075.607539:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.71@tcp1(10.31.131.71@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.607545:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.607551:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.607553:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.607555:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.607560:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.607564:0:48972:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.607618:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.607625:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.607628:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.607635:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.607639:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccff0 00000800:00000200:10.0:1616513075.607642:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.607645:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.627513:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513075.627568:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.627577:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.627586:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.47@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.627595:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.47@tcp1 of length 224 into portal 26 MB=0x601838d16a7c0 00000400:00000200:16.0:1616513075.627602:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.47@tcp1 of length 224/224 into md 0xf97c6459 [1] + 0 00000400:00000200:16.0:1616513075.627607:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22550 00000400:00000200:16.0:1616513075.627610:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.627613:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513075.627618:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22550 00000800:00000200:28.0:1616513075.627621:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.627621:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.627623:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.627628:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.627631:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.627632:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.627641:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.47@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.627649:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.47@tcp1 of length 224 into portal 12 MB=0x601838d16a880 00000400:00000200:28.0:1616513075.627657:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.47@tcp1 of length 224/224 into md 0x865b9f31 [1] + 86872 00000400:00000200:28.0:1616513075.627663:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.627665:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.627672:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.627675:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.627677:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.627682:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514379679680 00010000:00000200:15.0:1616513075.627692:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1690514379679680/t0(0) o400->8ac35ec9-05a4-1f00-719d-6316a292db90@10.31.136.47@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.627705:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514379679680, offset 192 00000400:00000200:15.0:1616513075.627712:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.47@tcp1 00000400:00000200:15.0:1616513075.627718:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.47@tcp1 routed destination 00000400:00000200:15.0:1616513075.627733:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.47@tcp1(10.31.136.47@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513075.627738:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514379679872 00000800:00000200:15.0:1616513075.627739:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.627745:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.627748:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.627750:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513075.627751:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf81200 x1690514379679872/t0(0) o400->637aaf14-10ab-b2ff-5590-6ade03e47281@10.31.136.47@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.627754:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.627758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.627761:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513075.627766:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514379679872, offset 192 00000400:00000200:24.0:1616513075.627772:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.47@tcp1 00000400:00000200:24.0:1616513075.627780:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.47@tcp1 routed destination 00000400:00000200:24.0:1616513075.627795:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.47@tcp1(10.31.136.47@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.627802:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513075.627808:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.627811:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513075.627811:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513075.627813:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.627818:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513075.627819:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.627822:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513075.627822:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513075.627825:0:48974:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513075.627829:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.627835:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6000 00000800:00000200:20.0:1616513075.627839:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513075.627880:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.627887:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.627890:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.627896:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.627903:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3210 00000800:00000200:22.0:1616513075.627907:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.627910:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.642229:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513075.642243:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.642253:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513075.642261:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.177@tcp1 : PUT - for me 00000400:00000200:10.0:1616513075.642269:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.177@tcp1 of length 224 into portal 26 MB=0x5ffd465b28ec0 00000400:00000200:10.0:1616513075.642275:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.177@tcp1 of length 224/224 into md 0xf97c6489 [1] + 0 00000400:00000200:10.0:1616513075.642279:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22440 00000400:00000200:10.0:1616513075.642282:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.642284:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513075.642288:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22440 00000800:00000200:10.0:1616513075.642291:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.642294:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.642295:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.642298:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.642353:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662587903680 00010000:00000200:15.0:1616513075.642364:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688662587903680/t0(0) o400->13edffd3-d540-488f-b0c9-643a3de6ef76@10.31.138.177@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.642379:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662587903680, offset 192 00000400:00000200:15.0:1616513075.642385:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.177@tcp1 00000400:00000200:15.0:1616513075.642394:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.177@tcp1 routed destination 00000400:00000200:15.0:1616513075.642409:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.177@tcp1(10.31.138.177@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.642416:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.642422:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.642425:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.642427:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.642432:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.642439:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:15.0:1616513075.642453:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513075.642493:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.642500:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.642504:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513075.642506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513075.642510:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.642517:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6b28 00000800:00000200:12.0:1616513075.642520:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.642523:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513075.642570:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.642579:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.642588:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.227@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.642597:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.227@tcp1 of length 224 into portal 26 MB=0x600b219450a40 00000400:00000200:22.0:1616513075.642604:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.227@tcp1 of length 224/224 into md 0xf97c6499 [1] + 0 00000400:00000200:22.0:1616513075.642609:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22220 00000400:00000200:22.0:1616513075.642612:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.642615:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513075.642620:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22220 00000800:00000200:22.0:1616513075.642623:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.642624:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.642626:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.642629:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.642634:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689614788397632 00000800:00000200:11.2:1616513075.642638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:15.0:1616513075.642641:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1689614788397632/t0(0) o400->15d4229f-99ce-2390-1eb1-354a07eab373@10.31.141.227@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513075.642649:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.642652:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689614788397632, offset 192 00000400:00000200:15.0:1616513075.642658:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.227@tcp1 00000800:00000200:16.0:1616513075.642659:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:15.0:1616513075.642665:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.227@tcp1 routed destination 00000400:00000200:16.0:1616513075.642668:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.227@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.642677:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.227@tcp1 of length 224 into portal 12 MB=0x600b219450ac0 00000400:00000200:15.0:1616513075.642679:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.227@tcp1(10.31.141.227@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513075.642684:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.227@tcp1 of length 224/224 into md 0x865b9f31 [1] + 87096 00000800:00000200:15.0:1616513075.642685:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513075.642689:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.642691:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:15.0:1616513075.642691:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.642693:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.642695:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513075.642698:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:15.0:1616513075.642699:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513075.642701:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513075.642702:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:15.0:1616513075.642703:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.642704:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513075.642707:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513075.642716:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.642723:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.642727:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513075.642727:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513075.642733:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.642739:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6dd0 00000800:00000200:14.0:1616513075.642743:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513075.642763:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689614788397760 00010000:00000200:24.0:1616513075.642776:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5100 x1689614788397760/t0(0) o400->0f8a7420-7590-7c6e-63b6-77b41fc4cc92@10.31.141.227@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.642791:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689614788397760, offset 192 00000400:00000200:24.0:1616513075.642797:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.227@tcp1 00000400:00000200:24.0:1616513075.642805:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.227@tcp1 routed destination 00000400:00000200:24.0:1616513075.642820:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.227@tcp1(10.31.141.227@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.642827:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.642833:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.642836:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.642838:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.642843:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.642849:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.642857:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.642861:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.642863:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.642867:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.642868:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.642871:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3cb0 00000800:00000200:12.0:1616513075.642875:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.643104:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.643116:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.643124:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513075.643126:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513075.643133:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.177@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.643142:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.177@tcp1 of length 224 into portal 12 MB=0x5ffd465b28f80 00000400:00000200:2.0:1616513075.643148:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.177@tcp1 of length 224/224 into md 0xf96450b9 [1] + 52432 00000400:00000200:2.0:1616513075.643154:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.643156:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.643164:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.643167:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.643169:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.643230:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662587903872 00010000:00000200:18.0:1616513075.643243:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120c380 x1688662587903872/t0(0) o400->4d1831ed-5a3a-1b3d-d88a-ee6635449581@10.31.138.177@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.643258:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662587903872, offset 192 00000400:00000200:18.0:1616513075.643264:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.177@tcp1 00000400:00000200:18.0:1616513075.643272:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.177@tcp1 routed destination 00000400:00000200:18.0:1616513075.643287:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.177@tcp1(10.31.138.177@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.643293:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.643299:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.643302:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.643304:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.643309:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.643320:0:48976:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.643329:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.643332:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.643334:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513075.643337:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.643339:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.643341:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc000 00000800:00000200:10.0:1616513075.643344:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.652758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513075.652787:0:48977:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513075.652813:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.652822:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513075.652831:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.184@tcp1 : PUT - for me 00000400:00000200:20.0:1616513075.652840:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.184@tcp1 of length 224 into portal 26 MB=0x604a29b8f1f40 00000800:00000200:22.0:1616513075.652846:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513075.652847:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.184@tcp1 of length 224/224 into md 0xf97c64c9 [1] + 0 00000400:00000200:20.0:1616513075.652851:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23210 00000400:00000200:20.0:1616513075.652854:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513075.652855:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:20.0:1616513075.652857:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513075.652862:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23210 00000400:00000200:22.0:1616513075.652864:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.184@tcp1 : PUT - for me 00000800:00000200:20.0:1616513075.652864:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.652867:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.652869:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513075.652872:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.184@tcp1 of length 224 into portal 12 MB=0x604a29b8f2280 00000800:00000200:20.0:1616513075.652873:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513075.652879:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.184@tcp1 of length 224/224 into md 0xf96450b9 [1] + 52656 00000400:00000200:22.0:1616513075.652884:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.652887:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513075.652893:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.652896:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.652898:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.652901:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.652932:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946301325120 00010000:00000200:15.0:1616513075.652943:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1693946301325120/t0(0) o400->c9044cd1-dd0e-ec51-16fe-408cf3ffe4ce@10.31.140.184@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.652957:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946301325120, offset 192 00000100:00000200:18.0:1616513075.652959:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946301325952 00000400:00000200:15.0:1616513075.652965:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.184@tcp1 00010000:00000200:18.0:1616513075.652972:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2ba80 x1693946301325952/t0(0) o400->451aeec2-6245-d862-a767-0817d6d225eb@10.31.140.184@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513075.652973:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.184@tcp1 routed destination 00000100:00000200:18.0:1616513075.652987:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946301325952, offset 192 00000400:00000200:15.0:1616513075.652989:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.184@tcp1(10.31.140.184@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513075.652993:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.184@tcp1 00000800:00000200:15.0:1616513075.652996:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513075.653001:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.184@tcp1 routed destination 00000800:00000200:15.0:1616513075.653002:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.653005:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.653007:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.653012:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513075.653016:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.184@tcp1(10.31.140.184@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513075.653017:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.653019:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513075.653023:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.653029:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.653032:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.653034:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.653039:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.653043:0:48977:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513075.653070:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.653077:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.653081:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.653088:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.653094:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7320 00000800:00000200:28.0:1616513075.653098:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.653101:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513075.653101:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.653107:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.653110:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513075.653116:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.653120:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc7f8 00000800:00000200:14.0:1616513075.653124:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.653127:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.682724:0:48980:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513075.682780:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.682790:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.682800:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.70@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.682810:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.70@tcp1 of length 224 into portal 26 MB=0x5f91cc4512100 00000400:00000200:22.0:1616513075.682816:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.70@tcp1 of length 224/224 into md 0xf97c64d9 [1] + 0 00000400:00000200:22.0:1616513075.682821:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23ba0 00000400:00000200:22.0:1616513075.682827:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.682830:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513075.682834:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23ba0 00000800:00000200:2.0:1616513075.682836:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.682838:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.682840:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513075.682843:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.682846:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.682847:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.682856:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.70@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.682865:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.70@tcp1 of length 224 into portal 12 MB=0x5f91cc4512180 00000400:00000200:2.0:1616513075.682871:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.70@tcp1 of length 224/224 into md 0xf96450b9 [1] + 52880 00000400:00000200:2.0:1616513075.682877:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.682881:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.682888:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.682892:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.682894:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.682896:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681276831604992 00010000:00000200:15.0:1616513075.682907:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1681276831604992/t0(0) o400->443d4b64-a8a7-f754-2a7f-5b389b860d7c@10.31.131.70@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.682920:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681276831604992, offset 192 00000400:00000200:15.0:1616513075.682925:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.70@tcp1 00000400:00000200:15.0:1616513075.682932:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.70@tcp1 routed destination 00000400:00000200:15.0:1616513075.682947:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.70@tcp1(10.31.131.70@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.682953:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.682959:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:18.0:1616513075.682960:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681276831605120 00000800:00000200:15.0:1616513075.682962:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.682964:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.682968:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:18.0:1616513075.682973:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461209f80 x1681276831605120/t0(0) o400->100b477c-a541-2557-fce4-33cf82ec3836@10.31.131.70@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513075.682975:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513075.682975:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:18.0:1616513075.682988:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681276831605120, offset 192 00000800:00000200:16.0:1616513075.682988:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513075.682995:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.70@tcp1 00000800:00000200:16.0:1616513075.682995:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.682998:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513075.683002:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.70@tcp1 routed destination 00000400:00000200:16.0:1616513075.683004:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.683010:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6908 00000800:00000200:16.0:1616513075.683015:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.683018:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513075.683019:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.70@tcp1(10.31.131.70@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.683026:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.683032:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.683035:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.683037:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.683043:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.683048:0:48980:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.683101:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.683108:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.683111:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513075.683112:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.683118:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.683122:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd6d8 00000800:00000200:10.0:1616513075.683125:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.683562:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.683636:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.683645:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.683654:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.197@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.683664:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.197@tcp1 of length 224 into portal 26 MB=0x5ffd49f6eed80 00000400:00000200:28.0:1616513075.683671:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.197@tcp1 of length 224/224 into md 0xf97c64e9 [1] + 0 00000400:00000200:28.0:1616513075.683676:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d760 00000400:00000200:28.0:1616513075.683679:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.683681:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.683687:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d760 00000800:00000200:12.0:1616513075.683689:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.683690:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.683693:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513075.683696:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.683699:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.683700:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:15.0:1616513075.683702:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663556550016 00000400:00000200:12.0:1616513075.683708:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.197@tcp1 : PUT - for me 00010000:00000200:15.0:1616513075.683709:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688663556550016/t0(0) o400->1369c49a-76ef-5d7f-5485-4ccd2841c928@10.31.136.197@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513075.683718:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.197@tcp1 of length 224 into portal 12 MB=0x5ffd49f6eee80 00000100:00000200:15.0:1616513075.683721:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663556550016, offset 192 00000400:00000200:12.0:1616513075.683724:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.197@tcp1 of length 224/224 into md 0x865b9f31 [1] + 87320 00000400:00000200:15.0:1616513075.683726:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.197@tcp1 00000400:00000200:12.0:1616513075.683730:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.683732:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:15.0:1616513075.683733:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.197@tcp1 routed destination 00000400:00000200:15.0:1616513075.683747:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.197@tcp1(10.31.136.197@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513075.683756:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.683757:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513075.683759:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.683761:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:15.0:1616513075.683762:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.683764:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.683766:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.683769:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513075.683774:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513075.683774:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.683785:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.683792:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.683795:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513075.683795:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513075.683802:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:24.0:1616513075.683804:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663556550272 00000400:00000200:20.0:1616513075.683808:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6880 00000800:00000200:20.0:1616513075.683812:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:24.0:1616513075.683816:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661200 x1688663556550272/t0(0) o400->e7212e28-8e4c-cdc4-bbfd-91148bfd023d@10.31.136.197@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.683841:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663556550272, offset 192 00000400:00000200:24.0:1616513075.683848:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.197@tcp1 00000400:00000200:24.0:1616513075.683855:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.197@tcp1 routed destination 00000400:00000200:24.0:1616513075.683870:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.197@tcp1(10.31.136.197@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.683877:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513075.683883:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.683886:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513075.683888:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.683893:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.683900:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.683913:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.683920:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.683923:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.683924:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513075.683930:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.683934:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3f58 00000800:00000200:14.0:1616513075.683937:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.684291:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513075.684301:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.684307:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513075.684308:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.684314:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.684315:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.49@tcp1 : PUT - for me 00000800:00000200:28.0:1616513075.684316:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.684321:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.49@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.684324:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.49@tcp1 of length 224 into portal 26 MB=0x60182e8d13d80 00000400:00000200:12.0:1616513075.684328:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.49@tcp1 of length 224 into portal 12 MB=0x60182e8d13f80 00000400:00000200:16.0:1616513075.684331:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.49@tcp1 of length 224/224 into md 0xf97c64f9 [1] + 0 00000400:00000200:12.0:1616513075.684333:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.49@tcp1 of length 224/224 into md 0x865b9f31 [1] + 87544 00000400:00000200:16.0:1616513075.684336:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2cc38 00000400:00000200:12.0:1616513075.684337:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513075.684338:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.684339:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:16.0:1616513075.684341:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513075.684344:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513075.684346:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cc38 00000800:00000200:12.0:1616513075.684346:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.684348:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.684349:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.684351:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.684353:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.684356:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511623667584 00000100:00000200:15.0:1616513075.684359:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511623667072 00010000:00000200:24.0:1616513075.684363:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c662d00 x1690511623667584/t0(0) o400->fd46d34f-63fa-e183-ffa3-f440ae7547b1@10.31.136.49@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:15.0:1616513075.684365:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1690511623667072/t0(0) o400->6410c710-6517-c39e-5d55-3763f2bc4cc0@10.31.136.49@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513075.684375:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511623667584, offset 192 00000100:00000200:15.0:1616513075.684376:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511623667072, offset 192 00000400:00000200:24.0:1616513075.684381:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.49@tcp1 00000400:00000200:15.0:1616513075.684382:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.49@tcp1 00000400:00000200:24.0:1616513075.684388:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.49@tcp1 routed destination 00000400:00000200:15.0:1616513075.684388:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.49@tcp1 routed destination 00000400:00000200:24.0:1616513075.684404:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.49@tcp1(10.31.136.49@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:15.0:1616513075.684404:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.49@tcp1(10.31.136.49@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.684410:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.684411:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513075.684416:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.684416:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.684418:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.684419:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513075.684420:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.684421:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.684424:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:15.0:1616513075.684425:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513075.684428:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.684429:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.684429:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513075.684452:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.684456:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.684458:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.684460:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513075.684465:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.684468:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf26e8 00000800:00000200:20.0:1616513075.684470:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513075.684484:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.684491:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.684494:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513075.684495:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513075.684501:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.684507:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7d38 00000800:00000200:22.0:1616513075.684511:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.685038:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.685051:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.685057:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513075.685060:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513075.685064:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.12@tcp1 : PUT - for me 00000800:00000200:11.2:1616513075.685076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513075.685076:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.12@tcp1 of length 224 into portal 12 MB=0x5ffe6340da680 00000400:00000200:2.0:1616513075.685082:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.12@tcp1 of length 224/224 into md 0xf96450b9 [1] + 53104 00000800:00000200:12.0:1616513075.685085:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513075.685086:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.685088:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.685090:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513075.685092:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513075.685093:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513075.685094:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.685096:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513075.685097:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.12@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.685103:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.12@tcp1 of length 224 into portal 26 MB=0x5ffe6340da600 00000400:00000200:12.0:1616513075.685109:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.12@tcp1 of length 224/224 into md 0xf97c6519 [1] + 0 00000400:00000200:12.0:1616513075.685113:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2db18 00000400:00000200:12.0:1616513075.685115:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.685118:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513075.685122:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2db18 00000800:00000200:12.0:1616513075.685125:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.685127:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.685130:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.685135:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739064423936 00010000:00000200:15.0:1616513075.685141:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688739064423936/t0(0) o400->7d9dc319-b8e2-df55-931d-1f20d04e0900@10.31.142.12@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.685151:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688739064423936, offset 192 00000400:00000200:15.0:1616513075.685156:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.12@tcp1 00000100:00000200:18.0:1616513075.685157:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739064424064 00000400:00000200:15.0:1616513075.685162:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.12@tcp1 routed destination 00010000:00000200:18.0:1616513075.685169:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120b600 x1688739064424064/t0(0) o400->f3920f93-4859-5578-ea76-8fb20659c4ba@10.31.142.12@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513075.685175:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.12@tcp1(10.31.142.12@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.685181:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:18.0:1616513075.685184:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688739064424064, offset 192 00000800:00000200:15.0:1616513075.685186:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.685188:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513075.685191:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.12@tcp1 00000800:00000200:15.0:1616513075.685191:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.685194:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513075.685198:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.12@tcp1 routed destination 00000800:00000200:11.2:1616513075.685198:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.685199:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513075.685207:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.685211:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513075.685213:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.12@tcp1(10.31.142.12@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513075.685214:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513075.685215:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.685220:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513075.685220:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.685226:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513075.685227:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7dc0 00000800:00000200:18.0:1616513075.685229:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513075.685230:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513075.685231:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.685235:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.685240:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.685247:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.685251:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.685253:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513075.685256:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.685258:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.685261:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cda08 00000800:00000200:10.0:1616513075.685264:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.695891:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.695945:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.695955:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.695964:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.113@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.695973:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.113@tcp1 of length 224 into portal 26 MB=0x5ffddf5ceddc0 00000400:00000200:14.0:1616513075.695980:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.113@tcp1 of length 224/224 into md 0xf97c6559 [1] + 0 00000400:00000200:14.0:1616513075.695985:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c5d8 00000400:00000200:14.0:1616513075.695989:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.695993:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513075.695997:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c5d8 00000800:00000200:2.0:1616513075.695999:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.696000:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.696003:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513075.696006:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.696009:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.696010:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.696018:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.113@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.696027:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.113@tcp1 of length 224 into portal 12 MB=0x5ffddf5cedf00 00000400:00000200:2.0:1616513075.696033:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.113@tcp1 of length 224/224 into md 0xf96450b9 [1] + 53328 00000400:00000200:2.0:1616513075.696038:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.696041:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.696047:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.696050:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.696051:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.696062:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703660383680 00010000:00000200:15.0:1616513075.696073:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688703660383680/t0(0) o400->77993f20-f0d4-632d-49ca-88b448a811ae@10.31.141.113@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.696088:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703660383680, offset 192 00000400:00000200:15.0:1616513075.696094:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.113@tcp1 00000400:00000200:15.0:1616513075.696102:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.113@tcp1 routed destination 00000100:00000200:18.0:1616513075.696111:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703660384000 00000400:00000200:15.0:1616513075.696117:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.113@tcp1(10.31.141.113@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:18.0:1616513075.696123:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461208480 x1688703660384000/t0(0) o400->1c06c049-9ec4-2e85-9242-1295b6059f96@10.31.141.113@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.696124:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.696130:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.696133:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.696135:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:18.0:1616513075.696139:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703660384000, offset 192 00000800:00000200:15.0:1616513075.696140:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513075.696145:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.113@tcp1 00000800:00000200:11.2:1616513075.696146:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.696147:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513075.696152:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.113@tcp1 routed destination 00000400:00000200:18.0:1616513075.696167:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.113@tcp1(10.31.141.113@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.696174:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.696180:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.696182:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.696186:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.696191:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.696196:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513075.696207:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.696214:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.696217:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513075.696224:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.696230:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7c28 00000800:00000200:16.0:1616513075.696234:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.696237:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513075.696266:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.696273:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.696277:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.696278:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.696284:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.696289:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd8f8 00000800:00000200:10.0:1616513075.696292:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.705147:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.705202:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.705212:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.705221:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.104@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.705229:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.104@tcp1 of length 224 into portal 26 MB=0x5ffd3f1e53ec0 00000400:00000200:2.0:1616513075.705236:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.104@tcp1 of length 224/224 into md 0xf97c6569 [1] + 0 00000400:00000200:2.0:1616513075.705241:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c880 00000400:00000200:2.0:1616513075.705244:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.705259:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513075.705261:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513075.705264:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c880 00000800:00000200:2.0:1616513075.705267:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.705269:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.705271:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.705327:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660645068480 00010000:00000200:15.0:1616513075.705337:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688660645068480/t0(0) o400->49a3c1fb-5825-2ede-60da-9f404b270f57@10.31.128.104@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.705350:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660645068480, offset 192 00000400:00000200:15.0:1616513075.705356:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.104@tcp1 00000400:00000200:15.0:1616513075.705364:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.104@tcp1 routed destination 00000400:00000200:15.0:1616513075.705378:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.104@tcp1(10.31.128.104@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.705384:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.705390:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.705393:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.705395:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.705399:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513075.705406:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513075.705406:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.705418:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.705424:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.705427:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513075.705445:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.705450:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6d48 00000800:00000200:20.0:1616513075.705454:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.705456:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.705494:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.705546:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.705554:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.705562:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.104@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.705570:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.104@tcp1 of length 224 into portal 12 MB=0x5ffd3f1e54300 00000400:00000200:14.0:1616513075.705576:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.104@tcp1 of length 224/224 into md 0xf96450b9 [1] + 53552 00000400:00000200:14.0:1616513075.705581:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.705583:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.705589:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.705592:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.705594:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.705597:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.705651:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660645069568 00010000:00000200:18.0:1616513075.705662:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e3f00 x1688660645069568/t0(0) o400->b9ba1ec4-319b-ee13-0947-c54dd380a0ba@10.31.128.104@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.705676:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660645069568, offset 192 00000400:00000200:18.0:1616513075.705682:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.104@tcp1 00000400:00000200:18.0:1616513075.705689:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.104@tcp1 routed destination 00000400:00000200:18.0:1616513075.705703:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.104@tcp1(10.31.128.104@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.705709:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.705715:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.705717:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.705719:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.705723:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.705728:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.705738:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.705744:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.705746:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.705747:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513075.705753:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.705757:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd870 00000800:00000200:22.0:1616513075.705761:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.706034:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513075.706045:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.706054:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513075.706056:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.706063:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.199@tcp1 : PUT - for me 00000400:00000200:10.0:1616513075.706072:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.199@tcp1 of length 224 into portal 26 MB=0x5ffd495ee2f80 00000400:00000200:10.0:1616513075.706078:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.199@tcp1 of length 224/224 into md 0xf97c6599 [1] + 0 00000400:00000200:10.0:1616513075.706082:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d100 00000400:00000200:10.0:1616513075.706085:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.706088:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513075.706093:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d100 00000800:00000200:10.0:1616513075.706096:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.706098:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.706100:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.706106:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663397117824 00010000:00000200:15.0:1616513075.706111:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688663397117824/t0(0) o400->a4363236-3b97-9167-716f-677f45ea11d2@10.31.136.199@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.706124:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663397117824, offset 192 00000400:00000200:15.0:1616513075.706129:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.199@tcp1 00000400:00000200:15.0:1616513075.706136:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.199@tcp1 routed destination 00000400:00000200:15.0:1616513075.706148:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.199@tcp1(10.31.136.199@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.706154:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.706159:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.706161:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.706163:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.706166:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513075.706171:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513075.706173:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513075.706183:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.706191:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.706197:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513075.706199:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513075.706203:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.199@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.706210:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.199@tcp1 of length 224 into portal 12 MB=0x5ffd495ee3080 00000400:00000200:2.0:1616513075.706215:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.199@tcp1 of length 224/224 into md 0xf96450b9 [1] + 53776 00000400:00000200:2.0:1616513075.706219:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.706221:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513075.706226:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513075.706226:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.706228:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.706230:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513075.706232:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.706235:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:18.0:1616513075.706239:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663397118080 00000400:00000200:16.0:1616513075.706241:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00010000:00000200:18.0:1616513075.706245:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120cc80 x1688663397118080/t0(0) o400->a430b6a3-6865-2002-aacf-4c8ec4596d3d@10.31.136.199@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513075.706247:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7100 00000800:00000200:16.0:1616513075.706251:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.706253:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:18.0:1616513075.706256:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663397118080, offset 192 00000800:00000200:16.0:1616513075.706259:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513075.706261:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513075.706261:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.199@tcp1 00000800:00000200:16.0:1616513075.706261:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.706263:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.706265:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513075.706267:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.199@tcp1 routed destination 00000400:00000200:18.0:1616513075.706279:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.199@tcp1(10.31.136.199@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.706284:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.706289:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.706291:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.706292:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.706295:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.706300:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.706307:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.706311:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.706313:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513075.706315:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513075.706318:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.706321:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd7e8 00000800:00000200:14.0:1616513075.706324:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.750062:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513075.750120:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.750130:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.750140:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.16@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.750151:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.16@tcp1 of length 224 into portal 26 MB=0x5ffe6929529c0 00000400:00000200:22.0:1616513075.750158:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.16@tcp1 of length 224/224 into md 0xf97c65b9 [1] + 0 00000400:00000200:22.0:1616513075.750163:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d980 00000400:00000200:22.0:1616513075.750166:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.750169:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513075.750174:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d980 00000800:00000200:2.0:1616513075.750176:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.750178:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.750181:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513075.750186:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.750188:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513075.750189:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513075.750197:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.16@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.750206:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.16@tcp1 of length 224 into portal 12 MB=0x5ffe692952c00 00000400:00000200:2.0:1616513075.750213:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.16@tcp1 of length 224/224 into md 0xf96450b9 [1] + 54000 00000400:00000200:2.0:1616513075.750219:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.750221:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.750229:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.750232:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.750234:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.750239:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688740650363328 00010000:00000200:15.0:1616513075.750250:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688740650363328/t0(0) o400->81222762-8adf-9954-9ca0-8c37316f04cb@10.31.142.16@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.750264:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688740650363328, offset 192 00000400:00000200:15.0:1616513075.750270:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.16@tcp1 00000400:00000200:15.0:1616513075.750279:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.16@tcp1 routed destination 00000100:00000200:18.0:1616513075.750299:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688740650363904 00000400:00000200:15.0:1616513075.750307:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.16@tcp1(10.31.142.16@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:18.0:1616513075.750313:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461208d80 x1688740650363904/t0(0) o400->fabd2be5-ffc3-86fb-dbdc-50ae27efdd5c@10.31.142.16@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.750313:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.750319:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.750322:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.750324:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.750328:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:18.0:1616513075.750329:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688740650363904, offset 192 00000800:00000200:11.2:1616513075.750334:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.750335:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513075.750337:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.16@tcp1 00000400:00000200:18.0:1616513075.750346:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.16@tcp1 routed destination 00000400:00000200:18.0:1616513075.750361:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.16@tcp1(10.31.142.16@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.750368:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.750374:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.750377:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.750380:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.750387:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513075.750390:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513075.750391:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513075.750398:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.750402:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513075.750403:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.750408:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.750415:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6110 00000800:00000200:28.0:1616513075.750418:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513075.750454:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.750461:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.750463:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513075.750465:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513075.750470:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.750474:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccf68 00000800:00000200:14.0:1616513075.750478:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.755454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513075.755515:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.755524:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.755535:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.203@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.755544:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.203@tcp1 of length 224 into portal 26 MB=0x605a072d0dfc0 00000400:00000200:16.0:1616513075.755552:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.203@tcp1 of length 224/224 into md 0xf97c65c9 [1] + 0 00000400:00000200:16.0:1616513075.755557:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2da08 00000400:00000200:16.0:1616513075.755560:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.755563:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513075.755567:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2da08 00000800:00000200:12.0:1616513075.755568:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.755571:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.755573:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.755575:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.755631:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036539461568 00010000:00000200:15.0:1616513075.755642:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1695036539461568/t0(0) o400->a3fb901f-567b-6e00-427a-de4c5f286dd9@10.31.129.203@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.755656:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036539461568, offset 192 00000400:00000200:15.0:1616513075.755663:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.203@tcp1 00000400:00000200:15.0:1616513075.755670:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.203@tcp1 routed destination 00000400:00000200:15.0:1616513075.755688:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.203@tcp1(10.31.129.203@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.755695:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.755701:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.755704:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.755706:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.755711:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.755716:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.755718:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513075.755726:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.755734:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.755737:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513075.755744:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.755750:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6220 00000800:00000200:20.0:1616513075.755754:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.755757:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.755930:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.755941:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.755948:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513075.755951:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.755957:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.203@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.755965:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.203@tcp1 of length 224 into portal 12 MB=0x605a072d0e080 00000400:00000200:12.0:1616513075.755971:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.203@tcp1 of length 224/224 into md 0x865b9f31 [1] + 87768 00000400:00000200:12.0:1616513075.755976:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.755978:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.755984:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.755987:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.755989:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.756051:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036539461760 00010000:00000200:24.0:1616513075.756065:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c660480 x1695036539461760/t0(0) o400->8f872a88-24a6-520f-414c-100a59a33b8f@10.31.129.203@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.756079:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036539461760, offset 192 00000400:00000200:24.0:1616513075.756085:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.203@tcp1 00000400:00000200:24.0:1616513075.756093:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.203@tcp1 routed destination 00000400:00000200:24.0:1616513075.756112:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.203@tcp1(10.31.129.203@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.756120:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513075.756127:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.756129:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513075.756131:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.756136:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.756142:0:48988:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.756197:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.756204:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.756207:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.756214:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.756217:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf34b8 00000800:00000200:10.0:1616513075.756221:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.756224:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.769935:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513075.769990:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.769999:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.770009:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.149@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.770019:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.149@tcp1 of length 224 into portal 12 MB=0x60595331d1fc0 00000400:00000200:22.0:1616513075.770025:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.149@tcp1 of length 224/224 into md 0xf96450b9 [1] + 54224 00000400:00000200:22.0:1616513075.770031:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.770034:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513075.770041:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.770044:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.770044:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.770048:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.770100:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513075.770105:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694988226076608 00010000:00000200:18.0:1616513075.770117:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2d580 x1694988226076608/t0(0) o400->599f600b-6c09-7ee5-2b9d-dd2b3eb84365@10.31.143.149@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.770131:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694988226076608, offset 192 00000400:00000200:18.0:1616513075.770137:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.149@tcp1 00000400:00000200:18.0:1616513075.770144:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.149@tcp1 routed destination 00000400:00000200:18.0:1616513075.770159:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.149@tcp1(10.31.143.149@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.770165:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513075.770167:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.770170:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.770173:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513075.770175:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513075.770177:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.770179:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:20.0:1616513075.770185:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.149@tcp1 : PUT - for me 00000400:00000200:20.0:1616513075.770194:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.149@tcp1 of length 224 into portal 26 MB=0x60595331d1d40 00000400:00000200:20.0:1616513075.770201:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.149@tcp1 of length 224/224 into md 0xf97c65e9 [1] + 0 00000400:00000200:20.0:1616513075.770206:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d210 00000400:00000200:20.0:1616513075.770209:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513075.770212:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513075.770217:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d210 00000800:00000200:20.0:1616513075.770220:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513075.770221:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513075.770223:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513075.770228:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513075.770230:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513075.770231:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.770232:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.770239:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.770243:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd100 00000800:00000200:28.0:1616513075.770247:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.770283:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694988226075968 00010000:00000200:15.0:1616513075.770294:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1694988226075968/t0(0) o400->8504c03a-8b15-9f0b-0198-0985d42ad8ad@10.31.143.149@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.770308:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694988226075968, offset 192 00000400:00000200:15.0:1616513075.770315:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.149@tcp1 00000400:00000200:15.0:1616513075.770323:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.149@tcp1 routed destination 00000400:00000200:15.0:1616513075.770339:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.149@tcp1(10.31.143.149@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.770345:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.770352:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.770355:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.770357:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.770362:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.770366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.770368:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513075.770419:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.770426:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.770430:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.770431:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.770451:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.770457:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7650 00000800:00000200:10.0:1616513075.770461:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.779574:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.779629:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.779641:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.779650:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.128@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.779660:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.128@tcp1 of length 224 into portal 12 MB=0x604a2e5e2cb40 00000800:00000200:11.2:1616513075.779663:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513075.779666:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.128@tcp1 of length 224/224 into md 0xf96450b9 [1] + 54448 00000400:00000200:14.0:1616513075.779671:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.779674:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.779681:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.779684:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.779685:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.779689:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513075.779719:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.779728:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.779737:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.128@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.779746:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.128@tcp1 of length 224 into portal 26 MB=0x604a2e5e2c900 00000100:00000200:18.0:1616513075.779753:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947548322624 00000400:00000200:12.0:1616513075.779753:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.128@tcp1 of length 224/224 into md 0xf97c6629 [1] + 0 00000400:00000200:12.0:1616513075.779758:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d6d8 00000400:00000200:12.0:1616513075.779761:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:18.0:1616513075.779764:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e4380 x1693947548322624/t0(0) o400->189913e4-4f95-6873-f225-3147ab92e101@10.31.140.128@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513075.779764:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513075.779768:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d6d8 00000800:00000200:12.0:1616513075.779771:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.779773:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.779774:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:18.0:1616513075.779778:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693947548322624, offset 192 00000800:00000200:12.0:1616513075.779779:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513075.779784:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.128@tcp1 00000400:00000200:18.0:1616513075.779791:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.128@tcp1 routed destination 00000400:00000200:18.0:1616513075.779805:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.128@tcp1(10.31.140.128@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.779811:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.779817:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.779821:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.779823:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.779828:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513075.779832:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947548322048 00000800:00000200:11.2:1616513075.779832:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:15.0:1616513075.779842:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1693947548322048/t0(0) o400->d4a064b0-8b5f-fab4-b615-7c5ecc46c8cd@10.31.140.128@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.779855:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693947548322048, offset 192 00000400:00000200:15.0:1616513075.779861:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.128@tcp1 00000400:00000200:15.0:1616513075.779869:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.128@tcp1 routed destination 00000400:00000200:15.0:1616513075.779884:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.128@tcp1(10.31.140.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513075.779885:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:15.0:1616513075.779890:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513075.779892:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.779895:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:15.0:1616513075.779895:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.779900:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513075.779901:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:15.0:1616513075.779902:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513075.779905:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd320 00000800:00000200:15.0:1616513075.779907:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513075.779909:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513075.779911:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513075.779912:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.779913:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513075.779923:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.779930:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.779934:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513075.779940:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.779947:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c73a8 00000800:00000200:2.0:1616513075.779950:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.779955:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.781220:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513075.781273:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.781282:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513075.781291:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.187@tcp1 : PUT - for me 00000400:00000200:20.0:1616513075.781301:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.187@tcp1 of length 224 into portal 26 MB=0x6018cd2115e00 00000400:00000200:20.0:1616513075.781307:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.187@tcp1 of length 224/224 into md 0xf97c6639 [1] + 0 00000400:00000200:20.0:1616513075.781312:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d5c8 00000400:00000200:20.0:1616513075.781315:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513075.781318:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513075.781322:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d5c8 00000800:00000200:20.0:1616513075.781325:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.781328:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.781329:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513075.781333:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.781384:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690554191666688 00010000:00000200:15.0:1616513075.781395:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1690554191666688/t0(0) o400->ed275dca-343b-aac1-7906-aad235cf0c21@10.31.128.187@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.781408:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690554191666688, offset 192 00000400:00000200:15.0:1616513075.781414:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.187@tcp1 00000400:00000200:15.0:1616513075.781421:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.187@tcp1 routed destination 00000400:00000200:15.0:1616513075.781448:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.187@tcp1(10.31.128.187@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.781454:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.781460:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.781463:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.781465:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.781469:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.781473:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.781476:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513075.781526:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.781533:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.781536:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.781543:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.781549:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6550 00000800:00000200:28.0:1616513075.781552:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.781555:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.781561:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513075.781563:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.781569:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.187@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.781577:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.187@tcp1 of length 224 into portal 12 MB=0x6018cd2116140 00000400:00000200:28.0:1616513075.781583:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.187@tcp1 of length 224/224 into md 0x865b9f31 [1] + 87992 00000400:00000200:28.0:1616513075.781589:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.781591:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.781599:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.781601:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.781603:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.781664:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690554191667520 00010000:00000200:24.0:1616513075.781676:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf82400 x1690554191667520/t0(0) o400->08fb0456-8a31-b0d1-dfb2-6a0a2c6eac07@10.31.128.187@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.781690:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690554191667520, offset 192 00000400:00000200:24.0:1616513075.781697:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.187@tcp1 00000400:00000200:24.0:1616513075.781704:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.187@tcp1 routed destination 00000400:00000200:24.0:1616513075.781719:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.187@tcp1(10.31.128.187@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.781726:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513075.781732:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.781735:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513075.781737:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.781742:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.781748:0:48991:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.781803:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.781809:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.781812:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.781819:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.781823:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3760 00000800:00000200:22.0:1616513075.781827:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.781830:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.789830:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513075.789886:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.789895:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513075.789904:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.182@tcp1 : PUT - for me 00000400:00000200:20.0:1616513075.789913:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.182@tcp1 of length 224 into portal 26 MB=0x6058f2f3af440 00000400:00000200:20.0:1616513075.789920:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.182@tcp1 of length 224/224 into md 0xf97c6659 [1] + 0 00000400:00000200:20.0:1616513075.789925:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c4c8 00000400:00000200:20.0:1616513075.789928:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513075.789931:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513075.789936:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c4c8 00000800:00000200:12.0:1616513075.789938:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.789939:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.789942:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513075.789944:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.789998:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694962391118912 00010000:00000200:15.0:1616513075.790009:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1694962391118912/t0(0) o400->a42241de-c30b-2477-e0ef-68fd339fd562@10.31.129.182@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.790022:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694962391118912, offset 192 00000400:00000200:15.0:1616513075.790028:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.182@tcp1 00000400:00000200:15.0:1616513075.790036:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.182@tcp1 routed destination 00000400:00000200:15.0:1616513075.790050:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.182@tcp1(10.31.129.182@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.790057:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.790062:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.790065:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.790067:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.790072:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.790076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.790078:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513075.790128:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.790135:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.790139:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513075.790146:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.790152:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c78f8 00000800:00000200:16.0:1616513075.790156:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.790159:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.791171:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.791226:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.791236:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.791245:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.182@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.791253:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.182@tcp1 of length 224 into portal 12 MB=0x6058f2f3af4c0 00000400:00000200:2.0:1616513075.791259:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.182@tcp1 of length 224/224 into md 0xf96450b9 [1] + 54672 00000400:00000200:2.0:1616513075.791265:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.791267:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.791274:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.791277:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.791279:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.791281:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.791338:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694962391119040 00010000:00000200:18.0:1616513075.791350:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120ad00 x1694962391119040/t0(0) o400->158fdebb-6f6a-5b4a-3fd6-84332087fa9d@10.31.129.182@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.791365:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694962391119040, offset 192 00000400:00000200:18.0:1616513075.791371:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.182@tcp1 00000400:00000200:18.0:1616513075.791379:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.182@tcp1 routed destination 00000400:00000200:18.0:1616513075.791406:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.182@tcp1(10.31.129.182@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.791412:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.791417:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.791420:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.791422:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.791426:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.791430:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.791482:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.791490:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.791492:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.791499:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.791503:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc440 00000800:00000200:22.0:1616513075.791507:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.791510:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.807179:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.807233:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.807242:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.807251:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.152@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.807260:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.152@tcp1 of length 224 into portal 26 MB=0x601866a9c39c0 00000400:00000200:12.0:1616513075.807267:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.152@tcp1 of length 224/224 into md 0xf97c6669 [1] + 0 00000400:00000200:12.0:1616513075.807274:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2dd38 00000400:00000200:12.0:1616513075.807277:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.807279:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513075.807284:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2dd38 00000800:00000200:20.0:1616513075.807286:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.807287:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.807290:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.807292:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.807351:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690526686132672 00010000:00000200:15.0:1616513075.807362:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1690526686132672/t0(0) o400->a5f3a17d-e2e0-60b7-a92b-af949524d3f3@10.31.141.152@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.807376:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690526686132672, offset 192 00000400:00000200:15.0:1616513075.807383:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.152@tcp1 00000400:00000200:15.0:1616513075.807391:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.152@tcp1 routed destination 00000400:00000200:15.0:1616513075.807416:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.152@tcp1(10.31.141.152@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.807422:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.807428:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.807430:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.807432:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.807449:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.807454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.807457:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513075.807515:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.807523:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.807526:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.807531:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.807533:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.807539:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6aa0 00000800:00000200:16.0:1616513075.807543:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.807556:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.807612:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.807622:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513075.807623:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513075.807631:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.152@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.807639:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.152@tcp1 of length 224 into portal 12 MB=0x601866a9c4240 00000400:00000200:2.0:1616513075.807645:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.152@tcp1 of length 224/224 into md 0xf96450b9 [1] + 54896 00000400:00000200:2.0:1616513075.807651:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.807654:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.807660:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.807663:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.807665:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.807725:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690526686134848 00010000:00000200:18.0:1616513075.807738:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120f980 x1690526686134848/t0(0) o400->8f41c04b-6b33-1ed9-41aa-6b8000491682@10.31.141.152@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.807752:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690526686134848, offset 192 00000400:00000200:18.0:1616513075.807759:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.152@tcp1 00000400:00000200:18.0:1616513075.807766:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.152@tcp1 routed destination 00000400:00000200:18.0:1616513075.807781:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.152@tcp1(10.31.141.152@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.807788:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.807794:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.807797:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.807799:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.807803:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.807808:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.807862:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.807869:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.807872:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513075.807875:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513075.807879:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.807883:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cda90 00000800:00000200:22.0:1616513075.807886:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.831344:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513075.831399:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.831409:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513075.831418:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.57@tcp1 : PUT - for me 00000400:00000200:20.0:1616513075.831428:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.57@tcp1 of length 224 into portal 12 MB=0x5ffd4f8827280 00000400:00000200:20.0:1616513075.831448:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.57@tcp1 of length 224/224 into md 0x865b9f31 [1] + 88216 00000400:00000200:20.0:1616513075.831453:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513075.831456:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.831457:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.831465:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.831468:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.831470:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.831530:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665051001472 00010000:00000200:24.0:1616513075.831542:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07980 x1688665051001472/t0(0) o400->636da6f0-d0b1-0532-1976-9efae747a308@10.31.138.57@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.831556:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665051001472, offset 192 00000400:00000200:24.0:1616513075.831563:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.57@tcp1 00000400:00000200:24.0:1616513075.831571:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.57@tcp1 routed destination 00000400:00000200:24.0:1616513075.831586:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.57@tcp1(10.31.138.57@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.831593:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.831598:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.831601:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.831603:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.831608:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.831613:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.831667:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.831674:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.831677:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.831684:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.831688:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:28.0:1616513075.831692:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.831695:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.832936:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513075.832948:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.832956:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513075.832965:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.57@tcp1 : PUT - for me 00000400:00000200:10.0:1616513075.832974:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.57@tcp1 of length 224 into portal 26 MB=0x5ffd4f88270c0 00000400:00000200:10.0:1616513075.832980:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.57@tcp1 of length 224/224 into md 0xf97c6679 [1] + 0 00000400:00000200:10.0:1616513075.832985:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2cbb0 00000400:00000200:10.0:1616513075.832987:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.832990:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513075.832994:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cbb0 00000800:00000200:10.0:1616513075.832997:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.832999:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.833001:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.833003:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.833058:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665051001024 00010000:00000200:15.0:1616513075.833068:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688665051001024/t0(0) o400->0721bd97-2efc-4d8d-11a0-791f39c791e5@10.31.138.57@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.833081:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665051001024, offset 192 00000400:00000200:15.0:1616513075.833087:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.57@tcp1 00000400:00000200:15.0:1616513075.833095:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.57@tcp1 routed destination 00000400:00000200:15.0:1616513075.833110:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.57@tcp1(10.31.138.57@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.833116:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.833122:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.833125:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.833127:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.833132:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.833136:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.833138:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513075.833147:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.833154:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.833157:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513075.833164:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.833170:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c74b8 00000800:00000200:14.0:1616513075.833174:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.833177:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.837097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.837153:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.837162:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.837171:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.25@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.837180:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.25@tcp1 of length 224 into portal 26 MB=0x5ffd3b47a1540 00000400:00000200:12.0:1616513075.837187:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.25@tcp1 of length 224/224 into md 0xf97c6699 [1] + 0 00000400:00000200:12.0:1616513075.837192:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2ded0 00000400:00000200:12.0:1616513075.837195:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.837198:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513075.837203:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2ded0 00000800:00000200:20.0:1616513075.837206:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.837206:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.837209:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.837210:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.837269:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659614635328 00010000:00000200:15.0:1616513075.837280:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688659614635328/t0(0) o400->ab7cca56-7fb5-6a12-d778-77e20ffc88d9@10.31.138.25@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.837294:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688659614635328, offset 192 00000400:00000200:15.0:1616513075.837302:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.25@tcp1 00000400:00000200:15.0:1616513075.837311:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.25@tcp1 routed destination 00000400:00000200:15.0:1616513075.837326:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.25@tcp1(10.31.138.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.837333:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.837339:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.837341:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.837344:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.837349:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.837353:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.837356:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513075.837406:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.837418:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.837422:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.837428:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.837448:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7ba0 00000800:00000200:28.0:1616513075.837452:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.837455:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.837461:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513075.837464:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.837469:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.25@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.837478:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.25@tcp1 of length 224 into portal 12 MB=0x5ffd3b47a16c0 00000400:00000200:28.0:1616513075.837484:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.25@tcp1 of length 224/224 into md 0x865b9f31 [1] + 88440 00000400:00000200:28.0:1616513075.837489:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.837491:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.837497:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.837500:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.837502:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.837561:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659614635712 00010000:00000200:24.0:1616513075.837573:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf84380 x1688659614635712/t0(0) o400->9f58c359-8312-13db-6585-9fb36f82f9f3@10.31.138.25@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.837587:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688659614635712, offset 192 00000400:00000200:24.0:1616513075.837594:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.25@tcp1 00000400:00000200:24.0:1616513075.837601:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.25@tcp1 routed destination 00000400:00000200:24.0:1616513075.837617:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.25@tcp1(10.31.138.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.837624:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513075.837630:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.837633:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513075.837635:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.837640:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.837644:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.837655:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.837660:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.837662:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.837668:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.837671:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3650 00000800:00000200:10.0:1616513075.837674:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.837677:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.847874:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.847930:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.847940:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.847949:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.102@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.847958:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.102@tcp1 of length 224 into portal 26 MB=0x5ffd4030b5bc0 00000400:00000200:12.0:1616513075.847965:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.102@tcp1 of length 224/224 into md 0xf97c66b9 [1] + 0 00000400:00000200:12.0:1616513075.847970:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d078 00000400:00000200:12.0:1616513075.847973:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.847976:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513075.847980:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d078 00000800:00000200:12.0:1616513075.847983:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.847984:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.847986:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.847988:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.848045:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660932778944 00010000:00000200:15.0:1616513075.848056:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688660932778944/t0(0) o400->bee0971e-d954-3451-25a6-272a114f2b7e@10.31.128.102@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.848070:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660932778944, offset 192 00000400:00000200:15.0:1616513075.848076:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.102@tcp1 00000400:00000200:15.0:1616513075.848085:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.102@tcp1 routed destination 00000400:00000200:15.0:1616513075.848100:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.102@tcp1(10.31.128.102@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.848107:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.848113:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.848116:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.848118:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.848123:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.848128:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.848130:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513075.848182:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.848189:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.848192:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513075.848199:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.848205:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6770 00000800:00000200:16.0:1616513075.848209:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.848212:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.848977:0:48998:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.848994:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.849002:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.849011:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.102@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.849020:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.102@tcp1 of length 224 into portal 12 MB=0x5ffd4030b5d40 00000400:00000200:14.0:1616513075.849025:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.102@tcp1 of length 224/224 into md 0xf96450b9 [1] + 55120 00000400:00000200:14.0:1616513075.849030:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.849033:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.849039:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.849042:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.849043:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.849046:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.849110:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660932779328 00010000:00000200:18.0:1616513075.849123:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e4800 x1688660932779328/t0(0) o400->079bc347-a959-2b3e-0adc-e12673965d31@10.31.128.102@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.849149:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660932779328, offset 192 00000400:00000200:18.0:1616513075.849155:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.102@tcp1 00000400:00000200:18.0:1616513075.849163:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.102@tcp1 routed destination 00000400:00000200:18.0:1616513075.849178:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.102@tcp1(10.31.128.102@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.849185:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.849190:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.849193:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.849195:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.849200:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.849205:0:48998:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.849215:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.849220:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.849222:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.849226:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.849228:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.849231:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdcb0 00000800:00000200:10.0:1616513075.849234:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513075.855362:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513075.855417:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513075.855426:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.855451:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.34@tcp1 : PUT - for me 00000400:00000200:22.0:1616513075.855461:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.34@tcp1 of length 224 into portal 26 MB=0x5ffde0f264380 00000400:00000200:22.0:1616513075.855468:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.34@tcp1 of length 224/224 into md 0xf97c66e9 [1] + 0 00000800:00000200:14.0:1616513075.855470:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513075.855472:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c908 00000400:00000200:22.0:1616513075.855475:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513075.855478:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513075.855479:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513075.855483:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c908 00000800:00000200:22.0:1616513075.855486:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513075.855487:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.34@tcp1 : PUT - for me 00000800:00000200:22.0:1616513075.855489:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513075.855491:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513075.855494:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513075.855496:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.34@tcp1 of length 224 into portal 12 MB=0x5ffde0f264400 00000400:00000200:14.0:1616513075.855504:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.34@tcp1 of length 224/224 into md 0xf96450b9 [1] + 55344 00000400:00000200:14.0:1616513075.855508:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.855511:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513075.855517:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.855520:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.855522:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.855558:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704085541760 00010000:00000200:15.0:1616513075.855569:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688704085541760/t0(0) o400->ed6a396d-7960-683f-3233-ba3b39596062@10.31.138.34@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513075.855580:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704085541888 00000100:00000200:15.0:1616513075.855583:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704085541760, offset 192 00000400:00000200:15.0:1616513075.855590:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.34@tcp1 00010000:00000200:18.0:1616513075.855592:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e4c80 x1688704085541888/t0(0) o400->279078e3-c118-2d65-5839-be49ece13eba@10.31.138.34@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513075.855598:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.34@tcp1 routed destination 00000100:00000200:18.0:1616513075.855605:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704085541888, offset 192 00000400:00000200:18.0:1616513075.855612:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.34@tcp1 00000400:00000200:15.0:1616513075.855614:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.34@tcp1(10.31.138.34@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513075.855619:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.34@tcp1 routed destination 00000800:00000200:15.0:1616513075.855621:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.855629:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.855632:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513075.855634:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.34@tcp1(10.31.138.34@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.855634:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.855639:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513075.855641:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.855647:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:15.0:1616513075.855647:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513075.855649:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.855651:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.855656:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.855661:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.855669:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.855673:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.855675:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513075.855678:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513075.855679:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.855682:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccdd0 00000800:00000200:10.0:1616513075.855684:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.855686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.855700:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.855707:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.855711:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513075.855717:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.855723:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6bb0 00000800:00000200:20.0:1616513075.855727:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.855730:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:23.0:1616513075.870619:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:23.0:1616513075.870626:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513075.877285:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513075.877342:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.877352:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.877361:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.62@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.877373:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.62@tcp1 of length 224 into portal 26 MB=0x5ffdc13160f40 00000400:00000200:16.0:1616513075.877380:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.62@tcp1 of length 224/224 into md 0xf97c6709 [1] + 0 00000400:00000200:16.0:1616513075.877385:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d7e8 00000400:00000200:16.0:1616513075.877388:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.877391:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513075.877396:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513075.877396:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d7e8 00000800:00000200:16.0:1616513075.877399:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.877402:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513075.877404:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.877406:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513075.877407:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.877415:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.62@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.877424:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.62@tcp1 of length 224 into portal 12 MB=0x5ffdc13161140 00000400:00000200:28.0:1616513075.877430:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.62@tcp1 of length 224/224 into md 0x865b9f31 [1] + 88664 00000400:00000200:28.0:1616513075.877448:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.877452:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.877459:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.877462:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513075.877463:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695561654080 00000800:00000200:28.0:1616513075.877464:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:15.0:1616513075.877474:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688695561654080/t0(0) o400->ba1bd05f-7544-2a23-f833-a5a5f081b559@10.31.142.62@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.877487:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695561654080, offset 192 00000400:00000200:15.0:1616513075.877493:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.62@tcp1 00000400:00000200:15.0:1616513075.877501:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.62@tcp1 routed destination 00000400:00000200:15.0:1616513075.877516:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.62@tcp1(10.31.142.62@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.877522:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513075.877526:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695561654592 00000800:00000200:15.0:1616513075.877528:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.877530:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.877532:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.877537:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:24.0:1616513075.877539:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf87500 x1688695561654592/t0(0) o400->01eeb139-70b3-64dd-e09f-e1d2b8ac29a3@10.31.142.62@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513075.877542:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.877544:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513075.877553:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695561654592, offset 192 00000400:00000200:24.0:1616513075.877559:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.62@tcp1 00000400:00000200:24.0:1616513075.877568:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.62@tcp1 routed destination 00000800:00000200:12.0:1616513075.877578:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513075.877583:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.62@tcp1(10.31.142.62@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513075.877587:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513075.877589:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513075.877590:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513075.877591:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513075.877596:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513075.877597:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513075.877599:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513075.877601:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513075.877603:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7e48 00000800:00000200:24.0:1616513075.877606:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513075.877606:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.877612:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.877675:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513075.877682:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513075.877685:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513075.877687:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513075.877691:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513075.877695:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:2.0:1616513075.877699:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513075.882973:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924944, portal 4 00000100:00000200:6.0:1616513075.882977:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 24, xid 1693067725924944, offset 0 00000400:00000200:6.0:1616513075.882981:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:6.0:1616513075.882986:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:6.0:1616513075.882994:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513075.882999:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:6.0:1616513075.883003:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513075.883005:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:6.0:1616513075.883006:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:6.0:1616513075.883010:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513075.883057:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:22.0:1616513075.883114:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:22.0:1616513075.883121:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:22.0:1616513075.883124:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513075.883124:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:22.0:1616513075.883130:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000100:00000200:22.0:1616513075.883136:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a7e780 x1693067725924944/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513075.883148:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00000 00000800:00000200:22.0:1616513075.883152:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:31.2:1616513075.883245:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:14.0:1616513075.883297:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:14.0:1616513075.883306:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.234@o2ib1 00000800:00000200:2.0:1616513075.883309:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:14.0:1616513075.883314:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513075.883321:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 336 into portal 4 MB=0x603d60c626a50 00000400:00000200:14.0:1616513075.883328:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.234@o2ib1 of length 336/336 into md 0xf97c6da9 [1] + 192 00000400:00000200:14.0:1616513075.883332:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.883337:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a7e780 x1693067725924944/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513075.883350:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:14.0:1616513075.883353:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)-- 00000100:00000200:6.0:1616513075.883354:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a7e780 x1693067725924944/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513075.883355:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:6.0:1616513075.883362:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a7e780 x1693067725924944/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513075.883369:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01a08 00000800:00000200:11.2:1616513075.887154:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513075.887212:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.887221:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513075.887230:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.104@tcp1 : PUT - for me 00000400:00000200:20.0:1616513075.887239:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.104@tcp1 of length 224 into portal 26 MB=0x5ffe3c5390e00 00000400:00000200:20.0:1616513075.887246:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.104@tcp1 of length 224/224 into md 0xf97c6729 [1] + 0 00000400:00000200:20.0:1616513075.887251:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d298 00000400:00000200:20.0:1616513075.887254:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513075.887256:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513075.887261:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d298 00000800:00000200:20.0:1616513075.887264:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.887266:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.887267:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513075.887273:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.887276:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513075.887276:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.887285:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.104@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.887294:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.104@tcp1 of length 224 into portal 12 MB=0x5ffe3c53911c0 00000400:00000200:28.0:1616513075.887300:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.104@tcp1 of length 224/224 into md 0x865b9f31 [1] + 88888 00000400:00000200:28.0:1616513075.887305:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.887308:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.887315:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.887318:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.887320:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.887326:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728615063040 00010000:00000200:15.0:1616513075.887337:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688728615063040/t0(0) o400->aa9e5dc3-884e-ac64-2c6b-2954c8838fa0@10.31.141.104@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.887351:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688728615063040, offset 192 00000400:00000200:15.0:1616513075.887357:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.104@tcp1 00000400:00000200:15.0:1616513075.887365:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.104@tcp1 routed destination 00000100:00000200:24.0:1616513075.887378:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728615064000 00000400:00000200:15.0:1616513075.887381:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.104@tcp1(10.31.141.104@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.887388:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:24.0:1616513075.887391:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf85580 x1688728615064000/t0(0) o400->398404c3-5397-bd8e-d5b6-80cb86b569d1@10.31.141.104@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.887394:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.887397:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.887399:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.887404:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513075.887405:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688728615064000, offset 192 00000800:00000200:11.2:1616513075.887409:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513075.887411:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.104@tcp1 00000100:00000200:15.0:1616513075.887413:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513075.887419:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.104@tcp1 routed destination 00000800:00000200:12.0:1616513075.887429:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.887444:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.887450:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.887453:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513075.887456:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.104@tcp1(10.31.141.104@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513075.887460:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513075.887463:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513075.887466:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7870 00000800:00000200:24.0:1616513075.887469:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513075.887470:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513075.887472:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513075.887474:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513075.887479:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.887485:0:49003:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.887495:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.887500:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.887502:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.887507:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.887510:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf22a8 00000800:00000200:10.0:1616513075.887514:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.887516:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.893828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.893885:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.893894:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.893903:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.177@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.893913:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.177@tcp1 of length 224 into portal 26 MB=0x5ffde5a04d200 00000400:00000200:28.0:1616513075.893920:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.177@tcp1 of length 224/224 into md 0xf97c6739 [1] + 0 00000400:00000200:28.0:1616513075.893925:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2ca18 00000400:00000200:28.0:1616513075.893927:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.893930:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.893935:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2ca18 00000800:00000200:28.0:1616513075.893938:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.893938:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.893941:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.893943:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.893998:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705341641216 00010000:00000200:15.0:1616513075.894010:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688705341641216/t0(0) o400->22b9c6af-3c58-4ad0-f32e-98f20f906b25@10.31.132.177@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.894024:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705341641216, offset 192 00000400:00000200:15.0:1616513075.894031:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.177@tcp1 00000400:00000200:15.0:1616513075.894039:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.177@tcp1 routed destination 00000400:00000200:15.0:1616513075.894061:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.177@tcp1(10.31.132.177@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.894068:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513075.894074:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.894076:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513075.894079:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513075.894084:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.894089:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513075.894090:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513075.894100:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.894107:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.894111:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513075.894117:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.894123:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c63b8 00000800:00000200:20.0:1616513075.894127:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.894130:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.894529:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513075.894583:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.894591:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513075.894594:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513075.894602:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.177@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.894611:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.177@tcp1 of length 224 into portal 12 MB=0x5ffde5a04d2c0 00000400:00000200:2.0:1616513075.894617:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.177@tcp1 of length 224/224 into md 0xf96450b9 [1] + 55568 00000400:00000200:2.0:1616513075.894622:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.894624:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.894631:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.894634:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.894635:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.894695:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705341641408 00010000:00000200:18.0:1616513075.894707:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120a400 x1688705341641408/t0(0) o400->434d209f-1297-77f9-800c-9b8d99355ec4@10.31.132.177@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513075.894722:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705341641408, offset 192 00000400:00000200:18.0:1616513075.894729:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.177@tcp1 00000400:00000200:18.0:1616513075.894737:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.177@tcp1 routed destination 00000400:00000200:18.0:1616513075.894752:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.177@tcp1(10.31.132.177@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.894761:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513075.894767:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.894769:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513075.894772:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513075.894776:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.894782:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.894794:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.894799:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.894801:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513075.894806:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.894809:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccbb0 00000800:00000200:10.0:1616513075.894812:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513075.894814:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.903449:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513075.903505:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.903514:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.903523:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.903532:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x60399133ce2c0 00000400:00000200:16.0:1616513075.903538:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0x865b9f31 [1] + 89112 00000400:00000200:16.0:1616513075.903543:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.903546:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513075.903552:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.903555:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.903556:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.903560:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.903617:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805847900864 00010000:00000200:24.0:1616513075.903687:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5a00 x1692805847900864/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:701/0 lens 336/336 e 0 to 0 dl 1616513081 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513075.903702:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805847900864, offset 192 00000400:00000200:24.0:1616513075.903708:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:24.0:1616513075.903715:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:24.0:1616513075.903730:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.903736:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.903742:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.903744:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.903747:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.903751:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.903757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.903809:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.903816:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.903819:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.903822:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513075.903826:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.903830:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2cc0 00000800:00000200:20.0:1616513075.903834:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.911391:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.911448:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.911457:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.911465:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.107@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.911473:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.107@tcp1 of length 224 into portal 26 MB=0x6012d6f30a400 00000400:00000200:28.0:1616513075.911480:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.107@tcp1 of length 224/224 into md 0xf97c6759 [1] + 0 00000400:00000200:28.0:1616513075.911486:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2cf68 00000400:00000200:28.0:1616513075.911489:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.911491:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.911496:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cf68 00000800:00000200:28.0:1616513075.911498:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.911501:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513075.911501:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513075.911508:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.911511:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.911512:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.911520:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.107@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.911529:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.107@tcp1 of length 224 into portal 12 MB=0x6012d6f30a480 00000400:00000200:16.0:1616513075.911535:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.107@tcp1 of length 224/224 into md 0x865b9f31 [1] + 89448 00000400:00000200:16.0:1616513075.911541:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.911543:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513075.911550:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.911553:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.911555:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.911559:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690144510878720 00010000:00000200:15.0:1616513075.911569:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1690144510878720/t0(0) o400->3589c2b8-a4e7-a178-7ec8-330bf1705c0b@10.31.140.107@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.911583:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690144510878720, offset 192 00000400:00000200:15.0:1616513075.911588:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.107@tcp1 00000400:00000200:15.0:1616513075.911595:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.107@tcp1 routed destination 00000400:00000200:15.0:1616513075.911609:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.107@tcp1(10.31.140.107@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513075.911612:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690144510878848 00000800:00000200:15.0:1616513075.911616:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.911621:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:24.0:1616513075.911624:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd6300 x1690144510878848/t0(0) o400->75e57bb4-0333-2135-c0fa-843ef6b2a54f@10.31.140.107@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.911624:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.911626:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.911631:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513075.911637:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690144510878848, offset 192 00000100:00000200:15.0:1616513075.911637:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.911637:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513075.911643:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.107@tcp1 00000400:00000200:24.0:1616513075.911650:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.107@tcp1 routed destination 00000400:00000200:24.0:1616513075.911664:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.107@tcp1(10.31.140.107@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.911671:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.911676:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.911679:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.911682:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.911687:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513075.911689:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513075.911692:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513075.911696:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.911699:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513075.911702:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513075.911705:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513075.911709:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513075.911711:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6088 00000400:00000200:20.0:1616513075.911712:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.911713:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513075.911714:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.911717:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513075.911719:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.911723:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2110 00000800:00000200:20.0:1616513075.911727:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.924128:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513075.924186:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.924196:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513075.924205:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.91@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.924215:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.91@tcp1 of length 224 into portal 26 MB=0x5ffde3374adc0 00000400:00000200:12.0:1616513075.924222:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.91@tcp1 of length 224/224 into md 0xf97c6769 [1] + 0 00000400:00000200:12.0:1616513075.924227:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2cdd0 00000400:00000200:12.0:1616513075.924230:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.924233:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513075.924237:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cdd0 00000800:00000200:16.0:1616513075.924240:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.924240:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.924243:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513075.924246:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513075.924249:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.924250:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513075.924259:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.91@tcp1 : PUT - for me 00000400:00000200:16.0:1616513075.924268:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.91@tcp1 of length 224 into portal 12 MB=0x5ffde3374b080 00000400:00000200:16.0:1616513075.924275:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.91@tcp1 of length 224/224 into md 0x865b9f31 [1] + 89672 00000400:00000200:16.0:1616513075.924280:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.924282:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513075.924289:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.924292:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.924294:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.924311:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704694660544 00010000:00000200:15.0:1616513075.924322:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688704694660544/t0(0) o400->99c97b6c-44e3-7152-b000-e8f98afcbb85@10.31.142.91@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.924337:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704694660544, offset 192 00000400:00000200:15.0:1616513075.924345:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.91@tcp1 00000400:00000200:15.0:1616513075.924354:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.91@tcp1 routed destination 00000100:00000200:24.0:1616513075.924366:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704694661248 00000400:00000200:15.0:1616513075.924369:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.91@tcp1(10.31.142.91@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.924376:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:24.0:1616513075.924378:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd3600 x1688704694661248/t0(0) o400->99ea7927-b3c4-ecbb-bd1d-731a90e327c5@10.31.142.91@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.924382:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.924385:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.924387:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513075.924391:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704694661248, offset 192 00000800:00000200:15.0:1616513075.924392:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513075.924397:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.91@tcp1 00000800:00000200:7.2:1616513075.924399:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.924400:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513075.924404:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.91@tcp1 routed destination 00000800:00000200:10.0:1616513075.924409:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.924414:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.924416:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513075.924418:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.91@tcp1(10.31.142.91@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513075.924421:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513075.924424:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:10.0:1616513075.924426:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6f68 00000800:00000200:24.0:1616513075.924430:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513075.924430:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513075.924432:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513075.924432:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513075.924446:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.924450:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.924455:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.924466:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.924473:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.924476:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.924477:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513075.924482:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.924486:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000800:00000200:20.0:1616513075.924490:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.926980:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.927036:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.927045:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.927054:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.119@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.927063:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.119@tcp1 of length 224 into portal 26 MB=0x5ffdd7ca22280 00000400:00000200:28.0:1616513075.927070:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.119@tcp1 of length 224/224 into md 0xf97c67a9 [1] + 0 00000400:00000200:28.0:1616513075.927075:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c330 00000400:00000200:28.0:1616513075.927078:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.927080:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.927085:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c330 00000800:00000200:28.0:1616513075.927088:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.927089:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.927091:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.927093:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.927150:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701627409024 00010000:00000200:15.0:1616513075.927161:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688701627409024/t0(0) o400->881dfb22-d175-2953-4069-e0c1229f80b1@10.31.132.119@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.927175:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701627409024, offset 192 00000400:00000200:15.0:1616513075.927182:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.119@tcp1 00000400:00000200:15.0:1616513075.927190:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.119@tcp1 routed destination 00000400:00000200:15.0:1616513075.927205:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.119@tcp1(10.31.132.119@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.927211:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.927218:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.927234:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.927236:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.927240:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.927245:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.927247:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513075.927302:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513075.927323:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.927330:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513075.927333:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513075.927356:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513075.927362:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6660 00000800:00000200:14.0:1616513075.927365:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.927368:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.927376:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513075.927377:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.927379:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.927381:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.927382:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513075.927387:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.927397:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513075.927400:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.927405:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.119@tcp1 : PUT - for me 00000400:00000200:12.0:1616513075.927414:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.119@tcp1 of length 224 into portal 12 MB=0x5ffdd7ca22440 00000400:00000200:12.0:1616513075.927420:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.119@tcp1 of length 224/224 into md 0x865b9f31 [1] + 89896 00000400:00000200:12.0:1616513075.927425:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513075.927428:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513075.927446:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.927450:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513075.927452:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513075.927514:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701627409472 00010000:00000200:24.0:1616513075.927525:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661b00 x1688701627409472/t0(0) o400->cfa8be43-ad66-7cf9-8b11-153e15fe8723@10.31.132.119@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.927538:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701627409472, offset 192 00000400:00000200:24.0:1616513075.927544:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.119@tcp1 00000400:00000200:24.0:1616513075.927551:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.119@tcp1 routed destination 00000400:00000200:24.0:1616513075.927565:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.119@tcp1(10.31.132.119@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.927572:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.927577:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.927579:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.927582:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.927586:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.927591:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.927601:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.927607:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513075.927609:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.927611:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513075.927616:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513075.927620:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2880 00000800:00000200:16.0:1616513075.927624:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.962058:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513075.962115:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.962125:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513075.962134:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.205@tcp1 : PUT - for me 00000400:00000200:20.0:1616513075.962143:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.205@tcp1 of length 224 into portal 26 MB=0x601ab34273500 00000400:00000200:20.0:1616513075.962150:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.205@tcp1 of length 224/224 into md 0xf97c67b9 [1] + 0 00000400:00000200:20.0:1616513075.962155:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2de48 00000400:00000200:20.0:1616513075.962158:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513075.962161:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513075.962166:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2de48 00000800:00000200:12.0:1616513075.962168:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.962169:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.962172:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513075.962173:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513075.962231:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690684686284032 00010000:00000200:15.0:1616513075.962242:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1690684686284032/t0(0) o400->86af599f-3d04-b374-d2d1-3557f8c076ad@10.31.140.205@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.962257:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690684686284032, offset 192 00000400:00000200:15.0:1616513075.962263:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.205@tcp1 00000400:00000200:15.0:1616513075.962272:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.205@tcp1 routed destination 00000800:00000200:11.2:1616513075.962282:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:15.0:1616513075.962288:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.205@tcp1(10.31.140.205@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.962309:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.962316:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.962318:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.962320:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.962325:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513075.962331:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.962331:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513075.962335:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.962345:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.962353:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.205@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.962362:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.205@tcp1 of length 224 into portal 12 MB=0x601ab34273680 00000400:00000200:28.0:1616513075.962368:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.205@tcp1 of length 224/224 into md 0x865b9f31 [1] + 90120 00000400:00000200:28.0:1616513075.962374:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.962376:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513075.962383:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.962386:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.962388:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.962391:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513075.962392:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513075.962400:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.962402:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513075.962403:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513075.962410:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.962416:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6660 00000800:00000200:22.0:1616513075.962420:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513075.962460:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690684686284416 00010000:00000200:24.0:1616513075.962471:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf80d80 x1690684686284416/t0(0) o400->9705a2dc-c4da-30cb-59f0-966c1b2c49ae@10.31.140.205@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513075.962485:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690684686284416, offset 192 00000400:00000200:24.0:1616513075.962492:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.205@tcp1 00000400:00000200:24.0:1616513075.962500:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.205@tcp1 routed destination 00000400:00000200:24.0:1616513075.962514:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.205@tcp1(10.31.140.205@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513075.962520:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513075.962526:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513075.962528:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513075.962530:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513075.962535:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.962539:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.962550:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.962555:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.962558:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513075.962560:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.962564:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.962568:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3c28 00000800:00000200:12.0:1616513075.962572:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.977886:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.977899:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.977907:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513075.977915:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.7@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.977924:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.7@tcp1 of length 224 into portal 26 MB=0x5ffeeea14a800 00000400:00000200:14.0:1616513075.977930:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.7@tcp1 of length 224/224 into md 0xf97c67e9 [1] + 0 00000400:00000200:14.0:1616513075.977935:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d870 00000400:00000200:14.0:1616513075.977937:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.977940:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513075.977945:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d870 00000800:00000200:14.0:1616513075.977948:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513075.977950:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.977951:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513075.977958:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513075.977961:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.977962:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513075.977970:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.7@tcp1 : PUT - for me 00000400:00000200:2.0:1616513075.977979:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.7@tcp1 of length 224 into portal 12 MB=0x5ffeeea14a880 00000400:00000200:2.0:1616513075.977986:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.7@tcp1 of length 224/224 into md 0xf96450b9 [1] + 55792 00000400:00000200:2.0:1616513075.977991:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.977994:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513075.978001:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513075.978004:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513075.978006:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.978010:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688776478074880 00010000:00000200:15.0:1616513075.978021:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688776478074880/t0(0) o400->13fbb01e-ba27-35c1-c7df-3159b8ac190c@10.31.142.7@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513075.978035:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688776478074880, offset 192 00000400:00000200:15.0:1616513075.978041:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.7@tcp1 00000400:00000200:15.0:1616513075.978049:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.7@tcp1 routed destination 00000400:00000200:15.0:1616513075.978064:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.7@tcp1(10.31.142.7@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513075.978067:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688776478075008 00000800:00000200:15.0:1616513075.978071:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.978077:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:18.0:1616513075.978079:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120c800 x1688776478075008/t0(0) o400->f1ab44ea-681f-12c5-11ae-da71f8b2a81b@10.31.142.7@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513075.978080:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.978082:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.978087:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.978092:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:18.0:1616513075.978094:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688776478075008, offset 192 00000100:00000200:15.0:1616513075.978094:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513075.978101:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.7@tcp1 00000400:00000200:18.0:1616513075.978109:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.7@tcp1 routed destination 00000400:00000200:18.0:1616513075.978124:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.7@tcp1(10.31.142.7@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.978131:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.978137:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.978139:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.978142:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513075.978146:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513075.978147:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513075.978153:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.978156:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513075.978156:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513075.978163:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.978169:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6088 00000800:00000200:22.0:1616513075.978173:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.978190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.978245:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.978251:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.978254:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.978261:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.978265:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc5d8 00000800:00000200:28.0:1616513075.978269:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513075.978272:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513075.978794:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513075.978804:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.978809:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513075.978813:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513075.978816:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.18@tcp1 : PUT - for me 00000400:00000200:14.0:1616513075.978823:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.18@tcp1 of length 224 into portal 26 MB=0x5f90c17841300 00000400:00000200:14.0:1616513075.978828:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.18@tcp1 of length 224/224 into md 0xf97c6809 [1] + 0 00000400:00000200:14.0:1616513075.978832:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2caa0 00000400:00000200:14.0:1616513075.978834:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513075.978836:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513075.978840:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2caa0 00000800:00000200:14.0:1616513075.978841:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513075.978843:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.978845:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513075.978853:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681205213008640 00010000:00000200:15.0:1616513075.978860:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1681205213008640/t0(0) o400->54fe7d6b-9afb-3c2c-06f0-62712710d668@10.31.132.18@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513075.978863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513075.978871:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.978872:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681205213008640, offset 192 00000400:00000200:15.0:1616513075.978877:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.18@tcp1 00000800:00000200:10.0:1616513075.978877:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513075.978880:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:15.0:1616513075.978883:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.18@tcp1 routed destination 00000400:00000200:10.0:1616513075.978885:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.18@tcp1 : PUT - for me 00000400:00000200:15.0:1616513075.978897:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.18@tcp1(10.31.132.18@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.978903:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:10.0:1616513075.978906:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.18@tcp1 of length 224 into portal 12 MB=0x5f90c17841380 00000800:00000200:15.0:1616513075.978908:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.978911:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:10.0:1616513075.978911:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.18@tcp1 of length 224/224 into md 0xf96450b9 [1] + 56016 00000800:00000200:15.0:1616513075.978915:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000400:00000200:10.0:1616513075.978915:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.978917:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:15.0:1616513075.978918:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.978922:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:15.0:1616513075.978923:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.978923:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.978925:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:10.0:1616513075.978926:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.978928:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.978932:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513075.978933:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513075.978934:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:18.0:1616513075.978936:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681205213008768 00000400:00000200:10.0:1616513075.978938:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513075.978942:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7870 00010000:00000200:18.0:1616513075.978943:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdd580 x1681205213008768/t0(0) o400->8da5ab13-dc51-1400-5779-2e9fb530f0ee@10.31.132.18@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:10.0:1616513075.978946:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513075.978955:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681205213008768, offset 192 00000400:00000200:18.0:1616513075.978960:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.18@tcp1 00000400:00000200:18.0:1616513075.978966:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.18@tcp1 routed destination 00000400:00000200:18.0:1616513075.978979:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.18@tcp1(10.31.132.18@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513075.978985:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513075.978990:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513075.978992:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513075.978994:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513075.978997:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.979002:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.979013:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.979019:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513075.979023:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513075.979030:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513075.979033:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cca18 00000800:00000200:20.0:1616513075.979037:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.979040:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513075.979675:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.979727:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.979736:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513075.979745:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.181@tcp1 : PUT - for me 00000400:00000200:28.0:1616513075.979755:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.181@tcp1 of length 224 into portal 26 MB=0x5ffd38614cb40 00000400:00000200:28.0:1616513075.979762:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.181@tcp1 of length 224/224 into md 0xf97c6819 [1] + 0 00000400:00000200:28.0:1616513075.979767:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2dcb0 00000400:00000200:28.0:1616513075.979770:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513075.979773:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513075.979778:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2dcb0 00000800:00000200:28.0:1616513075.979781:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513075.979781:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.979784:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.979788:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513075.979791:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513075.979792:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:15.0:1616513075.979792:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658836245312 00010000:00000200:15.0:1616513075.979798:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688658836245312/t0(0) o400->bdc92e85-7587-d660-cf61-bf80b97b9e95@10.31.138.181@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513075.979801:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.181@tcp1 : PUT - for me 00000100:00000200:15.0:1616513075.979808:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658836245312, offset 192 00000400:00000200:16.0:1616513075.979811:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.181@tcp1 of length 224 into portal 12 MB=0x5ffd38614cbc0 00000400:00000200:15.0:1616513075.979813:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.181@tcp1 00000400:00000200:16.0:1616513075.979818:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.181@tcp1 of length 224/224 into md 0x865b9f31 [1] + 90344 00000400:00000200:15.0:1616513075.979820:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.181@tcp1 routed destination 00000400:00000200:16.0:1616513075.979824:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513075.979826:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:15.0:1616513075.979833:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.181@tcp1(10.31.138.181@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513075.979835:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513075.979838:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:15.0:1616513075.979839:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513075.979840:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:15.0:1616513075.979844:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.979847:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.979848:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.979851:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:28.0:1616513075.979856:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658836245440 00000100:00000200:15.0:1616513075.979856:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513075.979856:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:29.0:1616513075.979861:0:6495:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924960, portal 4 00000800:00000200:22.0:1616513075.979865:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:29.0:1616513075.979867:0:6495:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924960, offset 0 00010000:00000200:28.0:1616513075.979867:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd4c80 x1688658836245440/t0(0) o400->fca84381-901c-6f2d-1211-6814b029a1d0@10.31.138.181@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:22.0:1616513075.979869:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513075.979872:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513075.979873:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:29.0:1616513075.979874:0:6495:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513075.979877:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:29.0:1616513075.979880:0:6495:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000100:00000200:28.0:1616513075.979880:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658836245440, offset 192 00000400:00000200:22.0:1616513075.979882:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6bb0 00000800:00000200:22.0:1616513075.979884:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513075.979886:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.181@tcp1 00000400:00000200:28.0:1616513075.979894:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.181@tcp1 routed destination 00000400:00000200:29.0:1616513075.979895:0:6495:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513075.979902:0:6495:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:29.0:1616513075.979908:0:6495:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513075.979909:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.181@tcp1(10.31.138.181@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513075.979912:0:6495:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:29.0:1616513075.979914:0:6495:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:28.0:1616513075.979916:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:29.0:1616513075.979919:0:6495:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:28.0:1616513075.979922:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513075.979924:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.979926:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.979930:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.979935:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513075.979945:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513075.979951:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513075.979954:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513075.979955:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513075.979960:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513075.979963:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd8f8 00000800:00000200:12.0:1616513075.979967:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:25.2:1616513075.979968:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513075.979982:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513075.979986:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000800:00000200:10.0:1616513075.979988:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513075.979989:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513075.979994:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513075.980001:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1390000 x1693067725924960/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513075.980012:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b3dc0 00000800:00000200:14.0:1616513075.980015:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513075.980133:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513075.980143:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513075.980149:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513075.980151:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513075.980156:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513075.980162:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626a60 00000400:00000200:2.0:1616513075.980169:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c6e79 [1] + 192 00000400:00000200:2.0:1616513075.980173:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513075.980178:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1390000 x1693067725924960/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513075.980191:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513075.980194:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513075.980197:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:29.0:1616513075.980203:0:6495:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1390000 x1693067725924960/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:29.0:1616513075.980216:0:6495:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1390000 x1693067725924960/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513082 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:29.0:1616513075.980227:0:6495:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b25d8 00000800:00000200:7.2:1616513075.983389:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513075.983402:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513075.983408:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513075.983415:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.12@tcp1 : PUT - for me 00000400:00000200:10.0:1616513075.983421:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.12@tcp1 of length 224 into portal 26 MB=0x60393e005a140 00000400:00000200:10.0:1616513075.983427:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.12@tcp1 of length 224/224 into md 0xf97c6839 [1] + 0 00000400:00000200:10.0:1616513075.983430:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d320 00000400:00000200:10.0:1616513075.983432:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513075.983434:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513075.983445:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d320 00000800:00000200:10.0:1616513075.983447:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513075.983449:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513075.983450:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513075.983452:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513075.983454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513075.983509:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783513805120 00000800:00000200:20.0:1616513075.983511:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513075.983519:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00010000:00000200:15.0:1616513075.983520:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1692783513805120/t0(0) o400->046ca472-09a0-0214-0239-df9c31ff4298@10.31.140.12@tcp1:32/0 lens 224/224 e 0 to 0 dl 1616513167 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513075.983528:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.12@tcp1 : PUT - for me 00000100:00000200:15.0:1616513075.983535:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783513805120, offset 192 00000400:00000200:20.0:1616513075.983536:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.12@tcp1 of length 224 into portal 12 MB=0x60393e005a1c0 00000400:00000200:15.0:1616513075.983541:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.12@tcp1 00000400:00000200:20.0:1616513075.983542:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.12@tcp1 of length 224/224 into md 0x865b9f31 [1] + 90568 00000400:00000200:20.0:1616513075.983547:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:15.0:1616513075.983549:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.12@tcp1 routed destination 00000100:00000200:20.0:1616513075.983550:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513075.983556:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513075.983558:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513075.983560:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513075.983563:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:15.0:1616513075.983565:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.12@tcp1(10.31.140.12@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513075.983571:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513075.983578:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513075.983581:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513075.983583:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513075.983588:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513075.983593:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513075.983595:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513075.983622:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783513805248 00010000:00000200:28.0:1616513075.983634:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07080 x1692783513805248/t0(0) o400->de69e8c0-36f9-2f4e-3357-b11565387837@10.31.140.12@tcp1:701/0 lens 224/224 e 0 to 0 dl 1616513081 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:22.0:1616513075.983646:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:28.0:1616513075.983648:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783513805248, offset 192 00000800:00000200:22.0:1616513075.983653:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513075.983655:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.12@tcp1 00000400:00000200:22.0:1616513075.983656:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513075.983662:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.12@tcp1 routed destination 00000400:00000200:22.0:1616513075.983663:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513075.983669:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7ba0 00000800:00000200:22.0:1616513075.983673:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513075.983676:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513075.983677:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.12@tcp1(10.31.140.12@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513075.983684:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513075.983690:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513075.983693:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513075.983695:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.983700:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513075.983705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513075.983717:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513075.983721:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513075.983723:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513075.983726:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513075.983728:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513075.983731:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc000 00000800:00000200:28.0:1616513075.983734:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.035925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.035936:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.035943:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513076.035950:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.81@tcp1 : PUT - for me 00000400:00000200:10.0:1616513076.035957:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.81@tcp1 of length 224 into portal 26 MB=0x6012e107bdbc0 00000400:00000200:10.0:1616513076.035963:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.81@tcp1 of length 224/224 into md 0xf97c6849 [1] + 0 00000400:00000200:10.0:1616513076.035966:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2dc28 00000400:00000200:10.0:1616513076.035968:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513076.035971:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513076.035974:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2dc28 00000800:00000200:10.0:1616513076.035976:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513076.035978:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513076.035980:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513076.035982:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513076.036048:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690147216939968 00010000:00000200:15.0:1616513076.036059:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1690147216939968/t0(0) o400->6391f728-ec29-27b9-4ac1-da7aabf0c616@10.31.140.81@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513076.036075:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690147216939968, offset 192 00000400:00000200:15.0:1616513076.036081:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.81@tcp1 00000400:00000200:15.0:1616513076.036089:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.81@tcp1 routed destination 00000400:00000200:15.0:1616513076.036105:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.81@tcp1(10.31.140.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513076.036111:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513076.036120:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513076.036123:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513076.036125:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513076.036130:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.036135:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513076.036137:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513076.036189:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.036197:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.036200:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.036207:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.036214:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6aa0 00000800:00000200:22.0:1616513076.036218:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.036221:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.036647:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.036655:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.036661:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513076.036665:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513076.036668:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.81@tcp1 : PUT - for me 00000400:00000200:10.0:1616513076.036685:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.81@tcp1 of length 224 into portal 12 MB=0x6012e107bde00 00000400:00000200:10.0:1616513076.036690:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.81@tcp1 of length 224/224 into md 0xf96450b9 [1] + 56240 00000400:00000200:10.0:1616513076.036696:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513076.036697:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513076.036701:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.036703:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513076.036704:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513076.036767:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690147216940544 00010000:00000200:18.0:1616513076.036780:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdd100 x1690147216940544/t0(0) o400->ac3f9fac-6fa1-eb21-7d92-76232f969dfb@10.31.140.81@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513076.036795:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690147216940544, offset 192 00000400:00000200:18.0:1616513076.036801:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.81@tcp1 00000400:00000200:18.0:1616513076.036809:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.81@tcp1 routed destination 00000400:00000200:18.0:1616513076.036824:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.81@tcp1(10.31.140.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513076.036831:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513076.036837:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.036840:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513076.036842:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513076.036847:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.036893:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.036962:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.036981:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.036983:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.036989:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.036993:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdba0 00000800:00000200:20.0:1616513076.036997:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.037000:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.038450:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513076.038506:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.038514:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.038522:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.38@tcp1 : PUT - for me 00000400:00000200:12.0:1616513076.038531:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.38@tcp1 of length 224 into portal 26 MB=0x5ffd3d52e70c0 00000400:00000200:12.0:1616513076.038537:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.38@tcp1 of length 224/224 into md 0xf97c6869 [1] + 0 00000800:00000200:7.2:1616513076.038539:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513076.038542:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2da90 00000400:00000200:12.0:1616513076.038544:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.038547:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513076.038551:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2da90 00000800:00000200:12.0:1616513076.038554:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.038557:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513076.038559:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.038563:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513076.038593:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.038603:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513076.038611:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.38@tcp1 : PUT - for me 00000100:00000200:15.0:1616513076.038615:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660163326144 00000400:00000200:22.0:1616513076.038620:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.38@tcp1 of length 224 into portal 12 MB=0x5ffd3d52e7440 00000400:00000200:22.0:1616513076.038627:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.38@tcp1 of length 224/224 into md 0xf96450b9 [1] + 56464 00010000:00000200:15.0:1616513076.038627:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688660163326144/t0(0) o400->812e8484-1a01-6d9a-a8e0-9e1b8bbfa77f@10.31.129.38@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513076.038632:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.038635:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513076.038641:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:15.0:1616513076.038641:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660163326144, offset 192 00000800:00000200:22.0:1616513076.038644:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.038646:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:15.0:1616513076.038648:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.38@tcp1 00000800:00000200:22.0:1616513076.038649:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513076.038655:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.38@tcp1 routed destination 00000400:00000200:15.0:1616513076.038671:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.38@tcp1(10.31.129.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513076.038677:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513076.038683:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513076.038686:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513076.038689:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513076.038693:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.038698:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513076.038700:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513076.038718:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660163327040 00010000:00000200:18.0:1616513076.038730:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2cc80 x1688660163327040/t0(0) o400->350b15be-395e-deb7-3730-d02497d80476@10.31.129.38@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513076.038745:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660163327040, offset 192 00000800:00000200:14.0:1616513076.038750:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513076.038751:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.38@tcp1 00000800:00000200:14.0:1616513076.038758:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513076.038759:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.38@tcp1 routed destination 00000800:00000200:22.0:1616513076.038761:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513076.038761:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.038767:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.038773:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6550 00000400:00000200:18.0:1616513076.038774:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.38@tcp1(10.31.129.38@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513076.038776:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513076.038781:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513076.038787:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.038790:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513076.038792:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513076.038796:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.038802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.038856:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.038862:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.038865:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.038871:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.038875:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccaa0 00000800:00000200:20.0:1616513076.038880:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.038882:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:1.0:1616513076.043086:0:6488:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924976, portal 4 00000100:00000200:1.0:1616513076.043092:0:6488:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924976, offset 0 00000400:00000200:1.0:1616513076.043099:0:6488:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:1.0:1616513076.043105:0:6488:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:1.0:1616513076.043118:0:6488:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:1.0:1616513076.043124:0:6488:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:1.0:1616513076.043131:0:6488:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513076.043134:0:6488:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:1.0:1616513076.043136:0:6488:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:1.0:1616513076.043142:0:6488:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513076.043149:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513076.043205:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513076.043213:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513076.043216:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.043223:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513076.043229:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843a2ed580 x1693067725924976/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513076.043241:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85cb0 00000800:00000200:22.0:1616513076.043245:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513076.043248:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513076.043362:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513076.043372:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513076.043378:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513076.043386:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513076.043393:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626a70 00000400:00000200:22.0:1616513076.043400:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c6ee9 [1] + 192 00000400:00000200:22.0:1616513076.043405:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.043409:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843a2ed580 x1693067725924976/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513076.043422:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513076.043423:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513076.043425:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:22.0:1616513076.043427:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:1.0:1616513076.043462:0:6488:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843a2ed580 x1693067725924976/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513076.043487:0:6488:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843a2ed580 x1693067725924976/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513076.043497:0:6488:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85100 00000800:00000200:7.2:1616513076.056849:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513076.056903:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.056913:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513076.056922:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.129@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.056931:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.129@tcp1 of length 224 into portal 26 MB=0x5ffd5154e1200 00000400:00000200:22.0:1616513076.056938:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.129@tcp1 of length 224/224 into md 0xf97c6889 [1] + 0 00000400:00000200:22.0:1616513076.056942:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2cb28 00000400:00000200:22.0:1616513076.056945:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.056948:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513076.056953:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cb28 00000800:00000200:22.0:1616513076.056956:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.056957:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.056958:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.056961:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.056999:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513076.057017:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665534108160 00010000:00000200:15.0:1616513076.057028:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688665534108160/t0(0) o400->6369aea2-4e23-ab91-117f-fcd186ff3807@10.31.136.129@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513076.057042:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665534108160, offset 192 00000400:00000200:15.0:1616513076.057048:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.129@tcp1 00000800:00000200:14.0:1616513076.057050:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:15.0:1616513076.057056:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.129@tcp1 routed destination 00000800:00000200:14.0:1616513076.057072:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513076.057074:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:15.0:1616513076.057083:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.129@tcp1(10.31.136.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513076.057090:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.057093:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.129@tcp1 : PUT - for me 00000800:00000200:15.0:1616513076.057096:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513076.057107:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.129@tcp1 of length 224 into portal 12 MB=0x5ffd5154e12c0 00000800:00000200:15.0:1616513076.057109:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513076.057111:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513076.057113:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.129@tcp1 of length 224/224 into md 0xf96450b9 [1] + 56688 00000800:00000200:15.0:1616513076.057116:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.057118:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.057120:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513076.057120:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:15.0:1616513076.057122:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513076.057127:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513076.057130:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513076.057131:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:14.0:1616513076.057135:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513076.057136:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513076.057138:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513076.057139:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513076.057146:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.057152:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7650 00000800:00000200:2.0:1616513076.057156:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513076.057190:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665534108352 00010000:00000200:18.0:1616513076.057203:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e5100 x1688665534108352/t0(0) o400->bb85e352-eceb-8900-d8bf-bb99fb73c633@10.31.136.129@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513076.057217:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665534108352, offset 192 00000400:00000200:18.0:1616513076.057224:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.129@tcp1 00000400:00000200:18.0:1616513076.057232:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.129@tcp1 routed destination 00000400:00000200:18.0:1616513076.057248:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.129@tcp1(10.31.136.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513076.057255:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513076.057261:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.057264:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513076.057266:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513076.057271:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513076.057302:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513076.057311:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.057316:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513076.057316:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513076.057318:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513076.057323:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.75@tcp1 : PUT - for me 00000800:00000200:14.0:1616513076.057324:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513076.057329:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.75@tcp1 of length 224 into portal 26 MB=0x5ffeac7a29840 00000400:00000200:14.0:1616513076.057331:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.75@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.057335:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.75@tcp1 of length 224/224 into md 0xf97c6899 [1] + 0 00000400:00000200:22.0:1616513076.057338:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c088 00000400:00000200:14.0:1616513076.057338:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.75@tcp1 of length 224 into portal 12 MB=0x5ffeac7a29a40 00000400:00000200:22.0:1616513076.057340:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.057342:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.75@tcp1 of length 224/224 into md 0xf96450b9 [1] + 56912 00000100:00000200:22.0:1616513076.057346:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.057346:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.057350:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c088 00000800:00000200:22.0:1616513076.057352:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000100:00000200:14.0:1616513076.057352:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513076.057354:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513076.057356:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.057357:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513076.057358:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.057360:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.057362:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513076.057364:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688758720305216 00000100:00000200:18.0:1616513076.057370:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688758720305728 00010000:00000200:15.0:1616513076.057370:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688758720305216/t0(0) o400->ff1064ad-e770-ca10-2695-cee0908c8faf@10.31.142.75@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513076.057372:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:18.0:1616513076.057377:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e5a00 x1688758720305728/t0(0) o400->7ccf42b9-c511-09a2-8937-0697a380f27b@10.31.142.75@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513076.057379:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:15.0:1616513076.057381:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688758720305216, offset 192 00000400:00000200:28.0:1616513076.057382:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:15.0:1616513076.057387:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.75@tcp1 00000400:00000200:28.0:1616513076.057389:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:18.0:1616513076.057389:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688758720305728, offset 192 00000400:00000200:15.0:1616513076.057392:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.75@tcp1 routed destination 00000400:00000200:18.0:1616513076.057395:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.75@tcp1 00000400:00000200:28.0:1616513076.057398:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccc38 00000800:00000200:28.0:1616513076.057402:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.057405:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:15.0:1616513076.057405:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.75@tcp1(10.31.142.75@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513076.057411:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513076.057413:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.75@tcp1 routed destination 00000800:00000200:15.0:1616513076.057416:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513076.057419:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513076.057420:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513076.057424:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513076.057427:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.75@tcp1(10.31.142.75@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:15.0:1616513076.057428:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513076.057428:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513076.057444:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513076.057449:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513076.057449:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513076.057452:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513076.057453:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513076.057453:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.057455:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513076.057456:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513076.057457:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513076.057461:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513076.057462:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513076.057465:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6110 00000800:00000200:2.0:1616513076.057468:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513076.057517:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.057523:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.057526:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.057533:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.057538:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc990 00000800:00000200:20.0:1616513076.057543:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.057545:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.067795:0:49023:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513076.067846:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.067854:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.067862:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.9@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.067869:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.9@tcp1 of length 224 into portal 26 MB=0x5ffd4cb213200 00000400:00000200:14.0:1616513076.067875:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.9@tcp1 of length 224/224 into md 0xf97c68b9 [1] + 0 00000400:00000200:14.0:1616513076.067879:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2dba0 00000400:00000200:14.0:1616513076.067881:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.067884:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.067888:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2dba0 00000800:00000200:14.0:1616513076.067891:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.067893:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.067895:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.067897:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513076.067950:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664289653248 00010000:00000200:15.0:1616513076.067962:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688664289653248/t0(0) o400->63752af4-4ab5-412c-5000-73b9f4b6bbb4@10.31.136.9@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513076.067975:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664289653248, offset 192 00000400:00000200:15.0:1616513076.067981:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.9@tcp1 00000400:00000200:15.0:1616513076.067988:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.9@tcp1 routed destination 00000400:00000200:15.0:1616513076.068007:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.9@tcp1(10.31.136.9@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513076.068013:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513076.068022:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513076.068026:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513076.068028:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513076.068033:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.068037:0:49023:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513076.068039:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513076.068089:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.068095:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.068098:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.068104:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.068110:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6d48 00000800:00000200:22.0:1616513076.068114:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.068116:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.068122:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513076.068126:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.068129:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.9@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.068137:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.9@tcp1 of length 224 into portal 12 MB=0x5ffd4cb213440 00000400:00000200:22.0:1616513076.068142:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.9@tcp1 of length 224/224 into md 0xf96450b9 [1] + 57136 00000400:00000200:22.0:1616513076.068147:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.068149:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513076.068154:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.068156:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.068158:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.068215:0:49023:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:18.0:1616513076.068217:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664289653824 00000800:00000200:14.0:1616513076.068224:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:18.0:1616513076.068229:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2f980 x1688664289653824/t0(0) o400->d248c8ab-7b19-b3d1-9637-637d1d1e929e@10.31.136.9@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513076.068229:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513076.068231:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513076.068234:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.78@tcp1 : PUT - for me 00000100:00000200:18.0:1616513076.068242:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664289653824, offset 192 00000800:00000200:2.0:1616513076.068244:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513076.068245:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.068246:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.78@tcp1 of length 224 into portal 26 MB=0x5ffdd8930f800 00000400:00000200:18.0:1616513076.068248:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.9@tcp1 00000400:00000200:14.0:1616513076.068252:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.78@tcp1 of length 224/224 into md 0xf97c68c9 [1] + 0 00000400:00000200:26.0:1616513076.068254:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.78@tcp1 : PUT - for me 00000400:00000200:18.0:1616513076.068255:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.9@tcp1 routed destination 00000400:00000200:14.0:1616513076.068256:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2ce58 00000400:00000200:14.0:1616513076.068257:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.068260:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.068263:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2ce58 00000400:00000200:26.0:1616513076.068265:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.78@tcp1 of length 224 into portal 12 MB=0x5ffdd8930f880 00000800:00000200:14.0:1616513076.068265:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.068267:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.068268:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513076.068269:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.9@tcp1(10.31.136.9@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513076.068271:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.78@tcp1 of length 224/224 into md 0xf96450b9 [1] + 57360 00000800:00000200:18.0:1616513076.068276:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513076.068277:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:15.0:1616513076.068277:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701838096384 00000100:00000200:26.0:1616513076.068279:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513076.068281:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:15.0:1616513076.068283:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688701838096384/t0(0) o400->e868a9b5-13c8-c5c1-e689-4832d75b2d2c@10.31.142.78@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513076.068284:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513076.068286:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513076.068286:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513076.068289:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513076.068291:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513076.068292:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513076.068294:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701838096384, offset 192 00000400:00000200:15.0:1616513076.068299:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.78@tcp1 00000100:00000200:18.0:1616513076.068300:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701838096512 00000400:00000200:15.0:1616513076.068305:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.78@tcp1 routed destination 00010000:00000200:18.0:1616513076.068306:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dde80 x1688701838096512/t0(0) o400->9fb1e898-94e9-b1f1-bae3-f3701134df54@10.31.142.78@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513076.068316:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701838096512, offset 192 00000400:00000200:15.0:1616513076.068319:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.78@tcp1(10.31.142.78@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513076.068322:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.78@tcp1 00000800:00000200:15.0:1616513076.068324:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513076.068327:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.78@tcp1 routed destination 00000800:00000200:15.0:1616513076.068329:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513076.068332:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513076.068333:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513076.068337:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513076.068339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513076.068341:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.78@tcp1(10.31.142.78@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:15.0:1616513076.068341:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513076.068341:0:49023:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513076.068346:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513076.068349:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513076.068350:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513076.068353:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513076.068353:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513076.068354:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:22.0:1616513076.068357:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513076.068357:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:2.0:1616513076.068357:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.068361:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.068365:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7dc0 00000800:00000200:22.0:1616513076.068368:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513076.068396:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513076.068402:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513076.068405:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.068412:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.068416:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdd38 00000800:00000200:28.0:1616513076.068420:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513076.068424:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.068430:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513076.068432:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513076.068449:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513076.068449:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:28.0:1616513076.068456:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513076.068457:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513076.068459:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.068460:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513076.068467:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.068471:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd650 00000800:00000200:12.0:1616513076.068477:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:21.0:1616513076.090701:0:6489:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725924992, portal 4 00000100:00000200:21.0:1616513076.090707:0:6489:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725924992, offset 0 00000400:00000200:21.0:1616513076.090714:0:6489:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:21.0:1616513076.090720:0:6489:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:21.0:1616513076.090733:0:6489:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:21.0:1616513076.090740:0:6489:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:21.0:1616513076.090746:0:6489:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000100:00000200:9.0:1616513076.090748:0:6491:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925008, portal 4 00000800:00000200:21.0:1616513076.090751:0:6489:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:21.0:1616513076.090753:0:6489:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:9.0:1616513076.090753:0:6491:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925008, offset 0 00000800:00000200:21.0:1616513076.090759:0:6489:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:9.0:1616513076.090761:0:6491:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:9.0:1616513076.090766:0:6491:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:5.2:1616513076.090767:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:9.0:1616513076.090779:0:6491:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:9.0:1616513076.090786:0:6491:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:9.0:1616513076.090792:0:6491:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:9.0:1616513076.090795:0:6491:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:9.0:1616513076.090797:0:6491:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:9.0:1616513076.090802:0:6491:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:17.0:1616513076.090812:0:6490:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925024, portal 4 00000100:00000200:17.0:1616513076.090818:0:6490:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925024, offset 0 00000800:00000200:28.0:1616513076.090823:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:17.0:1616513076.090824:0:6490:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:17.0:1616513076.090830:0:6490:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:28.0:1616513076.090831:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513076.090834:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.090841:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:17.0:1616513076.090843:0:6490:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513076.090848:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d853e8cbf00 x1693067725924992/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.2:1616513076.090850:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:17.0:1616513076.090850:0:6490:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:17.0:1616513076.090857:0:6490:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513076.090860:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf516d8 00000800:00000200:17.0:1616513076.090860:0:6490:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:17.0:1616513076.090862:0:6490:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:28.0:1616513076.090865:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513076.090867:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:17.0:1616513076.090867:0:6490:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:27.0:1616513076.090872:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:1.2:1616513076.090876:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:27.0:1616513076.090880:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513076.090882:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513076.090889:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:1.0:1616513076.090890:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513076.090895:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000100:00000200:27.0:1616513076.090896:0:6476:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1395100 x1693067725925008/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513076.090898:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:9.0:1616513076.090899:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513076.090905:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:27.0:1616513076.090908:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35760 00000800:00000200:27.0:1616513076.090912:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:1.0:1616513076.090914:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cab102400 x1693067725925024/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513076.090915:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:1.0:1616513076.090926:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3f8f8 00000800:00000200:1.0:1616513076.090930:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:5.2:1616513076.091001:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:27.2:1616513076.091019:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513076.091028:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513076.091034:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513076.091038:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513076.091042:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513076.091050:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626a90 00000800:00000200:12.0:1616513076.091056:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:27.0:1616513076.091057:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371661cd [1] + 192 00000400:00000200:27.0:1616513076.091062:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513076.091066:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000100:00000200:27.0:1616513076.091067:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1395100 x1693067725925008/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513076.091076:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:1.2:1616513076.091078:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:27.0:1616513076.091080:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513076.091083:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:12.0:1616513076.091084:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626a80 00000800:00000200:27.0:1616513076.091085:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:1.0:1616513076.091087:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:9.0:1616513076.091088:0:6491:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1395100 x1693067725925008/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513076.091091:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d4371 [1] + 192 00000800:00000200:1.0:1616513076.091093:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:12.0:1616513076.091096:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:5.0:1616513076.091096:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:12.0:1616513076.091102:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d853e8cbf00 x1693067725924992/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:9.0:1616513076.091102:0:6491:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1395100 x1693067725925008/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513076.091102:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:16.0:1616513076.091111:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:9.0:1616513076.091111:0:6491:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35650 00000400:00000200:1.0:1616513076.091111:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626aa0 00000800:00000200:12.0:1616513076.091117:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:1.0:1616513076.091117:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffab5 [1] + 192 00000800:00000200:12.0:1616513076.091120:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513076.091122:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000400:00000200:1.0:1616513076.091122:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:21.0:1616513076.091126:0:6489:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d853e8cbf00 x1693067725924992/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513076.091127:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cab102400 x1693067725925024/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:21.0:1616513076.091139:0:6489:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d853e8cbf00 x1693067725924992/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513076.091140:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513076.091143:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:17.0:1616513076.091145:0:6490:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cab102400 x1693067725925024/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513076.091145:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000400:00000200:21.0:1616513076.091150:0:6489:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51650 00000100:00000200:17.0:1616513076.091158:0:6490:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cab102400 x1693067725925024/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:17.0:1616513076.091168:0:6490:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3f980 00000100:00000200:21.0:1616513076.091517:0:6492:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925040, portal 4 00000100:00000200:21.0:1616513076.091520:0:6492:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925040, offset 0 00000400:00000200:21.0:1616513076.091525:0:6492:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:21.0:1616513076.091530:0:6492:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:21.0:1616513076.091542:0:6492:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:21.0:1616513076.091547:0:6492:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:21.0:1616513076.091553:0:6492:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513076.091556:0:6492:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:21.0:1616513076.091558:0:6492:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:21.0:1616513076.091561:0:6492:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513076.091608:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513076.091624:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513076.091632:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513076.091635:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.091642:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513076.091648:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdd68b180 x1693067725925040/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513076.091672:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50bb0 00000800:00000200:14.0:1616513076.091677:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513076.091680:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513076.091781:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513076.091836:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513076.091847:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:2.0:1616513076.091856:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513076.091864:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626ab0 00000400:00000200:2.0:1616513076.091871:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c6f79 [1] + 192 00000400:00000200:2.0:1616513076.091876:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.091881:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdd68b180 x1693067725925040/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513076.091889:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513076.091897:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513076.091900:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513076.091902:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:21.0:1616513076.091903:0:6492:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdd68b180 x1693067725925040/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:21.0:1616513076.091914:0:6492:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdd68b180 x1693067725925040/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:21.0:1616513076.091923:0:6492:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51650 00000800:00000200:7.2:1616513076.153521:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513076.153578:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.153589:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.153599:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.131@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.153609:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.131@tcp1 of length 224 into portal 26 MB=0x6020e5aa8af00 00000400:00000200:14.0:1616513076.153616:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.131@tcp1 of length 224/224 into md 0xf97c68d9 [1] + 0 00000400:00000200:14.0:1616513076.153621:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2ccc0 00000400:00000200:14.0:1616513076.153624:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.153627:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.153632:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2ccc0 00000800:00000200:26.0:1616513076.153634:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.153636:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.153639:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.153641:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513076.153697:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691110534065920 00010000:00000200:15.0:1616513076.153708:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1691110534065920/t0(0) o400->348c687f-5ac7-587f-0017-4cb429bbd7fd@10.31.129.131@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513076.153722:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691110534065920, offset 192 00000400:00000200:15.0:1616513076.153728:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.131@tcp1 00000400:00000200:15.0:1616513076.153735:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.131@tcp1 routed destination 00000400:00000200:15.0:1616513076.153749:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.131@tcp1(10.31.129.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513076.153755:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513076.153756:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513076.153762:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513076.153764:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513076.153766:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513076.153771:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.153776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513076.153778:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513076.153814:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.153824:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513076.153832:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513076.153833:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.131@tcp1 : PUT - for me 00000800:00000200:22.0:1616513076.153841:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513076.153842:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.131@tcp1 of length 224 into portal 12 MB=0x6020e5aa8b300 00000400:00000200:28.0:1616513076.153849:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.131@tcp1 of length 224/224 into md 0x865b9f31 [1] + 90792 00000400:00000200:22.0:1616513076.153851:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.153855:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513076.153858:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:22.0:1616513076.153858:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.153865:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6880 00000800:00000200:28.0:1616513076.153867:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.153867:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513076.153869:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513076.153870:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513076.153872:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513076.153872:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.153938:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691110534066944 00010000:00000200:0.0:1616513076.153954:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf80000 x1691110534066944/t0(0) o400->bf486cf5-4c36-5b73-f446-f07deae61960@10.31.129.131@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513076.153970:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691110534066944, offset 192 00000400:00000200:0.0:1616513076.153976:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.131@tcp1 00000400:00000200:0.0:1616513076.153984:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.131@tcp1 routed destination 00000400:00000200:0.0:1616513076.154000:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.131@tcp1(10.31.129.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.154007:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.154013:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.154016:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.154019:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.154024:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.154029:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.154085:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.154092:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.154095:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513076.154097:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513076.154101:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513076.154105:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78e58 00000800:00000200:16.0:1616513076.154112:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.155341:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513076.155391:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513076.155400:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.155409:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.155418:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.42@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.155428:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.42@tcp1 of length 224 into portal 12 MB=0x601837910f400 00000400:00000200:20.0:1616513076.155448:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.42@tcp1 of length 224/224 into md 0x865b9f31 [1] + 91016 00000400:00000200:20.0:1616513076.155454:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513076.155456:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513076.155456:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513076.155461:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.155463:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513076.155465:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513076.155466:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.155468:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513076.155474:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.171@tcp1 : PUT - for me 00000400:00000200:26.0:1616513076.155484:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.171@tcp1 of length 224 into portal 26 MB=0x603e4516a8dc0 00000400:00000200:26.0:1616513076.155491:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.171@tcp1 of length 224/224 into md 0xf97c68f9 [1] + 0 00000400:00000200:26.0:1616513076.155496:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c110 00000400:00000200:26.0:1616513076.155498:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513076.155501:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513076.155506:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c110 00000800:00000200:26.0:1616513076.155509:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:0.0:1616513076.155511:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514043761664 00000800:00000200:26.0:1616513076.155512:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513076.155513:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.155516:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513076.155523:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06c00 x1690514043761664/t0(0) o400->f36702c4-1238-5e0e-b3a3-915f9928aea8@10.31.138.42@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513076.155526:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.155535:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.171@tcp1 : PUT - for me 00000100:00000200:0.0:1616513076.155538:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514043761664, offset 192 00000400:00000200:14.0:1616513076.155543:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.171@tcp1 of length 224 into portal 12 MB=0x603e4516a8ec0 00000400:00000200:0.0:1616513076.155544:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.42@tcp1 00000400:00000200:14.0:1616513076.155550:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.171@tcp1 of length 224/224 into md 0xf96450b9 [1] + 57584 00000400:00000200:0.0:1616513076.155552:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.42@tcp1 routed destination 00000400:00000200:14.0:1616513076.155555:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.155558:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513076.155568:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.42@tcp1(10.31.138.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:15.0:1616513076.155570:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693129013628352 00000800:00000200:22.0:1616513076.155573:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.155573:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513076.155574:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513076.155576:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513076.155580:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513076.155580:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:15.0:1616513076.155582:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1693129013628352/t0(0) o400->9d495832-b1be-6d3d-8396-347d49f686b7@10.31.138.171@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513076.155583:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513076.155583:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.155584:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.155585:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513076.155590:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.155592:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.42@tcp1 : PUT - for me 00000800:00000200:7.2:1616513076.155595:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:15.0:1616513076.155596:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693129013628352, offset 192 00000400:00000200:22.0:1616513076.155602:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.42@tcp1 of length 224 into portal 26 MB=0x601837910f380 00000400:00000200:15.0:1616513076.155602:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.171@tcp1 00000800:00000200:26.0:1616513076.155605:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:22.0:1616513076.155609:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.42@tcp1 of length 224/224 into md 0xf97c6919 [1] + 0 00000800:00000200:26.0:1616513076.155610:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:15.0:1616513076.155610:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.171@tcp1 routed destination 00000400:00000200:26.0:1616513076.155612:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.155614:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c660 00000800:00000200:14.0:1616513076.155614:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.155617:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513076.155619:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:22.0:1616513076.155620:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513076.155622:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79540 00000800:00000200:26.0:1616513076.155625:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.155625:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c660 00000400:00000200:15.0:1616513076.155626:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.171@tcp1(10.31.138.171@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513076.155629:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.155632:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.155633:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:15.0:1616513076.155633:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:18.0:1616513076.155634:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693129013628608 00000800:00000200:15.0:1616513076.155639:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513076.155642:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513076.155644:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.155644:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514043761536 00010000:00000200:18.0:1616513076.155649:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e5e80 x1693129013628608/t0(0) o400->df84204e-510e-66fc-64b3-68e318e0af22@10.31.138.171@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513076.155649:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:0.0:1616513076.155650:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1690514043761536/t0(0) o400->ecd8b24d-2010-af2d-07f4-1f52affcd4fa@10.31.138.42@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513076.155655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513076.155657:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513076.155662:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514043761536, offset 192 00000100:00000200:18.0:1616513076.155666:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693129013628608, offset 192 00000800:00000200:12.0:1616513076.155666:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513076.155667:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.42@tcp1 00000400:00000200:18.0:1616513076.155672:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.171@tcp1 00000800:00000200:12.0:1616513076.155673:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513076.155673:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.42@tcp1 routed destination 00000400:00000200:12.0:1616513076.155677:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513076.155680:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.171@tcp1 routed destination 00000400:00000200:12.0:1616513076.155683:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.155689:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7320 00000800:00000200:12.0:1616513076.155693:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513076.155695:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.171@tcp1(10.31.138.171@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513076.155696:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513076.155702:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513076.155709:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.42@tcp1(10.31.138.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.155715:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513076.155716:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.155719:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.155720:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.155721:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.155722:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513076.155727:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513076.155727:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:0.0:1616513076.155730:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:7.2:1616513076.155732:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:0.0:1616513076.155736:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513076.155740:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513076.155744:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513076.155746:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513076.155747:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513076.155751:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513076.155752:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513076.155755:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.155755:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cddc0 00000800:00000200:2.0:1616513076.155757:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513076.155760:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:14.0:1616513076.155761:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513076.155764:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b794b8 00000800:00000200:26.0:1616513076.155766:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.171085:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513076.171104:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.171113:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513076.171122:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.191@tcp1 : PUT - for me 00000400:00000200:28.0:1616513076.171132:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.191@tcp1 of length 224 into portal 26 MB=0x6001cfde62b40 00000400:00000200:28.0:1616513076.171138:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.191@tcp1 of length 224/224 into md 0xf97c6939 [1] + 0 00000400:00000200:28.0:1616513076.171143:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2cd48 00000400:00000200:28.0:1616513076.171146:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513076.171150:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513076.171155:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cd48 00000800:00000200:20.0:1616513076.171157:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.171158:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.171161:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.171163:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.171219:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688974379068224 00000800:00000200:7.2:1616513076.171221:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:0.0:1616513076.171230:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688974379068224/t0(0) o400->216a9498-45e6-ca7e-43b6-a5dc88c3b71f@10.31.141.191@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.171245:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688974379068224, offset 192 00000400:00000200:0.0:1616513076.171252:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.191@tcp1 00000400:00000200:0.0:1616513076.171260:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.191@tcp1 routed destination 00000400:00000200:0.0:1616513076.171275:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.191@tcp1(10.31.141.191@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.171282:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513076.171286:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.171288:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.171291:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.171293:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513076.171296:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.171298:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.171303:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513076.171305:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.191@tcp1 : PUT - for me 00000100:00000200:0.0:1616513076.171305:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513076.171313:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.191@tcp1 of length 224 into portal 12 MB=0x6001cfde62bc0 00000400:00000200:22.0:1616513076.171320:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.191@tcp1 of length 224/224 into md 0xf96450b9 [1] + 57808 00000400:00000200:22.0:1616513076.171325:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.171328:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513076.171335:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.171338:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.171341:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.171345:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513076.171362:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.171369:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.171372:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513076.171379:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.171385:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78dd0 00000800:00000200:12.0:1616513076.171388:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.171391:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513076.171403:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688974379068352 00010000:00000200:18.0:1616513076.171415:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2e780 x1688974379068352/t0(0) o400->f69cbfd0-d556-0ad5-5424-68f4f8ca8605@10.31.141.191@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513076.171448:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688974379068352, offset 192 00000400:00000200:18.0:1616513076.171454:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.191@tcp1 00000400:00000200:18.0:1616513076.171471:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.191@tcp1 routed destination 00000400:00000200:18.0:1616513076.171486:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.191@tcp1(10.31.141.191@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513076.171492:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513076.171498:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.171501:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513076.171503:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513076.171508:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.171512:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.171566:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.171573:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.171576:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.171582:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.171587:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc880 00000800:00000200:14.0:1616513076.171590:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.171593:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.175117:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.175171:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.175180:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.175189:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.246@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.175197:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.246@tcp1 of length 224 into portal 26 MB=0x6000dc96f1340 00000400:00000200:2.0:1616513076.175212:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.246@tcp1 of length 224/224 into md 0xf97c6949 [1] + 0 00000400:00000200:2.0:1616513076.175217:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d8f8 00000400:00000200:2.0:1616513076.175219:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.175222:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513076.175225:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513076.175226:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d8f8 00000800:00000200:2.0:1616513076.175229:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513076.175232:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513076.175234:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.175235:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513076.175236:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.175242:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513076.175243:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.61@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.175249:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.246@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.175252:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.61@tcp1 of length 224 into portal 12 MB=0x6019921309340 00000400:00000200:2.0:1616513076.175255:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.246@tcp1 of length 224 into portal 12 MB=0x6000dc96f1500 00000400:00000200:22.0:1616513076.175258:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.61@tcp1 of length 224/224 into md 0xf96450b9 [1] + 58032 00000400:00000200:22.0:1616513076.175262:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513076.175264:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.246@tcp1 of length 224/224 into md 0xf96450b9 [1] + 58256 00000400:00000200:2.0:1616513076.175268:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.175269:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513076.175274:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000100:00000200:2.0:1616513076.175274:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513076.175277:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513076.175279:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.175280:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513076.175282:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.175282:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.175283:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.175289:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688909074338624 00010000:00000200:0.0:1616513076.175299:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688909074338624/t0(0) o400->d111cf5e-2454-5d5d-1c21-3547f037df3e@10.31.141.246@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513076.175305:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:2.0:1616513076.175307:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690607058719552 00000100:00000200:30.0:1616513076.175308:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688909074339072 00000100:00000200:0.0:1616513076.175313:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688909074338624, offset 192 00010000:00000200:2.0:1616513076.175318:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2a880 x1690607058719552/t0(0) o400->10fa7a34-ba16-dc0d-9843-704867fdb861@10.31.141.61@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:30.0:1616513076.175319:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120ba80 x1688909074339072/t0(0) o400->29ddc586-4706-6b02-0349-b9110c7030ee@10.31.141.246@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513076.175320:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.246@tcp1 00000400:00000200:0.0:1616513076.175327:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.246@tcp1 routed destination 00000100:00000200:2.0:1616513076.175331:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690607058719552, offset 192 00000100:00000200:30.0:1616513076.175333:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688909074339072, offset 192 00000400:00000200:2.0:1616513076.175337:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.61@tcp1 00000400:00000200:0.0:1616513076.175341:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.246@tcp1(10.31.141.246@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513076.175342:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.246@tcp1 00000400:00000200:2.0:1616513076.175344:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.61@tcp1 routed destination 00000800:00000200:0.0:1616513076.175347:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.175352:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.175355:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.175357:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.175358:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513076.175359:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.61@tcp1(10.31.141.61@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.175362:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513076.175365:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:30.0:1616513076.175367:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.246@tcp1 routed destination 00000800:00000200:20.0:1616513076.175368:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513076.175368:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513076.175374:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513076.175377:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.175378:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:30.0:1616513076.175381:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.246@tcp1(10.31.141.246@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513076.175382:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.175386:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513076.175387:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.175390:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.61@tcp1 : PUT - for me 00000800:00000200:30.0:1616513076.175393:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513076.175395:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:22.0:1616513076.175396:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513076.175397:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:22.0:1616513076.175399:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513076.175400:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.61@tcp1 of length 224 into portal 26 MB=0x60199213092c0 00000400:00000200:22.0:1616513076.175401:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513076.175402:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:2.0:1616513076.175404:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.175406:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.175406:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.61@tcp1 of length 224/224 into md 0xf97c6959 [1] + 0 00000800:00000200:28.0:1616513076.175411:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000400:00000200:20.0:1616513076.175411:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d4b8 00000400:00000200:22.0:1616513076.175414:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eedd38 00000400:00000200:20.0:1616513076.175414:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513076.175417:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513076.175417:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513076.175419:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (25)-- 00000400:00000200:28.0:1616513076.175421:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.175422:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d4b8 00000800:00000200:20.0:1616513076.175425:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000400:00000200:28.0:1616513076.175428:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513076.175428:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:20.0:1616513076.175429:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513076.175432:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:0.0:1616513076.175443:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690607058719424 00000400:00000200:28.0:1616513076.175447:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78330 00000800:00000200:20.0:1616513076.175447:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00010000:00000200:0.0:1616513076.175449:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1690607058719424/t0(0) o400->1afa5b3c-cdd5-5c28-202b-66be809be02b@10.31.141.61@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513076.175450:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513076.175451:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513076.175454:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513076.175455:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.175459:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee440 00000100:00000200:0.0:1616513076.175460:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690607058719424, offset 192 00000800:00000200:20.0:1616513076.175462:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513076.175465:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.61@tcp1 00000400:00000200:0.0:1616513076.175471:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.61@tcp1 routed destination 00000400:00000200:0.0:1616513076.175483:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.61@tcp1(10.31.141.61@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.175489:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.175493:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.175496:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.175497:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.175500:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.175505:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.175505:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513076.175516:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.175523:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.175526:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513076.175527:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513076.175532:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.175537:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78ff0 00000800:00000200:14.0:1616513076.175540:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.187449:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513076.187515:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.187524:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513076.187533:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.104@tcp1 : PUT - for me 00000400:00000200:28.0:1616513076.187542:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.104@tcp1 of length 224 into portal 26 MB=0x602156fc28cc0 00000400:00000200:28.0:1616513076.187549:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.104@tcp1 of length 224/224 into md 0xf97c6969 [1] + 0 00000400:00000200:28.0:1616513076.187554:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d188 00000400:00000200:28.0:1616513076.187557:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513076.187560:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513076.187564:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d188 00000800:00000200:28.0:1616513076.187567:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.187569:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.187570:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513076.187573:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.187629:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691140952853696 00010000:00000200:0.0:1616513076.187640:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1691140952853696/t0(0) o400->708102af-3a53-5d2c-7b68-702ada605d8f@10.31.140.104@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.187654:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691140952853696, offset 192 00000400:00000200:0.0:1616513076.187661:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.104@tcp1 00000400:00000200:0.0:1616513076.187668:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.104@tcp1 routed destination 00000400:00000200:0.0:1616513076.187683:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.104@tcp1(10.31.140.104@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.187694:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.187700:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.187703:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.187705:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.187710:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.187715:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.187717:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513076.187769:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.187776:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.187778:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.187785:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.187790:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b795c8 00000800:00000200:20.0:1616513076.187794:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.187797:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.187890:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513076.187900:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.187909:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513076.187910:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513076.187918:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.145@tcp1 : PUT - for me 00000400:00000200:12.0:1616513076.187934:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.145@tcp1 of length 224 into portal 26 MB=0x5ffd7241ea2c0 00000400:00000200:12.0:1616513076.187941:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.145@tcp1 of length 224/224 into md 0xf97c6989 [1] + 0 00000400:00000200:12.0:1616513076.187948:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c2a8 00000400:00000200:12.0:1616513076.187951:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.187953:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513076.187958:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c2a8 00000800:00000200:12.0:1616513076.187961:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.187964:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.187966:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.187970:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674372592320 00010000:00000200:0.0:1616513076.187975:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688674372592320/t0(0) o400->26775e76-808a-4b83-3ae7-497772562474@10.31.129.145@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.187986:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688674372592320, offset 192 00000400:00000200:0.0:1616513076.187991:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.145@tcp1 00000400:00000200:0.0:1616513076.187996:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.145@tcp1 routed destination 00000800:00000200:7.2:1616513076.188003:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513076.188008:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.145@tcp1(10.31.129.145@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.188013:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.188017:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.188020:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.188021:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513076.188024:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:0.0:1616513076.188029:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513076.188056:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513076.188058:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513076.188066:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513076.188067:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513076.188071:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513076.188075:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513076.188076:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.145@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.188078:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.104@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.188085:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.145@tcp1 of length 224 into portal 12 MB=0x5ffd7241ea780 00000400:00000200:20.0:1616513076.188086:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.104@tcp1 of length 224 into portal 12 MB=0x602156fc29300 00000400:00000200:20.0:1616513076.188091:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.104@tcp1 of length 224/224 into md 0x865b9f31 [1] + 91240 00000400:00000200:2.0:1616513076.188091:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.145@tcp1 of length 224/224 into md 0xf96450b9 [1] + 58480 00000400:00000200:20.0:1616513076.188096:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513076.188097:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.188099:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:2.0:1616513076.188099:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.188104:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.188107:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.188108:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513076.188108:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513076.188111:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513076.188111:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513076.188117:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513076.188118:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000100:00000200:0.0:1616513076.188118:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691140952855296 00000100:00000200:30.0:1616513076.188120:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674372593536 00000400:00000200:22.0:1616513076.188121:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513076.188121:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513076.188124:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01200 x1691140952855296/t0(0) o400->9ba797d0-939f-1ad8-b7ce-0bfa5174aecc@10.31.140.104@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:22.0:1616513076.188127:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:30.0:1616513076.188129:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464624c80 x1688674372593536/t0(0) o400->5d936dc2-ddcb-54de-cfac-26c2377dc1fc@10.31.129.145@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:22.0:1616513076.188133:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78b28 00000100:00000200:0.0:1616513076.188135:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691140952855296, offset 192 00000800:00000200:22.0:1616513076.188136:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513076.188139:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.104@tcp1 00000100:00000200:30.0:1616513076.188140:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688674372593536, offset 192 00000400:00000200:0.0:1616513076.188144:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.104@tcp1 routed destination 00000400:00000200:30.0:1616513076.188146:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.145@tcp1 00000400:00000200:30.0:1616513076.188151:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.145@tcp1 routed destination 00000400:00000200:0.0:1616513076.188156:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.104@tcp1(10.31.140.104@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.188162:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:30.0:1616513076.188164:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.145@tcp1(10.31.129.145@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.188166:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.188168:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513076.188170:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.188170:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.188173:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513076.188174:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513076.188177:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513076.188178:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513076.188178:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513076.188182:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513076.188186:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513076.188187:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513076.188190:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.188192:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513076.188194:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513076.188197:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513076.188198:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513076.188200:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78ff0 00000800:00000200:28.0:1616513076.188203:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513076.188204:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.188207:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513076.188208:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513076.188213:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.188217:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef8f8 00000800:00000200:14.0:1616513076.188221:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.192947:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513076.193010:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.193019:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.193032:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.95@tcp1 : PUT - for me 00000800:00000200:20.0:1616513076.193037:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513076.193040:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.95@tcp1 of length 224 into portal 26 MB=0x6002559152180 00000400:00000200:16.0:1616513076.193047:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.95@tcp1 of length 224/224 into md 0xf97c6999 [1] + 0 00000400:00000200:16.0:1616513076.193051:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c198 00000400:00000200:16.0:1616513076.193054:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.193057:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513076.193061:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c198 00000800:00000200:16.0:1616513076.193064:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.193067:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.193069:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.193127:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689010268610944 00010000:00000200:0.0:1616513076.193137:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1689010268610944/t0(0) o400->e0e903d0-6d7c-4e40-cd33-884d541e6152@10.31.142.95@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.193150:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689010268610944, offset 192 00000400:00000200:0.0:1616513076.193156:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.95@tcp1 00000400:00000200:0.0:1616513076.193163:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.95@tcp1 routed destination 00000400:00000200:0.0:1616513076.193177:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.95@tcp1(10.31.142.95@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.193183:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.193188:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.193191:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.193193:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.193197:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.193202:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.193203:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513076.193254:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.193261:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.193264:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513076.193268:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513076.193270:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.193275:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78330 00000800:00000200:12.0:1616513076.193279:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.193379:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513076.193390:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.193397:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513076.193399:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513076.193406:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.95@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.193414:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.95@tcp1 of length 224 into portal 12 MB=0x6002559152680 00000400:00000200:20.0:1616513076.193420:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.95@tcp1 of length 224/224 into md 0x865b9f31 [1] + 91464 00000400:00000200:20.0:1616513076.193424:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.193427:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.193445:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.193448:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.193449:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.193458:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689010268612224 00010000:00000200:0.0:1616513076.193465:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01b00 x1689010268612224/t0(0) o400->d4cdf783-5cdb-2a41-85aa-b37c5a682df7@10.31.142.95@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513076.193476:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689010268612224, offset 192 00000400:00000200:0.0:1616513076.193481:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.95@tcp1 00000400:00000200:0.0:1616513076.193486:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.95@tcp1 routed destination 00000400:00000200:0.0:1616513076.193498:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.95@tcp1(10.31.142.95@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.193503:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.193508:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.193510:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.193511:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.193515:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.193520:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.193572:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.193579:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.193581:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.193587:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.193591:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b794b8 00000800:00000200:22.0:1616513076.193595:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.193597:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.210346:0:49037:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.210402:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.210424:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.210432:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.9@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.210456:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.9@tcp1 of length 224 into portal 26 MB=0x604b852f03540 00000400:00000200:2.0:1616513076.210462:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.9@tcp1 of length 224/224 into md 0xf97c69b9 [1] + 0 00000800:00000200:14.0:1616513076.210466:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513076.210466:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c000 00000400:00000200:2.0:1616513076.210469:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.210472:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513076.210476:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c000 00000800:00000200:14.0:1616513076.210477:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513076.210479:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513076.210482:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513076.210484:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.210485:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.9@tcp1 : PUT - for me 00000800:00000200:2.0:1616513076.210487:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513076.210493:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.9@tcp1 of length 224 into portal 12 MB=0x604b852f035c0 00000400:00000200:14.0:1616513076.210499:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.9@tcp1 of length 224/224 into md 0xf96450b9 [1] + 58704 00000400:00000200:14.0:1616513076.210504:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.210507:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513076.210514:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.210517:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.210518:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.210558:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694039572231488 00010000:00000200:0.0:1616513076.210568:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1694039572231488/t0(0) o400->988377aa-5de0-4eb2-b5c3-33cdd33a06bf@10.31.141.9@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513076.210578:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694039572231616 00000100:00000200:0.0:1616513076.210581:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694039572231488, offset 192 00000400:00000200:0.0:1616513076.210589:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.9@tcp1 00010000:00000200:30.0:1616513076.210590:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e6300 x1694039572231616/t0(0) o400->589b6fb0-c3c1-a1c7-5a5c-8db146e45529@10.31.141.9@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513076.210596:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.9@tcp1 routed destination 00000100:00000200:30.0:1616513076.210603:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694039572231616, offset 192 00000400:00000200:30.0:1616513076.210609:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.9@tcp1 00000400:00000200:0.0:1616513076.210610:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.9@tcp1(10.31.141.9@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513076.210616:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.9@tcp1 routed destination 00000800:00000200:0.0:1616513076.210616:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.210621:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.210624:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.210626:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513076.210629:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.9@tcp1(10.31.141.9@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.210630:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513076.210635:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:0.0:1616513076.210637:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513076.210638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513076.210641:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513076.210644:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513076.210646:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513076.210650:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.210655:0:49037:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513076.210692:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.210699:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.210703:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.210709:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513076.210714:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513076.210715:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79540 00000800:00000200:28.0:1616513076.210718:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.210721:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513076.210722:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513076.210724:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513076.210725:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513076.210731:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513076.210735:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee088 00000800:00000200:26.0:1616513076.210739:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.211535:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513076.211590:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.211600:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.211609:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.123@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.211618:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.123@tcp1 of length 224 into portal 26 MB=0x60184ce05b680 00000400:00000200:16.0:1616513076.211624:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.123@tcp1 of length 224/224 into md 0xf97c69c9 [1] + 0 00000400:00000200:16.0:1616513076.211629:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c990 00000400:00000200:16.0:1616513076.211632:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.211635:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513076.211640:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c990 00000800:00000200:16.0:1616513076.211643:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.211646:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.211646:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.211651:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.211653:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519764055680 00010000:00000200:0.0:1616513076.211659:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1690519764055680/t0(0) o400->5a0361b8-8533-46b9-4024-003a8d5b6b1d@10.31.142.123@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.211670:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690519764055680, offset 192 00000400:00000200:0.0:1616513076.211675:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.123@tcp1 00000400:00000200:0.0:1616513076.211681:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.123@tcp1 routed destination 00000400:00000200:0.0:1616513076.211694:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.123@tcp1(10.31.142.123@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.211700:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.211705:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.211707:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.211709:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.211712:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513076.211716:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513076.211717:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.211727:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.211731:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.211732:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513076.211737:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513076.211738:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.211741:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79760 00000800:00000200:28.0:1616513076.211743:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.212148:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513076.212215:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.212225:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513076.212226:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.212234:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.123@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.212244:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.123@tcp1 of length 224 into portal 12 MB=0x60184ce05c400 00000400:00000200:22.0:1616513076.212251:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.123@tcp1 of length 224/224 into md 0xf96450b9 [1] + 58928 00000400:00000200:22.0:1616513076.212256:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.212259:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513076.212266:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.212269:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.212271:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513076.212330:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519764059136 00010000:00000200:2.0:1616513076.212343:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2b600 x1690519764059136/t0(0) o400->bf7cf039-2055-503b-26f4-23aec0f828c1@10.31.142.123@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513076.212357:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690519764059136, offset 192 00000400:00000200:2.0:1616513076.212363:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.123@tcp1 00000400:00000200:2.0:1616513076.212370:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.123@tcp1 routed destination 00000400:00000200:2.0:1616513076.212385:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.123@tcp1(10.31.142.123@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513076.212393:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513076.212399:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513076.212402:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.212404:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.212409:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.212415:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.212427:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.212431:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.212433:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513076.212448:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.212451:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec088 00000800:00000200:2.0:1616513076.212454:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.212457:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.216899:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513076.216955:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.216964:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.216974:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.93@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.216982:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.93@tcp1 of length 224 into portal 26 MB=0x60194b708acc0 00000400:00000200:20.0:1616513076.216991:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.93@tcp1 of length 224/224 into md 0xf97c69d9 [1] + 0 00000400:00000200:20.0:1616513076.216996:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2ddc0 00000400:00000200:20.0:1616513076.216998:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.217001:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513076.217005:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2ddc0 00000800:00000200:20.0:1616513076.217008:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.217010:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.217011:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513076.217014:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.217065:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588097850560 00010000:00000200:0.0:1616513076.217075:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1690588097850560/t0(0) o400->cc919787-7792-8889-17a7-4e816bbe2458@10.31.140.93@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.217087:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588097850560, offset 192 00000400:00000200:0.0:1616513076.217092:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.93@tcp1 00000400:00000200:0.0:1616513076.217099:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.93@tcp1 routed destination 00000400:00000200:0.0:1616513076.217111:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.93@tcp1(10.31.140.93@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.217117:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.217123:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.217126:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.217128:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.217132:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.217137:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.217138:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513076.217191:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.217197:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.217200:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513076.217206:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.217212:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78908 00000800:00000200:12.0:1616513076.217215:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.217218:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.217490:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513076.217542:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.217551:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.217559:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.93@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.217567:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.93@tcp1 of length 224 into portal 12 MB=0x60194b708b100 00000400:00000200:14.0:1616513076.217575:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.93@tcp1 of length 224/224 into md 0xf96450b9 [1] + 59152 00000400:00000200:14.0:1616513076.217580:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.217582:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513076.217588:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.217591:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.217593:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.217595:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513076.217656:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588097851648 00010000:00000200:2.0:1616513076.217668:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e6780 x1690588097851648/t0(0) o400->e6d63783-26fe-d56b-79eb-eb48ba647624@10.31.140.93@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513076.217683:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588097851648, offset 192 00000400:00000200:2.0:1616513076.217689:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.93@tcp1 00000400:00000200:2.0:1616513076.217696:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.93@tcp1 routed destination 00000400:00000200:2.0:1616513076.217711:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.93@tcp1(10.31.140.93@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513076.217717:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513076.217723:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513076.217726:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.217728:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.217733:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.217738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.217751:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.217755:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.217757:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513076.217763:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.217766:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec088 00000800:00000200:2.0:1616513076.217769:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.217771:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.220196:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513076.220249:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.220258:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.220267:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.180@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.220276:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.180@tcp1 of length 224 into portal 26 MB=0x604a2970d99c0 00000400:00000200:16.0:1616513076.220283:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.180@tcp1 of length 224/224 into md 0xf97c69e9 [1] + 0 00000400:00000200:16.0:1616513076.220289:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2cff0 00000400:00000200:16.0:1616513076.220292:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.220295:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513076.220299:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cff0 00000800:00000200:20.0:1616513076.220302:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.220302:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.220305:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.220309:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.220312:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.220313:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.220321:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.180@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.220330:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.180@tcp1 of length 224 into portal 12 MB=0x604a2970d9c40 00000400:00000200:20.0:1616513076.220336:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.180@tcp1 of length 224/224 into md 0x865b9f31 [1] + 91688 00000400:00000200:20.0:1616513076.220342:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.220344:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.220352:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.220355:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.220357:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.220382:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946225727936 00010000:00000200:0.0:1616513076.220392:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1693946225727936/t0(0) o400->ed591506-2e30-4ab4-b8dc-dec1d91f643b@10.31.140.180@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.220404:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946225727936, offset 192 00000400:00000200:0.0:1616513076.220410:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.180@tcp1 00000100:00000200:4.0:1616513076.220416:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946225728576 00000400:00000200:0.0:1616513076.220416:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.180@tcp1 routed destination 00010000:00000200:4.0:1616513076.220428:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01f80 x1693946225728576/t0(0) o400->0a4ea667-8303-acea-bda9-03a91703fa94@10.31.140.180@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513076.220429:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.180@tcp1(10.31.140.180@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.220446:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.220451:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.220453:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.220455:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513076.220456:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946225728576, offset 192 00000800:00000200:0.0:1616513076.220459:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513076.220462:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.180@tcp1 00000800:00000200:11.2:1616513076.220464:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.220466:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:4.0:1616513076.220469:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.180@tcp1 routed destination 00000400:00000200:4.0:1616513076.220483:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.180@tcp1(10.31.140.180@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.220489:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513076.220494:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.220497:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513076.220499:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513076.220503:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.220509:0:49038:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513076.220516:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.220524:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.220526:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513076.220528:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513076.220533:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.220538:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79430 00000800:00000200:28.0:1616513076.220542:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513076.220563:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.220571:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.220573:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513076.220576:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.220580:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.220584:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000800:00000200:22.0:1616513076.220588:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.230639:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513076.230694:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.230704:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.230713:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.65@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.230722:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.65@tcp1 of length 224 into portal 26 MB=0x5ffdb36d052c0 00000400:00000200:20.0:1616513076.230728:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.65@tcp1 of length 224/224 into md 0xf97c6a09 [1] + 0 00000400:00000200:20.0:1616513076.230733:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c440 00000400:00000200:20.0:1616513076.230736:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.230738:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513076.230743:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c440 00000800:00000200:20.0:1616513076.230747:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.230747:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.230749:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513076.230751:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.230807:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691866096320 00010000:00000200:0.0:1616513076.230818:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688691866096320/t0(0) o400->073ad6f1-62f9-f21b-8690-a20dd60219c0@10.31.141.65@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.230832:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688691866096320, offset 192 00000400:00000200:0.0:1616513076.230838:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.65@tcp1 00000400:00000200:0.0:1616513076.230845:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.65@tcp1 routed destination 00000400:00000200:0.0:1616513076.230860:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.65@tcp1(10.31.141.65@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.230876:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.230881:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.230883:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.230885:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513076.230887:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.230890:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.230894:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.230897:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513076.230958:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513076.230959:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513076.230965:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.230968:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513076.230969:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513076.230974:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.230978:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.65@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.230980:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78a18 00000800:00000200:16.0:1616513076.230984:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513076.230986:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.65@tcp1 of length 224 into portal 12 MB=0x5ffdb36d05540 00000800:00000200:16.0:1616513076.230987:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513076.230993:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.65@tcp1 of length 224/224 into md 0xf96450b9 [1] + 59376 00000400:00000200:2.0:1616513076.230998:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.231000:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513076.231008:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.231011:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.231013:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.231015:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513076.231065:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691866096960 00010000:00000200:6.0:1616513076.231074:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464626c00 x1688691866096960/t0(0) o400->25d80def-66d0-ade0-d8df-6c45084ca173@10.31.141.65@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513076.231085:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688691866096960, offset 192 00000400:00000200:6.0:1616513076.231089:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.65@tcp1 00000400:00000200:6.0:1616513076.231095:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.65@tcp1 routed destination 00000400:00000200:6.0:1616513076.231105:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.65@tcp1(10.31.141.65@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513076.231110:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:6.0:1616513076.231114:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513076.231118:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513076.231119:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513076.231123:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.231128:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.231141:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.231147:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.231150:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.231156:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.231159:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01a08 00000800:00000200:14.0:1616513076.231163:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.231165:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.233779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.233833:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.233842:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.233851:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.7@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.233859:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.7@tcp1 of length 224 into portal 12 MB=0x5ffd6ca008b00 00000400:00000200:2.0:1616513076.233866:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.7@tcp1 of length 224/224 into md 0xf96450b9 [1] + 59600 00000400:00000200:2.0:1616513076.233871:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.233874:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513076.233880:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.233883:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.233885:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513076.233887:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513076.233943:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672860670720 00010000:00000200:6.0:1616513076.233955:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623f00 x1688672860670720/t0(0) o400->2377887d-faf5-f170-b1be-5393c4a82e92@10.31.130.7@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513076.233970:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688672860670720, offset 192 00000400:00000200:6.0:1616513076.233976:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.7@tcp1 00000400:00000200:6.0:1616513076.233983:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.7@tcp1 routed destination 00000800:00000200:11.2:1616513076.233985:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:6.0:1616513076.233998:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.7@tcp1(10.31.130.7@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513076.234004:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:6.0:1616513076.234010:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513076.234013:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513076.234015:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:6.0:1616513076.234020:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.234039:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.234048:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.234057:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.7@tcp1 : PUT - for me 00000400:00000200:12.0:1616513076.234065:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.7@tcp1 of length 224 into portal 26 MB=0x5ffd6ca008a00 00000400:00000200:12.0:1616513076.234072:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.7@tcp1 of length 224/224 into md 0xf97c6a19 [1] + 0 00000400:00000200:12.0:1616513076.234077:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c770 00000400:00000200:12.0:1616513076.234079:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.234082:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513076.234086:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c770 00000800:00000200:12.0:1616513076.234089:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513076.234092:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:12.0:1616513076.234092:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513076.234099:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513076.234100:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513076.234102:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.234105:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.234111:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.234115:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00a18 00000800:00000200:20.0:1616513076.234119:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.234154:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672860670464 00010000:00000200:0.0:1616513076.234165:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688672860670464/t0(0) o400->f6c58715-3f5a-7050-98a9-809a4987ee07@10.31.130.7@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.234179:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688672860670464, offset 192 00000400:00000200:0.0:1616513076.234185:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.7@tcp1 00000400:00000200:0.0:1616513076.234192:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.7@tcp1 routed destination 00000400:00000200:0.0:1616513076.234207:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.7@tcp1(10.31.130.7@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.234214:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.234219:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.234222:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.234224:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.234229:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.234234:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.234236:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513076.234286:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.234293:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.234296:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.234303:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.234308:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79e48 00000800:00000200:14.0:1616513076.234312:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.234315:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.253883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.253939:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.253948:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.253957:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.178@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.253966:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.178@tcp1 of length 224 into portal 12 MB=0x5ffdc869b1000 00000400:00000200:2.0:1616513076.253972:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.178@tcp1 of length 224/224 into md 0xf96450b9 [1] + 59824 00000400:00000200:2.0:1616513076.253977:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.253979:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513076.253987:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.253990:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.253992:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.253994:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.254001:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.254007:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.178@tcp1 : PUT - for me 00000800:00000200:26.0:1616513076.254010:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513076.254012:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.178@tcp1 of length 224 into portal 26 MB=0x5ffdc869b0f40 00000400:00000200:2.0:1616513076.254017:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.178@tcp1 of length 224/224 into md 0xf97c6a39 [1] + 0 00000400:00000200:2.0:1616513076.254021:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c550 00000400:00000200:2.0:1616513076.254023:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.254025:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513076.254029:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c550 00000800:00000200:2.0:1616513076.254031:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.254033:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.254035:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513076.254070:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697499750400 00010000:00000200:10.0:1616513076.254082:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464626300 x1688697499750400/t0(0) o400->8530b69c-5993-263e-3dd3-5e27a7915d3d@10.31.138.178@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513076.254096:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697499750208 00000100:00000200:10.0:1616513076.254097:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697499750400, offset 192 00000400:00000200:10.0:1616513076.254103:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.178@tcp1 00010000:00000200:0.0:1616513076.254106:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688697499750208/t0(0) o400->176ad823-d63c-90a7-374b-3fe6a5124248@10.31.138.178@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:10.0:1616513076.254110:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.178@tcp1 routed destination 00000100:00000200:0.0:1616513076.254119:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697499750208, offset 192 00000400:00000200:10.0:1616513076.254125:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.178@tcp1(10.31.138.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513076.254125:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.178@tcp1 00000800:00000200:10.0:1616513076.254132:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513076.254134:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.178@tcp1 routed destination 00000800:00000200:10.0:1616513076.254137:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.254140:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513076.254142:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513076.254147:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513076.254148:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.178@tcp1(10.31.138.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.254154:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513076.254155:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.254160:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.254162:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.254164:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.254169:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.254174:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.254176:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513076.254209:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.254215:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.254218:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.254224:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.254228:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83c28 00000800:00000200:22.0:1616513076.254231:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513076.254232:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.254234:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513076.254238:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.254241:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513076.254242:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.254248:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.254253:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b783b8 00000800:00000200:22.0:1616513076.254256:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.258565:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.258623:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.258632:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.258641:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.209@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.258650:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.209@tcp1 of length 224 into portal 26 MB=0x5ffdf7e8ae200 00000400:00000200:2.0:1616513076.258657:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.209@tcp1 of length 224/224 into md 0xf97c6a89 [1] + 0 00000400:00000200:2.0:1616513076.258661:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c6e8 00000400:00000200:2.0:1616513076.258664:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.258667:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513076.258670:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c6e8 00000800:00000200:2.0:1616513076.258673:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.258676:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.258677:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513076.258684:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.258687:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.258688:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.258696:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.209@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.258705:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.209@tcp1 of length 224 into portal 12 MB=0x5ffdf7e8ae4c0 00000400:00000200:14.0:1616513076.258711:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.209@tcp1 of length 224/224 into md 0xf96450b9 [1] + 60048 00000400:00000200:14.0:1616513076.258717:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.258719:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513076.258726:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.258729:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.258731:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.258750:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688710249374208 00010000:00000200:0.0:1616513076.258760:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688710249374208/t0(0) o400->18a431c1-f5a9-4646-db09-e43ee3996c1c@10.31.141.209@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.258773:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688710249374208, offset 192 00000400:00000200:0.0:1616513076.258780:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.209@tcp1 00000400:00000200:0.0:1616513076.258787:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.209@tcp1 routed destination 00000100:00000200:10.0:1616513076.258793:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688710249374912 00000400:00000200:0.0:1616513076.258801:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.209@tcp1(10.31.141.209@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:10.0:1616513076.258805:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e6c00 x1688710249374912/t0(0) o400->ad99b1cb-3a4d-7535-7075-be527c2ef0d6@10.31.141.209@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513076.258807:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.258812:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.258814:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.258816:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:10.0:1616513076.258819:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688710249374912, offset 192 00000800:00000200:0.0:1616513076.258821:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513076.258825:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.209@tcp1 00000800:00000200:11.2:1616513076.258827:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.258827:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:10.0:1616513076.258832:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.209@tcp1 routed destination 00000400:00000200:10.0:1616513076.258847:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.209@tcp1(10.31.141.209@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.258853:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513076.258859:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.258862:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.258864:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.258868:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.258873:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513076.258897:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.258904:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.258907:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.258914:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.258919:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78990 00000800:00000200:20.0:1616513076.258923:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.258926:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513076.258929:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513076.258931:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513076.258936:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.258939:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.258939:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.142@tcp1 : PUT - for me 00000800:00000200:2.0:1616513076.258940:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.258945:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.258947:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.142@tcp1 of length 224 into portal 26 MB=0x5ffd4aed96d80 00000400:00000200:22.0:1616513076.258949:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82088 00000800:00000200:22.0:1616513076.258953:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513076.258954:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513076.258955:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.142@tcp1 of length 224/224 into md 0xf97c6a99 [1] + 0 00000400:00000200:20.0:1616513076.258959:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d430 00000400:00000200:20.0:1616513076.258961:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.258964:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513076.258968:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d430 00000800:00000200:20.0:1616513076.258971:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.258974:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.258975:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.258980:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663815187840 00010000:00000200:0.0:1616513076.258985:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688663815187840/t0(0) o400->f6327f35-8350-6f5c-223e-f383445e4a50@10.31.136.142@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.258996:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663815187840, offset 192 00000400:00000200:0.0:1616513076.259001:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.142@tcp1 00000400:00000200:0.0:1616513076.259006:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.142@tcp1 routed destination 00000400:00000200:0.0:1616513076.259018:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.142@tcp1(10.31.136.142@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.259023:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.259028:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.259030:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.259031:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.259034:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513076.259039:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513076.259045:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.259097:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.259105:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.259107:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513076.259109:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513076.259114:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:16.0:1616513076.259118:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.259119:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b784c8 00000800:00000200:20.0:1616513076.259120:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.259123:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513076.259127:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.142@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.259136:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.142@tcp1 of length 224 into portal 12 MB=0x5ffd4aed96f00 00000400:00000200:16.0:1616513076.259142:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.142@tcp1 of length 224/224 into md 0x865b9f31 [1] + 91912 00000400:00000200:16.0:1616513076.259147:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.259150:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513076.259157:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.259160:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.259161:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.259180:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663815188224 00010000:00000200:4.0:1616513076.259192:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd0480 x1688663815188224/t0(0) o400->c472a4e6-b5a6-7dad-4318-e5b2bffc674f@10.31.136.142@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.259205:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663815188224, offset 192 00000400:00000200:4.0:1616513076.259211:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.142@tcp1 00000400:00000200:4.0:1616513076.259218:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.142@tcp1 routed destination 00000400:00000200:4.0:1616513076.259232:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.142@tcp1(10.31.136.142@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.259237:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513076.259243:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.259245:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513076.259247:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513076.259252:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.259257:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.259266:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.259272:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.259275:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513076.259278:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513076.259280:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.259283:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bae58 00000800:00000200:14.0:1616513076.259287:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.266602:0:49043:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.266657:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.266666:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.266675:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.58@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.266684:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.58@tcp1 of length 224 into portal 26 MB=0x60137ad9ad5c0 00000400:00000200:2.0:1616513076.266691:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.58@tcp1 of length 224/224 into md 0xf97c6ab9 [1] + 0 00000400:00000200:2.0:1616513076.266695:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d650 00000400:00000200:2.0:1616513076.266698:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.266703:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513076.266707:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d650 00000800:00000200:2.0:1616513076.266710:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.266711:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.266713:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.266718:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.266721:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.266722:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513076.266731:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.58@tcp1 : PUT - for me 00000100:00000200:0.0:1616513076.266732:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690188507698624 00000400:00000200:22.0:1616513076.266739:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.58@tcp1 of length 224 into portal 12 MB=0x60137ad9ad640 00010000:00000200:0.0:1616513076.266743:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1690188507698624/t0(0) o400->242343f0-721a-f273-6452-4515f8dbe396@10.31.143.58@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513076.266745:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.58@tcp1 of length 224/224 into md 0xf96450b9 [1] + 60272 00000400:00000200:22.0:1616513076.266751:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.266753:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513076.266758:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690188507698624, offset 192 00000800:00000200:22.0:1616513076.266762:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513076.266764:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.58@tcp1 00000800:00000200:22.0:1616513076.266765:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.266767:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513076.266772:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.58@tcp1 routed destination 00000400:00000200:0.0:1616513076.266787:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.58@tcp1(10.31.143.58@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.266797:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.266803:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.266806:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.266808:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.266813:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513076.266819:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513076.266820:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:10.0:1616513076.266825:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690188507698752 00010000:00000200:10.0:1616513076.266839:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a29b00 x1690188507698752/t0(0) o400->17430ef0-4467-f3bb-3e3c-0af81607ef03@10.31.143.58@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513076.266854:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690188507698752, offset 192 00000400:00000200:10.0:1616513076.266860:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.58@tcp1 00000400:00000200:10.0:1616513076.266868:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.58@tcp1 routed destination 00000400:00000200:10.0:1616513076.266882:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.58@tcp1(10.31.143.58@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.266889:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513076.266892:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513076.266895:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.266897:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513076.266898:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:10.0:1616513076.266899:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513076.266901:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513076.266904:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513076.266908:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513076.266909:0:49043:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513076.266913:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78cc0 00000800:00000200:28.0:1616513076.266916:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.266919:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513076.266922:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.266929:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.266932:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513076.266933:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513076.266951:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.266955:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82f68 00000800:00000200:14.0:1616513076.266958:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.269850:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513076.269907:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513076.269916:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513076.269925:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.139@tcp1 : PUT - for me 00000400:00000200:26.0:1616513076.269934:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.139@tcp1 of length 224 into portal 26 MB=0x5ffe10b43ff80 00000400:00000200:26.0:1616513076.269940:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.139@tcp1 of length 224/224 into md 0xf97c6ac9 [1] + 0 00000400:00000200:26.0:1616513076.269945:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2cee0 00000400:00000200:26.0:1616513076.269947:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513076.269950:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513076.269954:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2cee0 00000800:00000200:26.0:1616513076.269957:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.269960:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513076.269961:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513076.269962:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513076.269964:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.270016:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716905283456 00000800:00000200:12.0:1616513076.270024:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:0.0:1616513076.270026:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688716905283456/t0(0) o400->5467cfb1-7cb6-6396-54e3-b430eb654100@10.31.141.139@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513076.270033:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513076.270039:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688716905283456, offset 192 00000400:00000200:12.0:1616513076.270042:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.139@tcp1 : PUT - for me 00000400:00000200:0.0:1616513076.270045:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.139@tcp1 00000400:00000200:0.0:1616513076.270052:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.139@tcp1 routed destination 00000400:00000200:0.0:1616513076.270066:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.139@tcp1(10.31.141.139@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513076.270072:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.139@tcp1 of length 224 into portal 12 MB=0x5ffe10b4402c0 00000800:00000200:0.0:1616513076.270072:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.270077:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513076.270078:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513076.270078:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.139@tcp1 of length 224/224 into md 0x865b9f31 [1] + 92136 00000800:00000200:0.0:1616513076.270080:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513076.270082:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513076.270084:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.270086:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513076.270086:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:11.2:1616513076.270091:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.270092:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513076.270092:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513076.270095:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513076.270097:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.270100:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.270104:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.270106:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513076.270111:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.270115:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b782a8 00000800:00000200:12.0:1616513076.270118:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.270121:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.270155:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716905284288 00010000:00000200:4.0:1616513076.270167:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661680 x1688716905284288/t0(0) o400->eefadad0-edf0-0927-55a6-182c5d732cb3@10.31.141.139@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.270182:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688716905284288, offset 192 00000400:00000200:4.0:1616513076.270189:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.139@tcp1 00000400:00000200:4.0:1616513076.270196:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.139@tcp1 routed destination 00000400:00000200:4.0:1616513076.270213:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.139@tcp1(10.31.141.139@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.270220:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513076.270225:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.270228:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513076.270231:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513076.270235:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.270241:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.270293:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.270299:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.270302:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513076.270308:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.270312:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb4b8 00000800:00000200:2.0:1616513076.270316:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.270319:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.271835:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513076.271891:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.271900:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.271909:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.43@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.271919:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.43@tcp1 of length 224 into portal 26 MB=0x5ffd4a89dab00 00000400:00000200:16.0:1616513076.271926:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.43@tcp1 of length 224/224 into md 0xf97c6ad9 [1] + 0 00000400:00000200:16.0:1616513076.271932:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2c7f8 00000400:00000200:16.0:1616513076.271935:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.271938:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513076.271942:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2c7f8 00000800:00000200:12.0:1616513076.271944:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.271946:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.271948:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.271950:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.272003:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663710608128 00010000:00000200:0.0:1616513076.272013:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688663710608128/t0(0) o400->a34d09c9-d8ff-b8a8-733b-0fb4ad978b2f@10.31.136.43@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.272026:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663710608128, offset 192 00000400:00000200:0.0:1616513076.272032:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.43@tcp1 00000400:00000200:0.0:1616513076.272040:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.43@tcp1 routed destination 00000800:00000200:7.2:1616513076.272053:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513076.272054:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.43@tcp1(10.31.136.43@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.272060:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.272065:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.272067:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.272071:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.272076:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.272080:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.272082:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513076.272107:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513076.272116:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513076.272125:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.43@tcp1 : PUT - for me 00000400:00000200:26.0:1616513076.272133:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.43@tcp1 of length 224 into portal 12 MB=0x5ffd4a89dac00 00000400:00000200:26.0:1616513076.272139:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.43@tcp1 of length 224/224 into md 0xf96450b9 [1] + 60496 00000800:00000200:20.0:1616513076.272144:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513076.272149:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513076.272150:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:26.0:1616513076.272152:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513076.272152:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.272158:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513076.272159:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.272161:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.272161:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513076.272163:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79b18 00000800:00000200:26.0:1616513076.272164:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513076.272167:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.272169:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513076.272181:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663710608384 00010000:00000200:10.0:1616513076.272192:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dd100 x1688663710608384/t0(0) o400->b8e06278-4d47-1c91-638d-4031f1305064@10.31.136.43@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513076.272206:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663710608384, offset 192 00000400:00000200:10.0:1616513076.272212:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.43@tcp1 00000400:00000200:10.0:1616513076.272219:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.43@tcp1 routed destination 00000400:00000200:10.0:1616513076.272233:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.43@tcp1(10.31.136.43@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.272239:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513076.272244:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.272247:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.272248:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.272253:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.272258:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.272314:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.272321:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.272323:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513076.272326:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513076.272329:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.272333:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83f58 00000800:00000200:2.0:1616513076.272337:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.281201:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513076.281255:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.281264:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.281272:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.43@tcp1 : PUT - for me 00000400:00000200:12.0:1616513076.281280:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.43@tcp1 of length 224 into portal 26 MB=0x604a2994794c0 00000400:00000200:12.0:1616513076.281286:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.43@tcp1 of length 224/224 into md 0xf97c6ae9 [1] + 0 00000400:00000200:12.0:1616513076.281291:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d540 00000400:00000200:12.0:1616513076.281294:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.281298:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513076.281302:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d540 00000800:00000200:12.0:1616513076.281305:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.281307:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.281308:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513076.281315:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.281318:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.281319:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.281327:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.43@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.281336:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.43@tcp1 of length 224 into portal 12 MB=0x604a299479540 00000400:00000200:16.0:1616513076.281342:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.43@tcp1 of length 224/224 into md 0x865b9f31 [1] + 92360 00000400:00000200:16.0:1616513076.281347:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.281350:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513076.281357:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.281360:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.281362:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.281363:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946263082176 00010000:00000200:0.0:1616513076.281373:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1693946263082176/t0(0) o400->898354d5-5ac0-12c6-bc91-8f548cd550bc@10.31.141.43@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.281387:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946263082176, offset 192 00000400:00000200:0.0:1616513076.281392:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.43@tcp1 00000400:00000200:0.0:1616513076.281399:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.43@tcp1 routed destination 00000400:00000200:0.0:1616513076.281413:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.43@tcp1(10.31.141.43@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513076.281418:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946263082304 00000800:00000200:0.0:1616513076.281419:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.281424:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.281427:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:4.0:1616513076.281429:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd7080 x1693946263082304/t0(0) o400->de9d2630-325b-672e-ec94-1b37f1dba06f@10.31.141.43@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513076.281429:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:11.2:1616513076.281443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513076.281446:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513076.281453:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:4.0:1616513076.281456:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946263082304, offset 192 00000400:00000200:4.0:1616513076.281462:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.43@tcp1 00000400:00000200:4.0:1616513076.281468:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.43@tcp1 routed destination 00000400:00000200:4.0:1616513076.281482:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.43@tcp1(10.31.141.43@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.281488:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513076.281494:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513076.281496:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513076.281496:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513076.281498:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513076.281503:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513076.281504:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.281506:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513076.281507:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513076.281509:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513076.281513:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.281520:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79650 00000800:00000200:28.0:1616513076.281524:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513076.281563:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.281570:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.281572:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.281578:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.281582:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb5c8 00000800:00000200:22.0:1616513076.281586:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.281588:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.290675:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513076.290730:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.290739:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.290748:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.68@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.290756:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.68@tcp1 of length 224 into portal 12 MB=0x5f908a80d0b80 00000400:00000200:14.0:1616513076.290762:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.68@tcp1 of length 224/224 into md 0xf96450b9 [1] + 60720 00000400:00000200:14.0:1616513076.290767:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.290769:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513076.290776:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.290779:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.290781:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.290783:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513076.290844:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681190458035072 00010000:00000200:10.0:1616513076.290856:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e7080 x1681190458035072/t0(0) o400->aef103db-571c-6171-e2dd-7228e2f9eeea@10.31.131.68@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513076.290869:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681190458035072, offset 192 00000400:00000200:10.0:1616513076.290874:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.68@tcp1 00000800:00000200:11.2:1616513076.290880:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:10.0:1616513076.290881:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.68@tcp1 routed destination 00000400:00000200:10.0:1616513076.290895:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.68@tcp1(10.31.131.68@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.290903:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513076.290908:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.290911:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513076.290913:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513076.290917:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.290933:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.290942:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.290951:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.68@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.290959:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.68@tcp1 of length 224 into portal 26 MB=0x5f908a80d0b00 00000400:00000200:16.0:1616513076.290965:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.68@tcp1 of length 224/224 into md 0xf97c6b19 [1] + 0 00000400:00000200:16.0:1616513076.290970:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3a2d3a8 00000400:00000200:16.0:1616513076.290972:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.290975:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513076.290980:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3a2d3a8 00000800:00000200:16.0:1616513076.290982:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513076.290985:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513076.290986:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:16.0:1616513076.290993:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513076.290994:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513076.290996:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.290997:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.291004:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.291010:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82aa0 00000800:00000200:20.0:1616513076.291014:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.291040:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681190458034944 00010000:00000200:0.0:1616513076.291050:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1681190458034944/t0(0) o400->cd2f1796-01c0-1785-f27c-e14c8c2ebdee@10.31.131.68@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.291062:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681190458034944, offset 192 00000400:00000200:0.0:1616513076.291067:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.68@tcp1 00000400:00000200:0.0:1616513076.291074:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.68@tcp1 routed destination 00000400:00000200:0.0:1616513076.291087:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.68@tcp1(10.31.131.68@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.291093:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.291098:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.291100:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.291102:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.291106:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.291112:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.291112:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513076.291166:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.291172:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.291175:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.291182:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.291187:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78f68 00000800:00000200:22.0:1616513076.291191:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.291194:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.292797:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513076.292852:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.292861:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513076.292870:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.251@tcp1 : PUT - for me 00000400:00000200:28.0:1616513076.292878:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.251@tcp1 of length 224 into portal 26 MB=0x5ffdc1b59a680 00000400:00000200:28.0:1616513076.292885:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.251@tcp1 of length 224/224 into md 0xf97c6b29 [1] + 0 00000800:00000200:7.2:1616513076.292889:0:49046:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513076.292890:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c67f8 00000400:00000200:28.0:1616513076.292892:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513076.292895:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513076.292899:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c67f8 00000800:00000200:28.0:1616513076.292902:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.292905:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.292907:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513076.292911:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513076.292943:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.292953:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.292962:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.251@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.292970:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.251@tcp1 of length 224 into portal 12 MB=0x5ffdc1b59aa80 00000400:00000200:2.0:1616513076.292976:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.251@tcp1 of length 224/224 into md 0xf96450b9 [1] + 60944 00000100:00000200:0.0:1616513076.292977:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695700301440 00000400:00000200:2.0:1616513076.292982:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.292984:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:0.0:1616513076.292988:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688695700301440/t0(0) o400->19d4e20c-044f-b364-569e-a8b8e5da2975@10.31.140.251@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513076.292991:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.292994:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.292996:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.292998:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.293008:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695700301440, offset 192 00000400:00000200:0.0:1616513076.293014:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.251@tcp1 00000400:00000200:0.0:1616513076.293022:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.251@tcp1 routed destination 00000400:00000200:0.0:1616513076.293037:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.251@tcp1(10.31.140.251@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.293043:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.293049:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.293051:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.293054:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:10.0:1616513076.293056:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695700302464 00000800:00000200:0.0:1616513076.293059:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.293064:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.293065:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:10.0:1616513076.293069:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623600 x1688695700302464/t0(0) o400->baf91fa2-27f8-b6f3-0838-0f5040c15963@10.31.140.251@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513076.293083:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695700302464, offset 192 00000400:00000200:10.0:1616513076.293089:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.251@tcp1 00000400:00000200:10.0:1616513076.293096:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.251@tcp1 routed destination 00000400:00000200:10.0:1616513076.293111:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.251@tcp1(10.31.140.251@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513076.293116:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513076.293117:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513076.293122:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:10.0:1616513076.293123:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513076.293125:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513076.293126:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.293128:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513076.293132:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:10.0:1616513076.293132:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513076.293137:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78000 00000800:00000200:7.2:1616513076.293137:0:49046:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513076.293141:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.293144:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513076.293215:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.293221:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.293224:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.293231:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.293235:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82e58 00000800:00000200:22.0:1616513076.293239:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.293242:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.298379:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.298447:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.298456:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.298465:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.193@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.298476:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.193@tcp1 of length 224 into portal 12 MB=0x5ffe1144575c0 00000400:00000200:2.0:1616513076.298482:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.193@tcp1 of length 224/224 into md 0xf96450b9 [1] + 61168 00000400:00000200:2.0:1616513076.298488:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.298490:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513076.298497:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.298499:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.298500:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.298504:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.298555:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:10.0:1616513076.298560:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717056374208 00010000:00000200:10.0:1616513076.298572:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464624800 x1688717056374208/t0(0) o400->53c566e5-b448-6384-3ff8-0cf7acb510ac@10.31.142.193@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513076.298586:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688717056374208, offset 192 00000400:00000200:10.0:1616513076.298593:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.193@tcp1 00000400:00000200:10.0:1616513076.298601:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.193@tcp1 routed destination 00000800:00000200:16.0:1616513076.298609:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513076.298615:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.193@tcp1(10.31.142.193@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513076.298618:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:10.0:1616513076.298622:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513076.298628:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513076.298630:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.193@tcp1 : PUT - for me 00000800:00000200:10.0:1616513076.298631:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513076.298633:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:10.0:1616513076.298637:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513076.298639:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.193@tcp1 of length 224 into portal 26 MB=0x5ffe114457340 00000400:00000200:16.0:1616513076.298646:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.193@tcp1 of length 224/224 into md 0xf97c6b39 [1] + 0 00000400:00000200:16.0:1616513076.298650:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6a18 00000400:00000200:16.0:1616513076.298653:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.298656:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513076.298661:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6a18 00000800:00000200:28.0:1616513076.298662:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513076.298664:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513076.298667:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:16.0:1616513076.298669:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513076.298670:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513076.298672:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513076.298673:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.298679:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.298683:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82908 00000800:00000200:28.0:1616513076.298687:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.298739:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717056373568 00010000:00000200:0.0:1616513076.298750:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688717056373568/t0(0) o400->e87c94bf-0227-c17f-3838-2097d9bed33e@10.31.142.193@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.298763:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688717056373568, offset 192 00000400:00000200:0.0:1616513076.298769:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.193@tcp1 00000400:00000200:0.0:1616513076.298776:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.193@tcp1 routed destination 00000400:00000200:0.0:1616513076.298790:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.193@tcp1(10.31.142.193@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.298796:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.298802:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.298804:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.298806:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.298811:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.298815:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.298817:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513076.298871:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.298877:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.298880:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.298887:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.298892:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b785d8 00000800:00000200:22.0:1616513076.298896:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.298899:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513076.314531:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925056, portal 4 00000100:00000200:15.0:1616513076.314536:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925056, offset 0 00000400:00000200:15.0:1616513076.314542:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:15.0:1616513076.314548:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:15.0:1616513076.314561:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513076.314567:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:15.0:1616513076.314574:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513076.314577:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:15.0:1616513076.314579:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:15.0:1616513076.314584:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513076.314630:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513076.314649:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513076.314655:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513076.314658:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513076.314665:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513076.314672:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63f500 x1693067725925056/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513076.314684:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6000 00000800:00000200:1.0:1616513076.314689:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513076.314691:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513076.314800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513076.314809:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513076.314815:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513076.314819:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513076.314824:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513076.314833:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626ac0 00000400:00000200:1.0:1616513076.314839:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffae5 [1] + 192 00000400:00000200:1.0:1616513076.314845:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513076.314849:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63f500 x1693067725925056/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513076.314863:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513076.314866:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513076.314867:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:15.0:1616513076.314870:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63f500 x1693067725925056/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513076.314881:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63f500 x1693067725925056/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513076.314889:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6b28 00000800:00000200:11.2:1616513076.325023:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513076.325078:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.325088:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.325097:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.123@tcp1 : PUT - for me 00000400:00000200:12.0:1616513076.325107:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.123@tcp1 of length 224 into portal 26 MB=0x5ffdb3f5fd240 00000400:00000200:12.0:1616513076.325113:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.123@tcp1 of length 224/224 into md 0xf97c6b59 [1] + 0 00000400:00000200:12.0:1616513076.325118:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c66e8 00000400:00000200:12.0:1616513076.325121:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.325124:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513076.325133:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c66e8 00000800:00000200:12.0:1616513076.325137:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.325139:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.325141:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.325144:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.325197:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692009718336 00010000:00000200:0.0:1616513076.325208:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688692009718336/t0(0) o400->4ad80938-f647-d559-0d4b-7f484cefd04d@10.31.141.123@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.325222:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692009718336, offset 192 00000400:00000200:0.0:1616513076.325228:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.123@tcp1 00000400:00000200:0.0:1616513076.325235:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.123@tcp1 routed destination 00000400:00000200:0.0:1616513076.325250:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.123@tcp1(10.31.141.123@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.325256:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.325262:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.325264:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.325266:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.325271:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.325277:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.325277:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513076.325331:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.325338:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.325341:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.325348:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.325354:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b793a8 00000800:00000200:28.0:1616513076.325357:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.325360:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.325366:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513076.325368:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513076.325373:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.123@tcp1 : PUT - for me 00000400:00000200:28.0:1616513076.325381:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.123@tcp1 of length 224 into portal 12 MB=0x5ffdb3f5fde00 00000400:00000200:28.0:1616513076.325388:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.123@tcp1 of length 224/224 into md 0x865b9f31 [1] + 92584 00000400:00000200:28.0:1616513076.325393:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513076.325395:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513076.325404:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.325407:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.325409:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.325469:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692009721344 00010000:00000200:4.0:1616513076.325482:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6723180 x1688692009721344/t0(0) o400->65c90c66-62b4-bfba-354c-1fe63ddbaebb@10.31.141.123@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.325497:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692009721344, offset 192 00000400:00000200:4.0:1616513076.325503:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.123@tcp1 00000400:00000200:4.0:1616513076.325511:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.123@tcp1 routed destination 00000400:00000200:4.0:1616513076.325526:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.123@tcp1(10.31.141.123@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.325533:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513076.325539:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.325542:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513076.325545:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513076.325551:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.325557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.325613:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.325620:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.325623:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.325629:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.325633:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba908 00000800:00000200:14.0:1616513076.325637:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.325640:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:15.2:1616513076.326388:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4c9148600] (20)++ 00000800:00000200:15.0:1616513076.326406:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c9148600] (21)++ 00000800:00000200:15.0:1616513076.326413:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.150@o2ib1 00000400:00000200:15.0:1616513076.326421:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.150@o2ib1 : PUT - for me 00000400:00000200:15.0:1616513076.326430:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.150@o2ib1 of length 224 into portal 26 MB=0x604a7211faa80 00000400:00000200:15.0:1616513076.326444:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.150@o2ib1 of length 224/224 into md 0xf97c6b69 [1] + 0 00000400:00000200:15.0:1616513076.326448:0:6476:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6330 00000400:00000200:15.0:1616513076.326451:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:15.0:1616513076.326453:0:6476:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:31.0:1616513076.326458:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c9148600] (22)++ 00000400:00000200:15.0:1616513076.326458:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6330 00000800:00000200:15.0:1616513076.326461:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4c9148600] (22)++ 00000800:00000200:15.0:1616513076.326464:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4c9148600] (24)-- 00000800:00000200:15.0:1616513076.326467:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c9148600] (23)-- 00000800:00000200:15.0:1616513076.326469:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c9148600] (22)-- 00000800:00000200:31.0:1616513076.326470:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.150@o2ib1 00000400:00000200:31.0:1616513076.326479:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.150@o2ib1 : PUT - for me 00000400:00000200:31.0:1616513076.326486:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.150@o2ib1 of length 224 into portal 12 MB=0x604a7211fac40 00000400:00000200:31.0:1616513076.326494:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.150@o2ib1 of length 224/224 into md 0x3657608d [1] + 90944 00000400:00000200:31.0:1616513076.326499:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:31.0:1616513076.326502:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:31.0:1616513076.326511:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4c9148600] (21)++ 00000800:00000200:31.0:1616513076.326514:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4c9148600] (22)-- 00000800:00000200:31.0:1616513076.326516:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c9148600] (21)-- 00000100:00000200:0.0:1616513076.326523:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693965722036864 00000100:00000200:7.0:1616513076.326531:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693965722037312 00010000:00000200:0.0:1616513076.326534:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1693965722036864/t0(0) o400->e6912ec4-81b9-97fc-7ea5-7e7349df81b9@10.242.46.150@o2ib1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:7.0:1616513076.326542:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cdc564c80 x1693965722037312/t0(0) o400->2c9c5ad4-61c1-2aca-d999-4bc882f7d9c4@10.242.46.150@o2ib1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513076.326549:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693965722036864, offset 192 00000100:00000200:7.0:1616513076.326552:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693965722037312, offset 192 00000400:00000200:0.0:1616513076.326556:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.150@o2ib1 00000400:00000200:7.0:1616513076.326557:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.150@o2ib1 00000400:00000200:7.0:1616513076.326563:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.150@o2ib1 local destination 00000400:00000200:0.0:1616513076.326565:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.150@o2ib1 local destination 00000400:00000200:7.0:1616513076.326574:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.150@o2ib1(10.242.46.150@o2ib1:10.242.46.150@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513076.326579:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.150@o2ib1 00000400:00000200:0.0:1616513076.326588:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.150@o2ib1(10.242.46.150@o2ib1:10.242.46.150@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513076.326595:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8dcff92f00] -> 10.242.46.150@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.326595:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.150@o2ib1 00000800:00000200:7.0:1616513076.326597:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4c9148600] (20)++ 00000800:00000200:7.0:1616513076.326599:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4c9148600] (21)++ 00000800:00000200:0.0:1616513076.326602:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8dcff92f00] -> 10.242.46.150@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513076.326603:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4c9148600] (22)-- 00000800:00000200:0.0:1616513076.326605:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4c9148600] (21)++ 00000800:00000200:0.0:1616513076.326607:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4c9148600] (22)++ 00000800:00000200:15.2:1616513076.326609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4c9148600] (22)++ 00000800:00000200:0.0:1616513076.326612:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4c9148600] (24)-- 00000800:00000200:15.0:1616513076.326617:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c9148600] (23)++ 00000100:00000200:0.0:1616513076.326619:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:15.0:1616513076.326622:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4c9148600] (24)-- 00000400:00000200:15.0:1616513076.326624:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:19.0:1616513076.326627:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c9148600] (23)++ 00000400:00000200:15.0:1616513076.326628:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.150@o2ib1: PUT: OK 00000400:00000200:15.0:1616513076.326631:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f5c8 00000800:00000200:15.0:1616513076.326634:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c9148600] (24)-- 00000800:00000200:19.0:1616513076.326635:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4c9148600] (24)-- 00000800:00000200:15.0:1616513076.326636:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c9148600] (23)-- 00000400:00000200:19.0:1616513076.326638:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:19.0:1616513076.326645:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.150@o2ib1: PUT: OK 00000400:00000200:19.0:1616513076.326651:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b786e8 00000800:00000200:19.0:1616513076.326654:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c9148600] (21)-- 00000800:00000200:7.2:1616513076.330718:0:49050:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513076.330773:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.330784:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513076.330794:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.97@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.330803:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.97@tcp1 of length 224 into portal 26 MB=0x5ffdcc3551e00 00000400:00000200:22.0:1616513076.330812:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.97@tcp1 of length 224/224 into md 0xf97c6b79 [1] + 0 00000400:00000200:22.0:1616513076.330817:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c64c8 00000400:00000200:22.0:1616513076.330819:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.330822:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513076.330826:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c64c8 00000800:00000200:22.0:1616513076.330829:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.330830:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.330832:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.330834:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.330867:0:49050:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513076.330891:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698518576640 00010000:00000200:0.0:1616513076.330902:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688698518576640/t0(0) o400->2744daa2-40f2-72ed-1748-28248132402a@10.31.142.97@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.330916:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698518576640, offset 192 00000800:00000200:14.0:1616513076.330921:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513076.330923:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.97@tcp1 00000800:00000200:14.0:1616513076.330930:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513076.330931:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.97@tcp1 routed destination 00000400:00000200:14.0:1616513076.330939:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.97@tcp1 : PUT - for me 00000400:00000200:0.0:1616513076.330945:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.97@tcp1(10.31.142.97@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513076.330950:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.97@tcp1 of length 224 into portal 12 MB=0x5ffdcc35520c0 00000800:00000200:0.0:1616513076.330952:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513076.330957:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.97@tcp1 of length 224/224 into md 0xf96450b9 [1] + 61392 00000800:00000200:0.0:1616513076.330958:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513076.330962:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513076.330962:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.330964:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:14.0:1616513076.330965:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513076.330969:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513076.330975:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513076.330976:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.330976:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513076.330978:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.330978:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.330982:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513076.331040:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698518577344 00000800:00000200:16.0:1616513076.331046:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:10.0:1616513076.331051:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e7500 x1688698518577344/t0(0) o400->54671edf-4cd8-b251-789d-0e2b4087d769@10.31.142.97@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513076.331052:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.331055:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513076.331061:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:10.0:1616513076.331065:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698518577344, offset 192 00000400:00000200:16.0:1616513076.331067:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79980 00000800:00000200:16.0:1616513076.331071:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513076.331071:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.97@tcp1 00000800:00000200:16.0:1616513076.331074:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513076.331078:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.97@tcp1 routed destination 00000400:00000200:10.0:1616513076.331092:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.97@tcp1(10.31.142.97@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.331098:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513076.331103:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.331106:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.331108:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.331112:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.331117:0:49050:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513076.331127:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.331133:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513076.331135:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513076.331137:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513076.331142:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513076.331146:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82908 00000800:00000200:26.0:1616513076.331150:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.342576:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.342638:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.342651:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513076.342657:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513076.342660:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.178@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.342670:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.178@tcp1 of length 224 into portal 26 MB=0x5ffde20406f40 00000400:00000200:2.0:1616513076.342676:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.178@tcp1 of length 224/224 into md 0xf97c6ba9 [1] + 0 00000400:00000200:2.0:1616513076.342681:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6ff0 00000400:00000200:2.0:1616513076.342684:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.342686:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513076.342690:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6ff0 00000800:00000200:2.0:1616513076.342694:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.342696:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.342698:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.342755:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704372469568 00010000:00000200:0.0:1616513076.342766:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688704372469568/t0(0) o400->815d0c15-ca4c-4725-21ef-7055f265a6b4@10.31.132.178@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.342780:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704372469568, offset 192 00000400:00000200:0.0:1616513076.342786:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.178@tcp1 00000400:00000200:0.0:1616513076.342794:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.178@tcp1 routed destination 00000400:00000200:0.0:1616513076.342809:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.178@tcp1(10.31.132.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.342815:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.342821:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.342823:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.342826:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.342830:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.342837:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.342837:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513076.342892:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.342899:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.342902:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.342908:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.342914:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79320 00000800:00000200:20.0:1616513076.342917:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.342920:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.343285:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513076.343343:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.343352:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513076.343361:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.178@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.343370:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.178@tcp1 of length 224 into portal 12 MB=0x5ffde20407080 00000400:00000200:22.0:1616513076.343376:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.178@tcp1 of length 224/224 into md 0xf96450b9 [1] + 61616 00000400:00000200:22.0:1616513076.343381:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.343384:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513076.343390:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.343393:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.343395:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513076.343398:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513076.343467:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704372469888 00010000:00000200:10.0:1616513076.343480:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2d100 x1688704372469888/t0(0) o400->3267f653-e1c1-46ae-7b92-6714a2a4f9b3@10.31.132.178@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513076.343494:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704372469888, offset 192 00000400:00000200:10.0:1616513076.343500:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.178@tcp1 00000400:00000200:10.0:1616513076.343507:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.178@tcp1 routed destination 00000400:00000200:10.0:1616513076.343522:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.178@tcp1(10.31.132.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.343529:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513076.343535:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.343537:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.343539:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.343544:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.343548:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.343559:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.343564:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.343567:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513076.343569:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513076.343574:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.343578:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82e58 00000800:00000200:14.0:1616513076.343581:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.362714:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513076.362770:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.362779:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513076.362789:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.66@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.362799:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.66@tcp1 of length 224 into portal 26 MB=0x60182b3866cc0 00000400:00000200:22.0:1616513076.362806:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.66@tcp1 of length 224/224 into md 0xf97c6bb9 [1] + 0 00000400:00000200:22.0:1616513076.362810:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6e58 00000400:00000200:22.0:1616513076.362813:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.362816:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513076.362820:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6e58 00000800:00000200:22.0:1616513076.362823:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.362824:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.362826:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.362829:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.362885:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510729571520 00010000:00000200:0.0:1616513076.362896:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1690510729571520/t0(0) o400->2d095b54-3848-76c7-c93b-fee1f20421dd@10.31.142.66@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.362911:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510729571520, offset 192 00000400:00000200:0.0:1616513076.362917:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.66@tcp1 00000400:00000200:0.0:1616513076.362925:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.66@tcp1 routed destination 00000400:00000200:0.0:1616513076.362940:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.66@tcp1(10.31.142.66@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.362946:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.362952:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.362955:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.362957:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.362961:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.362968:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.362968:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513076.363023:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.363030:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.363033:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513076.363039:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513076.363045:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79c28 00000800:00000200:16.0:1616513076.363048:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.363051:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.363057:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513076.363059:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.363062:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.363063:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.363065:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.363512:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513076.363567:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.363577:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513076.363579:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513076.363586:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.66@tcp1 : PUT - for me 00000400:00000200:28.0:1616513076.363595:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.66@tcp1 of length 224 into portal 12 MB=0x60182b3867a00 00000400:00000200:28.0:1616513076.363601:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.66@tcp1 of length 224/224 into md 0x865b9f31 [1] + 92808 00000400:00000200:28.0:1616513076.363606:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513076.363609:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513076.363616:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.363619:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.363621:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.363638:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510729574912 00010000:00000200:4.0:1616513076.363650:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6721b00 x1690510729574912/t0(0) o400->ee839548-9afa-2ad6-d6d4-531a2e6c24c0@10.31.142.66@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.363665:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510729574912, offset 192 00000400:00000200:4.0:1616513076.363671:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.66@tcp1 00000400:00000200:4.0:1616513076.363679:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.66@tcp1 routed destination 00000400:00000200:4.0:1616513076.363694:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.66@tcp1(10.31.142.66@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.363701:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513076.363707:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.363710:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513076.363712:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513076.363717:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.363722:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.363776:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.363784:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.363787:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513076.363793:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.363797:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6babb0 00000800:00000200:2.0:1616513076.363801:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.363804:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.366089:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513076.366145:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513076.366154:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513076.366163:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.80@tcp1 : PUT - for me 00000400:00000200:26.0:1616513076.366173:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.80@tcp1 of length 224 into portal 12 MB=0x5ffdd42bf3000 00000400:00000200:26.0:1616513076.366180:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.80@tcp1 of length 224/224 into md 0xf96450b9 [1] + 61840 00000400:00000200:26.0:1616513076.366185:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513076.366187:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513076.366195:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.366197:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.366198:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513076.366204:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.366210:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:10.0:1616513076.366259:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700656234496 00000800:00000200:12.0:1616513076.366268:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:10.0:1616513076.366272:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9200 x1688700656234496/t0(0) o400->0e9ad0ee-8f72-234e-5ffe-8aedd8b59154@10.31.132.80@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513076.366277:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.366286:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.80@tcp1 : PUT - for me 00000100:00000200:10.0:1616513076.366286:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700656234496, offset 192 00000400:00000200:10.0:1616513076.366292:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.80@tcp1 00000400:00000200:12.0:1616513076.366295:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.80@tcp1 of length 224 into portal 26 MB=0x5ffdd42bf2b80 00000400:00000200:10.0:1616513076.366300:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.80@tcp1 routed destination 00000400:00000200:12.0:1616513076.366302:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.80@tcp1 of length 224/224 into md 0xf97c6bd9 [1] + 0 00000400:00000200:12.0:1616513076.366307:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7540 00000400:00000200:12.0:1616513076.366310:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.366312:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513076.366317:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7540 00000400:00000200:10.0:1616513076.366317:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.80@tcp1(10.31.132.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513076.366322:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513076.366323:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513076.366325:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.366325:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.366329:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513076.366329:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.366332:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513076.366334:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513076.366339:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.366343:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.366379:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700656233344 00010000:00000200:0.0:1616513076.366390:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688700656233344/t0(0) o400->0a533d9f-dd06-6bcb-b505-75dc55d2b7ed@10.31.132.80@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513076.366400:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513076.366404:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700656233344, offset 192 00000800:00000200:16.0:1616513076.366406:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.366411:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513076.366411:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.80@tcp1 00000400:00000200:16.0:1616513076.366418:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513076.366419:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.80@tcp1 routed destination 00000400:00000200:16.0:1616513076.366422:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82aa0 00000800:00000200:16.0:1616513076.366426:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.366429:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513076.366446:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.80@tcp1(10.31.132.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.366453:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.366459:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.366462:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.366464:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.366469:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.366474:0:49054:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.366476:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513076.366528:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.366535:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.366538:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513076.366541:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513076.366544:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.366550:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78440 00000800:00000200:2.0:1616513076.366555:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.369873:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513076.369932:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.369942:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513076.369951:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.12@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.369960:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.12@tcp1 of length 224 into portal 26 MB=0x604a2df04e480 00000400:00000200:22.0:1616513076.369966:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.12@tcp1 of length 224/224 into md 0xf97c6be9 [1] + 0 00000400:00000200:22.0:1616513076.369971:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c75c8 00000400:00000200:22.0:1616513076.369974:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.369977:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513076.369981:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c75c8 00000800:00000200:22.0:1616513076.369984:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.369986:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.369986:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.369990:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.370045:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947433116800 00010000:00000200:0.0:1616513076.370056:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1693947433116800/t0(0) o400->e970b517-9acc-ec83-3ffb-3a46f4015b27@10.31.141.12@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.370070:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693947433116800, offset 192 00000800:00000200:7.2:1616513076.370076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513076.370076:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.12@tcp1 00000400:00000200:0.0:1616513076.370084:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.12@tcp1 routed destination 00000800:00000200:14.0:1616513076.370092:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513076.370099:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.12@tcp1(10.31.141.12@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513076.370101:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.370105:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513076.370110:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.12@tcp1 : PUT - for me 00000800:00000200:0.0:1616513076.370111:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.370114:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.370116:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513076.370119:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.12@tcp1 of length 224 into portal 12 MB=0x604a2df04e680 00000800:00000200:0.0:1616513076.370121:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513076.370125:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.12@tcp1 of length 224/224 into md 0xf96450b9 [1] + 62064 00000800:00000200:11.2:1616513076.370127:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.370127:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513076.370130:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.370132:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513076.370139:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.370142:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.370144:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.370147:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513076.370162:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947433117312 00010000:00000200:10.0:1616513076.370173:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e7980 x1693947433117312/t0(0) o400->85480832-d3fb-9cbe-de01-06155867bccb@10.31.141.12@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513076.370181:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:10.0:1616513076.370186:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693947433117312, offset 192 00000800:00000200:20.0:1616513076.370187:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.370190:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513076.370192:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.12@tcp1 00000400:00000200:20.0:1616513076.370197:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:10.0:1616513076.370199:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.12@tcp1 routed destination 00000400:00000200:20.0:1616513076.370202:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78bb0 00000800:00000200:20.0:1616513076.370206:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.370209:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513076.370212:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.12@tcp1(10.31.141.12@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.370219:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513076.370224:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.370226:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.370229:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.370233:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.370238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513076.370249:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.370254:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513076.370257:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513076.370259:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513076.370263:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513076.370268:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83f58 00000800:00000200:26.0:1616513076.370271:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.393202:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513076.393258:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.393268:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.393276:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.187@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.393285:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.187@tcp1 of length 224 into portal 26 MB=0x6058f37c877c0 00000400:00000200:14.0:1616513076.393291:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.187@tcp1 of length 224/224 into md 0xf97c6c09 [1] + 0 00000400:00000200:14.0:1616513076.393296:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c65d8 00000400:00000200:14.0:1616513076.393298:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.393304:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.393308:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c65d8 00000800:00000200:14.0:1616513076.393311:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.393312:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.393314:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.393317:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513076.393340:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513076.393374:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694962534610880 00010000:00000200:0.0:1616513076.393386:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1694962534610880/t0(0) o400->d18b96ec-fa8d-3f12-f8c2-10252fa2a4ea@10.31.129.187@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513076.393393:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.393400:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694962534610880, offset 192 00000800:00000200:22.0:1616513076.393402:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513076.393408:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.187@tcp1 00000400:00000200:22.0:1616513076.393410:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.187@tcp1 : PUT - for me 00000400:00000200:0.0:1616513076.393423:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.187@tcp1 routed destination 00000400:00000200:22.0:1616513076.393424:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.187@tcp1 of length 224 into portal 12 MB=0x6058f37c87840 00000400:00000200:22.0:1616513076.393430:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.187@tcp1 of length 224/224 into md 0xf96450b9 [1] + 62288 00000400:00000200:22.0:1616513076.393445:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.393448:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513076.393450:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.187@tcp1(10.31.129.187@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513076.393455:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513076.393457:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513076.393458:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.393460:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513076.393462:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513076.393463:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.393466:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.393468:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.393473:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.393480:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.393481:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:10.0:1616513076.393517:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694962534611008 00010000:00000200:10.0:1616513076.393528:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2da00 x1694962534611008/t0(0) o400->6431fd57-c7f1-1384-b687-6d7bbaacf3f3@10.31.129.187@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513076.393534:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.393541:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:10.0:1616513076.393542:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694962534611008, offset 192 00000400:00000200:16.0:1616513076.393543:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513076.393548:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.187@tcp1 00000400:00000200:16.0:1616513076.393550:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:10.0:1616513076.393554:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.187@tcp1 routed destination 00000400:00000200:16.0:1616513076.393556:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79c28 00000800:00000200:16.0:1616513076.393560:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.393562:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513076.393568:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.187@tcp1(10.31.129.187@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.393574:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513076.393579:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.393582:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.393586:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.393590:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.393595:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.393605:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.393611:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.393613:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513076.393614:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513076.393620:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.393624:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82f68 00000800:00000200:2.0:1616513076.393628:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.401477:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513076.401531:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.401540:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.401549:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.401558:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d0322a5240 00000400:00000200:20.0:1616513076.401565:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0x865b9f31 [1] + 93032 00000400:00000200:20.0:1616513076.401570:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.401573:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.401579:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.401582:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.401583:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513076.401589:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.401642:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042589979200 00010000:00000200:4.0:1616513076.401726:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e02400 x1693042589979200/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:702/0 lens 440/368 e 0 to 0 dl 1616513082 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513076.401739:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042589979200, offset 192 00000400:00000200:4.0:1616513076.401745:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:4.0:1616513076.401753:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:4.0:1616513076.401767:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.401773:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.401778:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.401781:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.401783:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.401787:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.401805:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.401857:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.401864:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.401867:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513076.401873:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513076.401877:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba770 00000800:00000200:16.0:1616513076.401881:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.401884:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.424640:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513076.424694:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.424704:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.424713:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.164@tcp1 : PUT - for me 00000400:00000200:12.0:1616513076.424722:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.164@tcp1 of length 224 into portal 26 MB=0x5ffd3e9253380 00000400:00000200:12.0:1616513076.424728:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.164@tcp1 of length 224/224 into md 0xf97c6c29 [1] + 0 00000400:00000200:12.0:1616513076.424733:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6198 00000400:00000200:12.0:1616513076.424736:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.424739:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513076.424744:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6198 00000800:00000200:12.0:1616513076.424747:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.424750:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.424751:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.424754:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.424765:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.424773:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.164@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.424782:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.164@tcp1 of length 224 into portal 12 MB=0x5ffd3e9253440 00000400:00000200:20.0:1616513076.424788:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.164@tcp1 of length 224/224 into md 0x865b9f31 [1] + 93472 00000400:00000200:20.0:1616513076.424793:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.424796:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.424803:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.424806:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513076.424807:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660498264960 00000800:00000200:20.0:1616513076.424808:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.424810:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513076.424818:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688660498264960/t0(0) o400->ba052916-8209-512d-d031-8f515e111034@10.31.138.164@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.424833:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660498264960, offset 192 00000400:00000200:0.0:1616513076.424839:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.164@tcp1 00000400:00000200:0.0:1616513076.424852:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.164@tcp1 routed destination 00000400:00000200:0.0:1616513076.424867:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.164@tcp1(10.31.138.164@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513076.424870:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660498265152 00000800:00000200:0.0:1616513076.424874:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.424880:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:4.0:1616513076.424882:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e02d00 x1688660498265152/t0(0) o400->641d6608-8192-7de0-6e25-b1d5bec81d55@10.31.138.164@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513076.424883:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.424885:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.424890:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513076.424897:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660498265152, offset 192 00000100:00000200:0.0:1616513076.424897:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513076.424898:0:49060:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513076.424905:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.164@tcp1 00000400:00000200:4.0:1616513076.424912:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.164@tcp1 routed destination 00000400:00000200:4.0:1616513076.424927:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.164@tcp1(10.31.138.164@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.424934:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.424940:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.424943:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.424945:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.424949:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513076.424953:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513076.424954:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513076.424960:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.424963:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.424969:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.424975:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79980 00000800:00000200:22.0:1616513076.424978:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.424981:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513076.425007:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.425014:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.425017:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513076.425018:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513076.425024:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.425028:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba7f8 00000800:00000200:28.0:1616513076.425032:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.433188:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513076.433242:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.433252:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.433261:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.13@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.433270:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.13@tcp1 of length 224 into portal 12 MB=0x605356d689cc0 00000400:00000200:14.0:1616513076.433277:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.13@tcp1 of length 224/224 into md 0xf96450b9 [1] + 62512 00000400:00000200:14.0:1616513076.433282:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.433285:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513076.433292:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.433295:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.433297:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.433299:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513076.433314:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694576887241920 00010000:00000200:10.0:1616513076.433326:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e5580 x1694576887241920/t0(0) o400->3de56a05-5af8-ecd5-bf0d-2e7fc301e137@10.31.128.13@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513076.433340:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694576887241920, offset 192 00000400:00000200:10.0:1616513076.433347:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.13@tcp1 00000400:00000200:10.0:1616513076.433355:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.13@tcp1 routed destination 00000800:00000200:11.2:1616513076.433356:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:10.0:1616513076.433369:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.13@tcp1(10.31.128.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.433376:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513076.433382:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.433384:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.433386:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.433391:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.433396:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513076.433409:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.433419:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.433428:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.13@tcp1 : PUT - for me 00000800:00000200:22.0:1616513076.433449:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513076.433452:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.13@tcp1 of length 224 into portal 26 MB=0x605356d689880 00000800:00000200:22.0:1616513076.433467:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:16.0:1616513076.433468:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.13@tcp1 of length 224/224 into md 0xf97c6c39 [1] + 0 00000400:00000200:22.0:1616513076.433470:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513076.433473:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7f58 00000400:00000200:16.0:1616513076.433476:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.433478:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513076.433479:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:16.0:1616513076.433483:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7f58 00000800:00000200:20.0:1616513076.433485:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513076.433486:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82088 00000800:00000200:16.0:1616513076.433486:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.433489:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513076.433490:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513076.433491:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513076.433493:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.433550:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694576887240832 00010000:00000200:0.0:1616513076.433561:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1694576887240832/t0(0) o400->ce48db64-a034-d408-5dcc-4ec9c84c92f0@10.31.128.13@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.433575:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694576887240832, offset 192 00000400:00000200:0.0:1616513076.433582:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.13@tcp1 00000400:00000200:0.0:1616513076.433589:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.13@tcp1 routed destination 00000400:00000200:0.0:1616513076.433604:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.13@tcp1(10.31.128.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.433615:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.433621:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.433623:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.433626:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.433630:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.433635:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.433637:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513076.433688:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.433694:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.433697:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513076.433703:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.433709:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b793a8 00000800:00000200:12.0:1616513076.433712:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.433715:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.435291:0:49061:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513076.435346:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.435355:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.435364:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.4@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.435373:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.4@tcp1 of length 224 into portal 26 MB=0x60006c9704d00 00000400:00000200:14.0:1616513076.435380:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.4@tcp1 of length 224/224 into md 0xf97c6c59 [1] + 0 00000400:00000200:14.0:1616513076.435385:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6ee0 00000400:00000200:14.0:1616513076.435387:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.435390:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.435394:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6ee0 00000800:00000200:14.0:1616513076.435397:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.435399:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.435400:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.435403:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.435470:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688879009647872 00010000:00000200:0.0:1616513076.435481:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688879009647872/t0(0) o400->45a79c2c-a5aa-b53b-c7f7-56b407e60cb2@10.31.133.4@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.435496:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688879009647872, offset 192 00000400:00000200:0.0:1616513076.435502:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.4@tcp1 00000400:00000200:0.0:1616513076.435510:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.4@tcp1 routed destination 00000400:00000200:0.0:1616513076.435525:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.4@tcp1(10.31.133.4@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513076.435527:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.435531:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.435537:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.435540:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.435542:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.435547:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.435552:0:49061:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.435553:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513076.435584:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.435593:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.435602:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.4@tcp1 : PUT - for me 00000800:00000200:22.0:1616513076.435608:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513076.435611:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.4@tcp1 of length 224 into portal 12 MB=0x60006c9704ec0 00000800:00000200:22.0:1616513076.435615:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513076.435617:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.4@tcp1 of length 224/224 into md 0x865b9f31 [1] + 93696 00000400:00000200:22.0:1616513076.435618:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513076.435622:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.435625:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:20.0:1616513076.435625:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.435631:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513076.435632:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78000 00000800:00000200:20.0:1616513076.435634:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513076.435636:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513076.435636:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.435638:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513076.435639:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513076.435697:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688879009648320 00010000:00000200:4.0:1616513076.435710:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03a80 x1688879009648320/t0(0) o400->5b9d0056-49b1-4d39-1eac-2e251eaea73d@10.31.133.4@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.435725:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688879009648320, offset 192 00000400:00000200:4.0:1616513076.435731:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.4@tcp1 00000400:00000200:4.0:1616513076.435739:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.4@tcp1 routed destination 00000400:00000200:4.0:1616513076.435754:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.4@tcp1(10.31.133.4@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.435760:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.435766:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.435769:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.435771:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.435776:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.435781:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.435798:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.435805:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.435807:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513076.435814:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.435818:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb188 00000800:00000200:12.0:1616513076.435822:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.435825:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.438903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.438956:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.438966:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.438975:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.108@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.438985:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.108@tcp1 of length 224 into portal 26 MB=0x60182e27b5600 00000400:00000200:2.0:1616513076.438992:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.108@tcp1 of length 224/224 into md 0xf97c6c89 [1] + 0 00000400:00000200:2.0:1616513076.438996:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7188 00000400:00000200:2.0:1616513076.438999:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.439002:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513076.439006:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7188 00000800:00000200:2.0:1616513076.439009:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.439010:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.439012:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.439014:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.439070:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511517373952 00010000:00000200:0.0:1616513076.439081:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1690511517373952/t0(0) o400->b75904b6-1881-abea-174a-edd7778af57a@10.31.136.108@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.439096:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511517373952, offset 192 00000400:00000200:0.0:1616513076.439102:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.108@tcp1 00000400:00000200:0.0:1616513076.439110:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.108@tcp1 routed destination 00000400:00000200:0.0:1616513076.439125:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.108@tcp1(10.31.136.108@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.439131:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.439137:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.439140:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.439144:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.439149:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.439153:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.439155:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513076.439207:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.439214:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.439217:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.439223:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.439229:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79650 00000800:00000200:22.0:1616513076.439232:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.439235:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.439347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513076.439404:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.439414:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.439423:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.108@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.439431:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.108@tcp1 of length 224 into portal 12 MB=0x60182e27b5700 00000400:00000200:20.0:1616513076.439464:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.108@tcp1 of length 224/224 into md 0x865b9f31 [1] + 93920 00000800:00000200:16.0:1616513076.439468:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513076.439469:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.439472:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.439478:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.439481:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.439483:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.439542:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511517374208 00010000:00000200:4.0:1616513076.439554:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03f00 x1690511517374208/t0(0) o400->ef6e5930-1b5a-0b64-ab40-889e5d57b5c3@10.31.136.108@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.439567:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511517374208, offset 192 00000400:00000200:4.0:1616513076.439573:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.108@tcp1 00000400:00000200:4.0:1616513076.439580:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.108@tcp1 routed destination 00000400:00000200:4.0:1616513076.439594:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.108@tcp1(10.31.136.108@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.439600:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.439605:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.439609:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.439611:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.439616:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.439621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.439633:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.439639:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.439642:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513076.439648:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.439652:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba5d8 00000800:00000200:12.0:1616513076.439656:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.439659:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.479817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513076.479873:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.479885:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.479894:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.243@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.479904:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.243@tcp1 of length 224 into portal 26 MB=0x5ffd5c274e140 00000400:00000200:16.0:1616513076.479910:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.243@tcp1 of length 224/224 into md 0xf97c6c99 [1] + 0 00000400:00000200:16.0:1616513076.479915:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6cc0 00000400:00000200:16.0:1616513076.479918:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.479921:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513076.479926:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6cc0 00000800:00000200:20.0:1616513076.479928:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.479929:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.479932:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513076.479935:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.479938:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.479939:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.479947:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.243@tcp1 : PUT - for me 00000100:00000200:0.0:1616513076.479952:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668439109952 00000400:00000200:20.0:1616513076.479956:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.243@tcp1 of length 224 into portal 12 MB=0x5ffd5c274e240 00000400:00000200:20.0:1616513076.479962:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.243@tcp1 of length 224/224 into md 0x865b9f31 [1] + 94144 00010000:00000200:0.0:1616513076.479964:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688668439109952/t0(0) o400->5b4c0819-ef35-e6d5-340b-5af21f7bba19@10.31.131.243@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513076.479967:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.479970:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.479977:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.479978:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668439109952, offset 192 00000800:00000200:20.0:1616513076.479980:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.479982:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513076.479984:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.243@tcp1 00000400:00000200:0.0:1616513076.479991:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.243@tcp1 routed destination 00000400:00000200:0.0:1616513076.480006:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.243@tcp1(10.31.131.243@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.480012:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.480018:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.480021:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.480023:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.480028:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513076.480035:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513076.480038:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513076.480042:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668439110208 00010000:00000200:4.0:1616513076.480055:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e04380 x1688668439110208/t0(0) o400->4303dbf0-b61d-c5bb-d8f1-494371277ed0@10.31.131.243@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.480069:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668439110208, offset 192 00000400:00000200:4.0:1616513076.480075:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.243@tcp1 00000400:00000200:4.0:1616513076.480082:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.243@tcp1 routed destination 00000800:00000200:14.0:1616513076.480095:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513076.480096:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.243@tcp1(10.31.131.243@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.480102:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513076.480103:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.480105:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513076.480107:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.480110:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513076.480112:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:4.0:1616513076.480112:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.480117:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513076.480120:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b782a8 00000800:00000200:11.2:1616513076.480122:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513076.480124:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.480127:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513076.480138:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.480145:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.480148:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513076.480149:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513076.480155:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.480159:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb980 00000800:00000200:28.0:1616513076.480163:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.544021:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513076.544077:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.544087:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.544097:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.24@tcp1 : PUT - for me 00000400:00000200:12.0:1616513076.544107:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.24@tcp1 of length 224 into portal 26 MB=0x60195c8ca03c0 00000400:00000200:12.0:1616513076.544114:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.24@tcp1 of length 224/224 into md 0xf97c6cb9 [1] + 0 00000400:00000200:12.0:1616513076.544119:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7298 00000400:00000200:12.0:1616513076.544122:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.544125:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513076.544130:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7298 00000800:00000200:20.0:1616513076.544132:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.544133:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.544136:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.544138:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.544195:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690592690701248 00010000:00000200:0.0:1616513076.544206:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1690592690701248/t0(0) o400->dbddc9e2-2f1c-2eb0-83e3-9ae96688943d@10.31.141.24@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.544222:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690592690701248, offset 192 00000400:00000200:0.0:1616513076.544228:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.24@tcp1 00000400:00000200:0.0:1616513076.544236:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.24@tcp1 routed destination 00000400:00000200:0.0:1616513076.544251:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.24@tcp1(10.31.141.24@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.544258:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.544264:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.544267:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.544269:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.544274:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513076.544281:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513076.544283:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.544340:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.544348:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.544351:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.544357:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.544363:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b784c8 00000800:00000200:22.0:1616513076.544367:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.544370:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.544418:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513076.544473:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.544482:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.544491:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.24@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.544500:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.24@tcp1 of length 224 into portal 12 MB=0x60195c8ca0640 00000400:00000200:16.0:1616513076.544506:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.24@tcp1 of length 224/224 into md 0x865b9f31 [1] + 94368 00000400:00000200:16.0:1616513076.544511:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.544514:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513076.544525:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.544528:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.544530:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.544533:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.544592:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690592690701888 00010000:00000200:4.0:1616513076.544605:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd7980 x1690592690701888/t0(0) o400->53a418cc-f994-f2da-c9b4-dd6e84be7008@10.31.141.24@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.544620:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690592690701888, offset 192 00000400:00000200:4.0:1616513076.544626:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.24@tcp1 00000400:00000200:4.0:1616513076.544634:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.24@tcp1 routed destination 00000400:00000200:4.0:1616513076.544649:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.24@tcp1(10.31.141.24@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.544656:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.544661:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.544664:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.544666:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.544671:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.544677:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.544688:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.544693:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.544695:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513076.544697:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513076.544702:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513076.544706:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb6d8 00000800:00000200:20.0:1616513076.544710:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.558785:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513076.558841:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.558851:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.558860:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.76@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.558870:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.76@tcp1 of length 224 into portal 26 MB=0x5ffd51ee2cdc0 00000400:00000200:14.0:1616513076.558877:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.76@tcp1 of length 224/224 into md 0xf97c6cd9 [1] + 0 00000400:00000200:14.0:1616513076.558882:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7430 00000400:00000200:14.0:1616513076.558885:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.558887:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.558892:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7430 00000800:00000200:14.0:1616513076.558895:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.558897:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.558898:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.558901:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.558957:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665694850496 00010000:00000200:0.0:1616513076.558968:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688665694850496/t0(0) o400->7dccfbac-0cf9-ad39-0acf-955f5f732cce@10.31.136.76@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.558984:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665694850496, offset 192 00000400:00000200:0.0:1616513076.558991:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.76@tcp1 00000400:00000200:0.0:1616513076.558999:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.76@tcp1 routed destination 00000400:00000200:0.0:1616513076.559014:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.76@tcp1(10.31.136.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.559020:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.559026:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.559029:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.559031:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.559036:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.559041:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.559043:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513076.559101:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.559108:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.559111:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.559118:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.559123:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b783b8 00000800:00000200:22.0:1616513076.559127:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.559130:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.559284:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513076.559340:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.559349:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513076.559359:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.76@tcp1 : PUT - for me 00000400:00000200:28.0:1616513076.559368:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.76@tcp1 of length 224 into portal 12 MB=0x5ffd51ee2cf40 00000400:00000200:28.0:1616513076.559374:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.76@tcp1 of length 224/224 into md 0x865b9f31 [1] + 94592 00000400:00000200:28.0:1616513076.559379:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513076.559382:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513076.559388:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.559393:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.559394:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513076.559398:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.559469:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665694850880 00010000:00000200:4.0:1616513076.559482:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6722d00 x1688665694850880/t0(0) o400->d0ad4a89-bcbd-0e36-71e1-106052dbec9a@10.31.136.76@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.559507:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665694850880, offset 192 00000400:00000200:4.0:1616513076.559513:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.76@tcp1 00000400:00000200:4.0:1616513076.559521:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.76@tcp1 routed destination 00000400:00000200:4.0:1616513076.559536:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.76@tcp1(10.31.136.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.559543:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.559548:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.559551:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.559553:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.559558:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.559564:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.559575:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.559582:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.559585:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513076.559591:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.559595:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb650 00000800:00000200:12.0:1616513076.559599:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.559602:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513076.570577:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925072, portal 4 00000100:00000200:22.0:1616513076.570582:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925072, offset 0 00000400:00000200:22.0:1616513076.570589:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:22.0:1616513076.570597:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:22.0:1616513076.570610:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513076.570617:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:22.0:1616513076.570623:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513076.570627:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:22.0:1616513076.570629:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:22.0:1616513076.570634:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513076.570681:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513076.570701:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513076.570708:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513076.570711:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513076.570718:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513076.570726:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dd67d2400 x1693067725925072/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513076.570739:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b8f8 00000800:00000200:27.0:1616513076.570744:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513076.570747:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513076.571365:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513076.571375:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513076.571381:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513076.571389:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513076.571398:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626ad0 00000400:00000200:27.0:1616513076.571407:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716621d [1] + 192 00000400:00000200:27.0:1616513076.571413:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513076.571417:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dd67d2400 x1693067725925072/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:19.0:1616513076.571431:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513076.571432:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513076.571448:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513076.571450:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:22.0:1616513076.571454:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dd67d2400 x1693067725925072/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513076.571467:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dd67d2400 x1693067725925072/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513083 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513076.571478:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:7.2:1616513076.621847:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.621906:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.621917:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513076.621924:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513076.621927:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.251@tcp1 : PUT - for me 00000800:00000200:14.0:1616513076.621936:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.621938:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.251@tcp1 of length 224 into portal 12 MB=0x5ffe4aa9c4440 00000400:00000200:2.0:1616513076.621945:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.251@tcp1 of length 224/224 into md 0xf96450b9 [1] + 62736 00000400:00000200:14.0:1616513076.621946:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.251@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.621951:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.621954:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513076.621956:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.251@tcp1 of length 224 into portal 26 MB=0x5ffe4aa9c43c0 00000800:00000200:2.0:1616513076.621963:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513076.621964:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.251@tcp1 of length 224/224 into md 0xf97c6ce9 [1] + 0 00000800:00000200:2.0:1616513076.621966:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513076.621969:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c62a8 00000800:00000200:2.0:1616513076.621970:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.621973:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513076.621973:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513076.621976:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.621982:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c62a8 00000800:00000200:14.0:1616513076.621985:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.621989:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.621991:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513076.621993:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688732463547456 00010000:00000200:10.0:1616513076.622006:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623a80 x1688732463547456/t0(0) o400->5e5ae4a0-e656-45e3-4132-7c7d1f3193d6@10.31.141.251@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513076.622020:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688732463547456, offset 192 00000400:00000200:10.0:1616513076.622026:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.251@tcp1 00000400:00000200:10.0:1616513076.622032:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.251@tcp1 routed destination 00000400:00000200:10.0:1616513076.622045:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.251@tcp1(10.31.141.251@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513076.622047:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688732463547328 00000800:00000200:10.0:1616513076.622051:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513076.622057:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513076.622058:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688732463547328/t0(0) o400->aeef2111-73f6-a7eb-6462-a6baa45e928e@10.31.141.251@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:10.0:1616513076.622060:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.622062:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513076.622067:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.622072:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.622073:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688732463547328, offset 192 00000400:00000200:0.0:1616513076.622079:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.251@tcp1 00000800:00000200:26.0:1616513076.622085:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513076.622086:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.251@tcp1 routed destination 00000800:00000200:26.0:1616513076.622092:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.622096:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513076.622097:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513076.622100:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.251@tcp1(10.31.141.251@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513076.622103:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513076.622106:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513076.622107:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83c28 00000800:00000200:26.0:1616513076.622111:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513076.622111:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.622114:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.622116:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.622121:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513076.622128:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513076.622129:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.622185:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.622193:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.622196:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513076.622202:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513076.622202:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513076.622208:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78a18 00000800:00000200:16.0:1616513076.622211:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.624288:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513076.624302:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.624312:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.624321:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.172@tcp1 : PUT - for me 00000400:00000200:12.0:1616513076.624330:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.172@tcp1 of length 224 into portal 26 MB=0x5ffd42e6309c0 00000400:00000200:12.0:1616513076.624336:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.172@tcp1 of length 224/224 into md 0xf97c6cf9 [1] + 0 00000400:00000200:12.0:1616513076.624341:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7cb0 00000400:00000200:12.0:1616513076.624344:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.624347:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513076.624351:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7cb0 00000800:00000200:12.0:1616513076.624354:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.624356:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.624357:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.624363:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.624366:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.624367:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.624375:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.172@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.624384:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.172@tcp1 of length 224 into portal 12 MB=0x5ffd42e630a40 00000400:00000200:20.0:1616513076.624390:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.172@tcp1 of length 224/224 into md 0x865b9f31 [1] + 94816 00000400:00000200:20.0:1616513076.624396:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.624398:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.624405:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.624409:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.624410:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.624415:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661659945408 00010000:00000200:0.0:1616513076.624426:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688661659945408/t0(0) o400->01594e61-67fa-a5db-f071-2f62e14c95f5@10.31.138.172@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.624476:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661659945408, offset 192 00000100:00000200:4.0:1616513076.624479:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661659945536 00000400:00000200:0.0:1616513076.624483:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.172@tcp1 00010000:00000200:4.0:1616513076.624491:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e04800 x1688661659945536/t0(0) o400->74e0a036-c701-0a40-9591-2afaf8a16cda@10.31.138.172@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513076.624491:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.172@tcp1 routed destination 00000100:00000200:4.0:1616513076.624505:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661659945536, offset 192 00000400:00000200:0.0:1616513076.624508:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.172@tcp1(10.31.138.172@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:4.0:1616513076.624511:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.172@tcp1 00000800:00000200:0.0:1616513076.624514:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:4.0:1616513076.624518:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.172@tcp1 routed destination 00000400:00000200:4.0:1616513076.624532:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.172@tcp1(10.31.138.172@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.624538:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.624539:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.624541:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513076.624543:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.624544:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513076.624546:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.624548:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.624548:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513076.624552:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513076.624553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.624556:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513076.624557:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513076.624607:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.624614:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.624617:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.624623:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513076.624626:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513076.624629:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78908 00000800:00000200:22.0:1616513076.624633:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513076.624634:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513076.624635:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513076.624637:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.624638:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513076.624645:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513076.624649:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba440 00000800:00000200:16.0:1616513076.624653:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.634002:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.634058:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513076.634063:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513076.634075:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.634084:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.225@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.634093:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.225@tcp1 of length 224 into portal 12 MB=0x60005bb0b0b00 00000400:00000200:2.0:1616513076.634100:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.225@tcp1 of length 224/224 into md 0xf96450b9 [1] + 62960 00000400:00000200:2.0:1616513076.634105:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.634107:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513076.634114:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.634117:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.634118:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.634121:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513076.634136:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.634145:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.634154:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.225@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.634163:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.225@tcp1 of length 224 into portal 26 MB=0x60005bb0b0a40 00000400:00000200:20.0:1616513076.634170:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.225@tcp1 of length 224/224 into md 0xf97c6d19 [1] + 0 00000400:00000200:20.0:1616513076.634177:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7980 00000400:00000200:20.0:1616513076.634180:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513076.634181:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688874473163520 00000100:00000200:20.0:1616513076.634183:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513076.634187:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7980 00000800:00000200:28.0:1616513076.634190:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.634190:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.634193:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:10.0:1616513076.634193:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464622d00 x1688874473163520/t0(0) o400->f732b910-59bf-40f5-57ad-c586ed2f6b3f@10.31.136.225@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513076.634195:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513076.634207:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688874473163520, offset 192 00000400:00000200:10.0:1616513076.634215:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.225@tcp1 00000400:00000200:10.0:1616513076.634222:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.225@tcp1 routed destination 00000400:00000200:10.0:1616513076.634237:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.225@tcp1(10.31.136.225@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.634244:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513076.634250:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.634252:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.634255:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.634257:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688874473163328 00000800:00000200:10.0:1616513076.634260:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.634264:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513076.634268:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688874473163328/t0(0) o400->2e81b947-05f4-0840-d25e-e379e26124ef@10.31.136.225@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.634282:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688874473163328, offset 192 00000400:00000200:0.0:1616513076.634288:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.225@tcp1 00000400:00000200:0.0:1616513076.634296:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.225@tcp1 routed destination 00000400:00000200:0.0:1616513076.634311:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.225@tcp1(10.31.136.225@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.634317:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513076.634318:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513076.634323:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513076.634325:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513076.634326:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513076.634328:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513076.634328:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.634332:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513076.634335:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513076.634337:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513076.634339:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83b18 00000100:00000200:0.0:1616513076.634339:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513076.634343:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.634346:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513076.634394:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.634401:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.634403:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513076.634410:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.634416:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79540 00000800:00000200:12.0:1616513076.634419:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.634422:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.655889:0:49083:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.655945:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.655955:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.655965:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.215@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.655976:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.215@tcp1 of length 224 into portal 26 MB=0x5ffec39fca740 00000400:00000200:2.0:1616513076.655983:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.215@tcp1 of length 224/224 into md 0xf97c6d39 [1] + 0 00000400:00000200:2.0:1616513076.655988:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7760 00000400:00000200:2.0:1616513076.655990:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.655993:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513076.655997:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7760 00000800:00000200:26.0:1616513076.656000:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.656001:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.656004:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.656006:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.656061:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688764933777216 00010000:00000200:0.0:1616513076.656073:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688764933777216/t0(0) o400->693df54d-399e-3659-a176-8eaa46bc00b9@10.31.128.215@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.656086:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688764933777216, offset 192 00000400:00000200:0.0:1616513076.656092:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.215@tcp1 00000400:00000200:0.0:1616513076.656100:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.215@tcp1 routed destination 00000400:00000200:0.0:1616513076.656114:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.215@tcp1(10.31.128.215@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.656121:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.656127:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.656131:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.656133:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.656138:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.656143:0:49083:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.656145:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513076.656196:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.656203:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.656206:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.656213:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.656219:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78330 00000800:00000200:22.0:1616513076.656222:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.656225:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.656233:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513076.656235:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.656241:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.215@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.656250:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.215@tcp1 of length 224 into portal 12 MB=0x5ffec39fca900 00000400:00000200:22.0:1616513076.656256:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.215@tcp1 of length 224/224 into md 0xf96450b9 [1] + 63184 00000400:00000200:22.0:1616513076.656261:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.656264:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513076.656270:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.656273:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.656274:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513076.656332:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688764933777664 00010000:00000200:10.0:1616513076.656344:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2de80 x1688764933777664/t0(0) o400->2fc761ca-e373-84fb-3c20-9de096a212dc@10.31.128.215@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513076.656358:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688764933777664, offset 192 00000400:00000200:10.0:1616513076.656364:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.215@tcp1 00000400:00000200:10.0:1616513076.656371:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.215@tcp1 routed destination 00000400:00000200:10.0:1616513076.656384:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.215@tcp1(10.31.128.215@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.656391:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513076.656396:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.656398:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513076.656400:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513076.656405:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.656412:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.656469:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.656476:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.656479:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.656485:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.656489:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83a90 00000800:00000200:28.0:1616513076.656494:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.656497:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.679090:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.679147:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.679158:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.679167:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.174@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.679176:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.174@tcp1 of length 224 into portal 26 MB=0x604a2aabc5880 00000400:00000200:2.0:1616513076.679183:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.174@tcp1 of length 224/224 into md 0xf97c6d49 [1] + 0 00000400:00000200:2.0:1616513076.679187:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6440 00000400:00000200:2.0:1616513076.679192:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.679195:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513076.679199:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6440 00000800:00000200:26.0:1616513076.679202:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.679202:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.679205:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.679207:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.679259:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513076.679264:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946555947136 00010000:00000200:0.0:1616513076.679275:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1693946555947136/t0(0) o400->422de234-9fd6-70c8-89b9-3d5091fa9f88@10.31.140.174@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.679290:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946555947136, offset 192 00000400:00000200:0.0:1616513076.679296:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.174@tcp1 00000400:00000200:0.0:1616513076.679303:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.174@tcp1 routed destination 00000800:00000200:12.0:1616513076.679315:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513076.679319:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.174@tcp1(10.31.140.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513076.679325:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.679326:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.679332:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513076.679335:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.174@tcp1 : PUT - for me 00000800:00000200:0.0:1616513076.679335:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.679337:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.679342:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513076.679343:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.174@tcp1 of length 224 into portal 12 MB=0x604a2aabc5900 00000800:00000200:7.2:1616513076.679347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513076.679349:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.174@tcp1 of length 224/224 into md 0x865b9f31 [1] + 95040 00000100:00000200:0.0:1616513076.679349:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513076.679355:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.679358:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513076.679364:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.679367:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.679370:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.679374:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.679393:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946555947264 00000800:00000200:14.0:1616513076.679402:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:4.0:1616513076.679406:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c667980 x1693946555947264/t0(0) o400->eff7c4b7-58bb-7d43-410d-e671792fa7fb@10.31.140.174@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513076.679409:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.679412:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.679419:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:4.0:1616513076.679420:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946555947264, offset 192 00000400:00000200:14.0:1616513076.679424:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78b28 00000800:00000200:14.0:1616513076.679428:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513076.679430:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.174@tcp1 00000800:00000200:14.0:1616513076.679433:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513076.679449:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.174@tcp1 routed destination 00000400:00000200:4.0:1616513076.679464:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.174@tcp1(10.31.140.174@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.679473:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.679479:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.679482:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.679484:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.679489:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.679494:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.679547:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.679555:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.679557:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.679564:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.679574:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba990 00000800:00000200:28.0:1616513076.679578:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.679581:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.698023:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513076.698081:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513076.698091:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513076.698100:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.23@tcp1 : PUT - for me 00000400:00000200:26.0:1616513076.698110:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.23@tcp1 of length 224 into portal 26 MB=0x604a468760e00 00000400:00000200:26.0:1616513076.698116:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.23@tcp1 of length 224/224 into md 0xf97c6d59 [1] + 0 00000400:00000200:26.0:1616513076.698121:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7a08 00000400:00000200:26.0:1616513076.698124:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513076.698127:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513076.698131:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7a08 00000800:00000200:26.0:1616513076.698134:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.698136:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513076.698137:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513076.698140:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.698197:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954033978880 00010000:00000200:0.0:1616513076.698208:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1693954033978880/t0(0) o400->33b8a9e7-b8e6-bfe5-033c-dd1f3b8b3990@10.31.141.23@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.698223:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954033978880, offset 192 00000400:00000200:0.0:1616513076.698229:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.23@tcp1 00000400:00000200:0.0:1616513076.698237:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.23@tcp1 routed destination 00000400:00000200:0.0:1616513076.698252:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.23@tcp1(10.31.141.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.698259:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.698265:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.698268:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.698270:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513076.698273:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.698275:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.698281:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.698282:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513076.698328:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.698337:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513076.698346:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513076.698347:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.23@tcp1 : PUT - for me 00000800:00000200:14.0:1616513076.698353:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:16.0:1616513076.698355:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.23@tcp1 of length 224 into portal 12 MB=0x604a468760f40 00000400:00000200:14.0:1616513076.698356:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513076.698361:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.23@tcp1 of length 224/224 into md 0x865b9f31 [1] + 95264 00000400:00000200:14.0:1616513076.698363:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:16.0:1616513076.698367:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.698369:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79540 00000100:00000200:16.0:1616513076.698371:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513076.698372:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.698375:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513076.698378:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.698381:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.698383:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.698387:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.698457:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954033979200 00010000:00000200:4.0:1616513076.698470:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd3180 x1693954033979200/t0(0) o400->cf264c46-b1f3-1866-e4ce-e9a04c618736@10.31.141.23@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.698484:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954033979200, offset 192 00000400:00000200:4.0:1616513076.698491:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.23@tcp1 00000400:00000200:4.0:1616513076.698498:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.23@tcp1 routed destination 00000400:00000200:4.0:1616513076.698513:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.23@tcp1(10.31.141.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.698520:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.698526:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.698528:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.698531:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.698535:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.698540:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.698593:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.698601:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.698604:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.698610:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.698614:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb100 00000800:00000200:28.0:1616513076.698618:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.698621:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.700064:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.700117:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.700126:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.700135:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.213@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.700143:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.213@tcp1 of length 224 into portal 12 MB=0x60380049f51c0 00000400:00000200:2.0:1616513076.700150:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.213@tcp1 of length 224/224 into md 0xf96450b9 [1] + 63408 00000400:00000200:2.0:1616513076.700155:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.700158:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513076.700165:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.700168:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513076.700170:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.700174:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513076.700230:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692698228511168 00010000:00000200:10.0:1616513076.700243:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464626780 x1692698228511168/t0(0) o400->145cdcc3-f691-4e97-1c7a-65a69ef243e6@10.31.141.213@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513076.700252:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:10.0:1616513076.700257:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692698228511168, offset 192 00000400:00000200:10.0:1616513076.700263:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.213@tcp1 00000400:00000200:10.0:1616513076.700271:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.213@tcp1 routed destination 00000400:00000200:10.0:1616513076.700285:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.213@tcp1(10.31.141.213@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513076.700292:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513076.700298:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513076.700302:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513076.700304:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513076.700307:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513076.700309:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.700314:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513076.700316:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.700325:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.213@tcp1 : PUT - for me 00000400:00000200:12.0:1616513076.700333:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.213@tcp1 of length 224 into portal 26 MB=0x60380049f4f80 00000400:00000200:12.0:1616513076.700340:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.213@tcp1 of length 224/224 into md 0xf97c6d89 [1] + 0 00000400:00000200:12.0:1616513076.700345:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7b18 00000400:00000200:12.0:1616513076.700348:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.700351:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513076.700355:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7b18 00000800:00000200:12.0:1616513076.700358:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.700360:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.700361:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.700364:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513076.700384:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.700390:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.700393:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.700400:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.700404:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e836d8 00000800:00000200:14.0:1616513076.700407:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.700410:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.700419:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692698228510592 00010000:00000200:0.0:1616513076.700430:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1692698228510592/t0(0) o400->fc869d03-555a-7648-f7a8-ba6c633e2f9c@10.31.141.213@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.700457:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692698228510592, offset 192 00000400:00000200:0.0:1616513076.700464:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.213@tcp1 00000400:00000200:0.0:1616513076.700471:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.213@tcp1 routed destination 00000400:00000200:0.0:1616513076.700486:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.213@tcp1(10.31.141.213@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.700493:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.700498:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.700501:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.700503:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.700508:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.700514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.700515:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513076.700567:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.700574:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.700577:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.700583:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.700589:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78a18 00000800:00000200:28.0:1616513076.700592:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.700595:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.731447:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513076.731504:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513076.731514:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513076.731523:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.88@tcp1 : PUT - for me 00000400:00000200:26.0:1616513076.731532:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.88@tcp1 of length 224 into portal 26 MB=0x5ffde40f0c140 00000400:00000200:26.0:1616513076.731539:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.88@tcp1 of length 224/224 into md 0xf97c6d99 [1] + 0 00000400:00000200:26.0:1616513076.731543:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7210 00000400:00000200:26.0:1616513076.731546:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513076.731549:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513076.731553:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7210 00000800:00000200:26.0:1616513076.731557:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.731558:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.731560:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513076.731565:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513076.731568:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.731569:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.731578:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.88@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.731587:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.88@tcp1 of length 224 into portal 12 MB=0x5ffde40f0c3c0 00000400:00000200:2.0:1616513076.731594:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.88@tcp1 of length 224/224 into md 0xf96450b9 [1] + 63632 00000400:00000200:2.0:1616513076.731599:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.731602:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513076.731611:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.731614:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.731615:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.731619:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704920895808 00010000:00000200:0.0:1616513076.731630:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688704920895808/t0(0) o400->2e1d57b7-0586-e90e-f295-669f0debfa42@10.31.142.88@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.731645:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704920895808, offset 192 00000400:00000200:0.0:1616513076.731651:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.88@tcp1 00000400:00000200:0.0:1616513076.731658:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.88@tcp1 routed destination 00000400:00000200:0.0:1616513076.731674:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.88@tcp1(10.31.142.88@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513076.731681:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704920896448 00000800:00000200:0.0:1616513076.731681:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.731687:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.731690:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.731692:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:18.0:1616513076.731693:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464625e80 x1688704920896448/t0(0) o400->ed51f80e-f878-1af9-2c10-36087ae36130@10.31.142.88@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513076.731697:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.731702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.731703:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513076.731708:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704920896448, offset 192 00000400:00000200:18.0:1616513076.731714:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.88@tcp1 00000400:00000200:18.0:1616513076.731722:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.88@tcp1 routed destination 00000800:00000200:22.0:1616513076.731732:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513076.731737:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.88@tcp1(10.31.142.88@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513076.731740:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.731743:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513076.731744:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513076.731744:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513076.731749:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513076.731750:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513076.731752:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513076.731754:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513076.731755:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b784c8 00000800:00000200:22.0:1616513076.731759:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513076.731759:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.731806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.731864:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.731871:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.731873:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513076.731880:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513076.731884:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc198 00000800:00000200:16.0:1616513076.731888:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.731891:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.734201:0:49091:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.734255:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.734265:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513076.734268:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513076.734274:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.16@tcp1 : PUT - for me 00000800:00000200:14.0:1616513076.734277:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.734282:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.16@tcp1 of length 224 into portal 26 MB=0x6048d6cc59c00 00000400:00000200:14.0:1616513076.734285:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.16@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.734289:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.16@tcp1 of length 224/224 into md 0xf97c6dc9 [1] + 0 00000400:00000200:14.0:1616513076.734294:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.16@tcp1 of length 224 into portal 12 MB=0x6048d6cc5a000 00000400:00000200:2.0:1616513076.734296:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7a90 00000400:00000200:2.0:1616513076.734299:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.734301:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.16@tcp1 of length 224/224 into md 0xf96450b9 [1] + 63856 00000400:00000200:14.0:1616513076.734306:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.734306:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513076.734310:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7a90 00000100:00000200:14.0:1616513076.734312:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513076.734313:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513076.734316:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513076.734318:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.734319:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513076.734321:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.734322:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.734324:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.734375:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855322053632 00010000:00000200:0.0:1616513076.734386:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1693855322053632/t0(0) o400->7e475b49-fe11-925f-42b7-ce0f37b8f6e8@10.31.141.16@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513076.734391:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855322054656 00000100:00000200:0.0:1616513076.734400:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855322053632, offset 192 00010000:00000200:18.0:1616513076.734403:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645e2400 x1693855322054656/t0(0) o400->444dd722-809c-7fae-6bde-261c5bbb96a3@10.31.141.16@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513076.734408:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.16@tcp1 00000400:00000200:0.0:1616513076.734415:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.16@tcp1 routed destination 00000100:00000200:18.0:1616513076.734417:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855322054656, offset 192 00000400:00000200:18.0:1616513076.734424:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.16@tcp1 00000400:00000200:0.0:1616513076.734431:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.16@tcp1(10.31.141.16@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.734450:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513076.734452:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.16@tcp1 routed destination 00000800:00000200:0.0:1616513076.734456:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.734458:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.734461:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.734466:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513076.734467:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.16@tcp1(10.31.141.16@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513076.734471:0:49091:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.734473:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513076.734474:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513076.734480:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.734482:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513076.734484:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513076.734486:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513076.734491:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513076.734491:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513076.734494:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513076.734497:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.734501:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.734507:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79650 00000800:00000200:22.0:1616513076.734510:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.734534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.734589:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.734597:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.734599:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.734606:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.734612:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd298 00000800:00000200:28.0:1616513076.734616:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.734619:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.764896:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513076.764952:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.764962:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.764972:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.126@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.764981:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.126@tcp1 of length 224 into portal 26 MB=0x60182db7aa400 00000400:00000200:16.0:1616513076.764988:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.126@tcp1 of length 224/224 into md 0xf97c6dd9 [1] + 0 00000400:00000200:16.0:1616513076.764993:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c77e8 00000400:00000200:16.0:1616513076.764996:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.764999:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513076.765004:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c77e8 00000800:00000200:16.0:1616513076.765007:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.765010:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.765012:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.765014:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.765069:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511399887872 00010000:00000200:0.0:1616513076.765081:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1690511399887872/t0(0) o400->3d6aece4-5763-ba8d-b0ca-cbd37f41dbb0@10.31.136.126@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.765096:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511399887872, offset 192 00000400:00000200:0.0:1616513076.765102:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.126@tcp1 00000400:00000200:0.0:1616513076.765110:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.126@tcp1 routed destination 00000400:00000200:0.0:1616513076.765125:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.126@tcp1(10.31.136.126@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.765132:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.765138:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.765141:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.765143:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.765148:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513076.765155:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513076.765156:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.765212:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.765219:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.765222:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513076.765229:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.765234:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b793a8 00000800:00000200:2.0:1616513076.765238:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.765241:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.765284:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513076.765356:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.765366:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513076.765375:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.126@tcp1 : PUT - for me 00000400:00000200:28.0:1616513076.765384:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.126@tcp1 of length 224 into portal 12 MB=0x60182db7aa500 00000400:00000200:28.0:1616513076.765390:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.126@tcp1 of length 224/224 into md 0x865b9f31 [1] + 95488 00000400:00000200:28.0:1616513076.765396:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513076.765398:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513076.765405:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.765409:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513076.765411:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.765415:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.765471:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511399888128 00010000:00000200:4.0:1616513076.765485:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6724c80 x1690511399888128/t0(0) o400->3f0928de-9ee6-c148-ae54-83d2c4ab4cb7@10.31.136.126@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.765499:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511399888128, offset 192 00000400:00000200:4.0:1616513076.765506:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.126@tcp1 00000400:00000200:4.0:1616513076.765513:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.126@tcp1 routed destination 00000400:00000200:4.0:1616513076.765528:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.126@tcp1(10.31.136.126@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.765535:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.765541:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.765543:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.765545:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.765550:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.765555:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.765566:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.765573:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.765576:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513076.765578:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513076.765582:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.765586:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb078 00000800:00000200:12.0:1616513076.765590:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.808789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513076.808847:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513076.808857:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513076.808867:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.201@tcp1 : PUT - for me 00000400:00000200:26.0:1616513076.808876:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.201@tcp1 of length 224 into portal 12 MB=0x5ffd5191d1dc0 00000400:00000200:26.0:1616513076.808883:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.201@tcp1 of length 224/224 into md 0xf96450b9 [1] + 64080 00000400:00000200:26.0:1616513076.808889:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513076.808892:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513076.808899:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.808902:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.808902:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513076.808906:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513076.808966:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665598008768 00010000:00000200:18.0:1616513076.808979:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647da880 x1688665598008768/t0(0) o400->3b3c1bab-e211-3771-4504-a6086c3cc445@10.31.132.201@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513076.808986:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513076.808994:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665598008768, offset 192 00000400:00000200:18.0:1616513076.809001:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.201@tcp1 00000400:00000200:18.0:1616513076.809009:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.201@tcp1 routed destination 00000400:00000200:18.0:1616513076.809023:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.201@tcp1(10.31.132.201@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513076.809030:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513076.809036:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.809039:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513076.809041:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513076.809044:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513076.809046:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.809052:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513076.809054:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.809063:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.201@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.809072:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.201@tcp1 of length 224 into portal 26 MB=0x5ffd5191d1d40 00000400:00000200:20.0:1616513076.809079:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.201@tcp1 of length 224/224 into md 0xf97c6df9 [1] + 0 00000400:00000200:20.0:1616513076.809084:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c76d8 00000400:00000200:20.0:1616513076.809087:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.809089:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513076.809094:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c76d8 00000800:00000200:20.0:1616513076.809098:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.809099:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.809100:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513076.809103:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513076.809120:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.809127:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.809130:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513076.809137:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.809141:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd298 00000800:00000200:2.0:1616513076.809145:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.809148:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.809161:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665598008640 00010000:00000200:0.0:1616513076.809172:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688665598008640/t0(0) o400->08fd1e1d-815f-ec97-0d9e-04a06dbd1f64@10.31.132.201@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.809187:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665598008640, offset 192 00000400:00000200:0.0:1616513076.809193:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.201@tcp1 00000400:00000200:0.0:1616513076.809200:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.201@tcp1 routed destination 00000400:00000200:0.0:1616513076.809215:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.201@tcp1(10.31.132.201@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.809226:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513076.809232:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.809235:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.809237:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.809242:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.809247:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.809249:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513076.809317:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.809324:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.809326:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513076.809333:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513076.809339:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79c28 00000800:00000200:16.0:1616513076.809342:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.809346:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.821642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513076.821698:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.821707:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513076.821717:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.27@tcp1 : PUT - for me 00000400:00000200:28.0:1616513076.821726:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.27@tcp1 of length 224 into portal 26 MB=0x60393d4c10540 00000400:00000200:28.0:1616513076.821733:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.27@tcp1 of length 224/224 into md 0xf97c6e09 [1] + 0 00000400:00000200:28.0:1616513076.821738:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6c38 00000400:00000200:28.0:1616513076.821741:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513076.821744:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513076.821749:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6c38 00000800:00000200:28.0:1616513076.821752:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.821752:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.821754:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513076.821759:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513076.821762:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.821763:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.821771:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.27@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.821780:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.27@tcp1 of length 224 into portal 12 MB=0x60393d4c10780 00000400:00000200:20.0:1616513076.821787:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.27@tcp1 of length 224/224 into md 0x865b9f31 [1] + 95712 00000400:00000200:20.0:1616513076.821792:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.821794:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.821801:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.821804:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513076.821806:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.821811:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783324759360 00010000:00000200:0.0:1616513076.821822:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1692783324759360/t0(0) o400->517a7987-8a50-1304-7c3f-631ae539d822@10.31.140.27@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.821837:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783324759360, offset 192 00000400:00000200:0.0:1616513076.821843:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.27@tcp1 00000400:00000200:0.0:1616513076.821851:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.27@tcp1 routed destination 00000100:00000200:4.0:1616513076.821865:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783324759936 00000400:00000200:0.0:1616513076.821866:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.27@tcp1(10.31.140.27@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.821872:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:4.0:1616513076.821878:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e04c80 x1692783324759936/t0(0) o400->d44240a8-e40a-8bca-b30e-9b63fc2b0579@10.31.140.27@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513076.821878:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.821881:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.821883:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.821888:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513076.821893:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783324759936, offset 192 00000100:00000200:0.0:1616513076.821895:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513076.821896:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513076.821899:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.27@tcp1 00000400:00000200:4.0:1616513076.821906:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.27@tcp1 routed destination 00000400:00000200:4.0:1616513076.821922:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.27@tcp1(10.31.140.27@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.821928:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.821934:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.821936:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.821939:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.821943:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.821949:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513076.821950:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.821958:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.821960:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513076.821961:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513076.821967:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513076.821968:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.821971:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513076.821972:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513076.821973:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79c28 00000800:00000200:22.0:1616513076.821976:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513076.821977:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513076.821979:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513076.821981:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba4c8 00000800:00000200:12.0:1616513076.821985:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.840615:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513076.840671:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.840681:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513076.840685:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513076.840691:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.139@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.840701:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.139@tcp1 of length 224 into portal 26 MB=0x6001f93c8a2c0 00000400:00000200:2.0:1616513076.840708:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.139@tcp1 of length 224/224 into md 0xf97c6e19 [1] + 0 00000400:00000200:2.0:1616513076.840712:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7078 00000400:00000200:2.0:1616513076.840715:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.840717:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513076.840722:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7078 00000800:00000200:2.0:1616513076.840725:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.840728:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.840729:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.840785:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688985483649728 00010000:00000200:0.0:1616513076.840796:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688985483649728/t0(0) o400->b42e3582-0292-cd53-996f-0f34729536a9@10.31.142.139@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.840809:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688985483649728, offset 192 00000400:00000200:0.0:1616513076.840815:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.139@tcp1 00000400:00000200:0.0:1616513076.840822:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.139@tcp1 routed destination 00000400:00000200:0.0:1616513076.840836:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.139@tcp1(10.31.142.139@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.840842:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.840848:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.840850:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.840854:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.840859:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.840864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.840865:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513076.840917:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.840924:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.840927:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.840933:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.840939:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79650 00000800:00000200:22.0:1616513076.840944:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.840947:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.841785:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513076.841841:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513076.841851:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513076.841860:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.139@tcp1 : PUT - for me 00000400:00000200:20.0:1616513076.841870:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.139@tcp1 of length 224 into portal 12 MB=0x6001f93c8b840 00000400:00000200:20.0:1616513076.841877:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.139@tcp1 of length 224/224 into md 0x865b9f31 [1] + 95936 00000400:00000200:20.0:1616513076.841882:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513076.841885:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513076.841891:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.841894:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.841896:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513076.841900:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.841954:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688985483655232 00010000:00000200:4.0:1616513076.841967:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05580 x1688985483655232/t0(0) o400->b9f8b5c0-f3b4-5fb4-2110-c041bc90ea78@10.31.142.139@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513076.841981:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688985483655232, offset 192 00000400:00000200:4.0:1616513076.841988:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.139@tcp1 00000400:00000200:4.0:1616513076.841995:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.139@tcp1 routed destination 00000400:00000200:4.0:1616513076.842010:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.139@tcp1(10.31.142.139@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513076.842016:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513076.842022:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513076.842025:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513076.842027:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513076.842032:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.842037:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.842091:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.842098:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.842100:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513076.842103:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513076.842107:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.842111:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba4c8 00000800:00000200:28.0:1616513076.842117:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.869577:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513076.869632:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.869641:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.869650:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.173@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.869660:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.173@tcp1 of length 224 into portal 26 MB=0x5ffd3d5b44240 00000400:00000200:14.0:1616513076.869667:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.173@tcp1 of length 224/224 into md 0xf97c6e29 [1] + 0 00000400:00000200:14.0:1616513076.869672:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6f68 00000400:00000200:14.0:1616513076.869674:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.869679:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.869683:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6f68 00000800:00000200:2.0:1616513076.869685:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.869686:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.869689:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513076.869691:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.869747:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660172096064 00010000:00000200:0.0:1616513076.869757:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688660172096064/t0(0) o400->f68c82b9-34bc-02b7-84e6-db8a8a04f29a@10.31.128.173@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.869770:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660172096064, offset 192 00000400:00000200:0.0:1616513076.869776:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.173@tcp1 00000400:00000200:0.0:1616513076.869783:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.173@tcp1 routed destination 00000400:00000200:0.0:1616513076.869797:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.173@tcp1(10.31.128.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.869804:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.869809:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.869812:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.869814:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.869818:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.869824:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.869825:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513076.869877:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513076.869884:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513076.869887:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513076.869893:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513076.869899:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78a18 00000800:00000200:22.0:1616513076.869902:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.869905:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.869911:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513076.869913:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.869919:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.173@tcp1 : PUT - for me 00000400:00000200:22.0:1616513076.869927:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.173@tcp1 of length 224 into portal 12 MB=0x5ffd3d5b445c0 00000400:00000200:22.0:1616513076.869934:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.173@tcp1 of length 224/224 into md 0xf96450b9 [1] + 64304 00000400:00000200:22.0:1616513076.869939:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513076.869941:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513076.869948:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513076.869951:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.869953:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513076.870013:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660172096960 00010000:00000200:18.0:1616513076.870025:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a29200 x1688660172096960/t0(0) o400->5d0f2978-aa69-540a-314a-f8460385d21a@10.31.128.173@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513076.870040:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660172096960, offset 192 00000400:00000200:18.0:1616513076.870046:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.173@tcp1 00000400:00000200:18.0:1616513076.870054:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.173@tcp1 routed destination 00000400:00000200:18.0:1616513076.870069:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.173@tcp1(10.31.128.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513076.870076:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513076.870081:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.870084:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513076.870086:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513076.870091:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.870138:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.870198:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.870205:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513076.870208:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513076.870214:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513076.870219:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc198 00000800:00000200:16.0:1616513076.870222:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.870226:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:23.0:1616513076.870603:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:23.0:1616513076.870611:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:7.2:1616513076.938121:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513076.938178:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513076.938191:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.938201:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.175@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.938210:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.175@tcp1 of length 224 into portal 26 MB=0x5ffdceb56c540 00000400:00000200:14.0:1616513076.938217:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.175@tcp1 of length 224/224 into md 0xf97c6e49 [1] + 0 00000400:00000200:14.0:1616513076.938222:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c63b8 00000400:00000200:14.0:1616513076.938224:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.938227:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.938232:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c63b8 00000800:00000200:14.0:1616513076.938235:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.938236:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.938238:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.938254:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513076.938256:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513076.938257:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513076.938265:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.175@tcp1 : PUT - for me 00000400:00000200:2.0:1616513076.938274:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.175@tcp1 of length 224 into portal 12 MB=0x5ffdceb56c640 00000400:00000200:2.0:1616513076.938280:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.175@tcp1 of length 224/224 into md 0xf96450b9 [1] + 64528 00000400:00000200:2.0:1616513076.938285:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.938288:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513076.938295:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.938299:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.938301:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.938326:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699189773632 00010000:00000200:0.0:1616513076.938337:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688699189773632/t0(0) o400->2a8dde79-9429-86ff-7f41-481c1262443a@10.31.141.175@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.938351:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699189773632, offset 192 00000400:00000200:0.0:1616513076.938358:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.175@tcp1 00000100:00000200:18.0:1616513076.938360:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699189773888 00000400:00000200:0.0:1616513076.938366:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.175@tcp1 routed destination 00010000:00000200:18.0:1616513076.938373:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464622880 x1688699189773888/t0(0) o400->c8c11ce1-d1cc-f966-1178-cd5224514b32@10.31.141.175@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513076.938381:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.175@tcp1(10.31.141.175@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513076.938388:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699189773888, offset 192 00000800:00000200:0.0:1616513076.938388:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.938394:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513076.938396:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.175@tcp1 00000800:00000200:0.0:1616513076.938397:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.938399:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513076.938404:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.175@tcp1 routed destination 00000800:00000200:0.0:1616513076.938404:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.938409:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.938411:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513076.938419:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.175@tcp1(10.31.141.175@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513076.938426:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513076.938426:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513076.938432:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.938435:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513076.938439:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513076.938445:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513076.938448:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513076.938450:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513076.938455:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513076.938455:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513076.938461:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78b28 00000800:00000200:22.0:1616513076.938464:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.938502:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.938558:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.938565:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.938567:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513076.938574:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.938578:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc880 00000800:00000200:12.0:1616513076.938582:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.938585:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513076.943913:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513076.943971:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.943980:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.943989:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.60@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.943997:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.60@tcp1 of length 224 into portal 26 MB=0x5ffdc4f0ab3c0 00000400:00000200:16.0:1616513076.944004:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.60@tcp1 of length 224/224 into md 0xf97c6e59 [1] + 0 00000400:00000200:16.0:1616513076.944009:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7e48 00000400:00000200:16.0:1616513076.944012:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.944015:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513076.944019:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7e48 00000800:00000200:16.0:1616513076.944022:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513076.944024:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.944025:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513076.944028:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513076.944081:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696567542720 00010000:00000200:0.0:1616513076.944092:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688696567542720/t0(0) o400->4e4d74ef-5081-e985-d191-e80184b491fb@10.31.142.60@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513076.944108:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696567542720, offset 192 00000400:00000200:0.0:1616513076.944115:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.60@tcp1 00000400:00000200:0.0:1616513076.944122:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.60@tcp1 routed destination 00000400:00000200:0.0:1616513076.944137:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.60@tcp1(10.31.142.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513076.944144:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.944150:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.944152:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513076.944155:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513076.944159:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513076.944165:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.944166:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513076.944219:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.944226:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.944229:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513076.944235:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.944241:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79650 00000800:00000200:2.0:1616513076.944244:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513076.944247:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513076.944681:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513076.944735:0:49113:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513076.944737:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.944746:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513076.944755:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.60@tcp1 : PUT - for me 00000400:00000200:12.0:1616513076.944764:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.60@tcp1 of length 224 into portal 12 MB=0x5ffdc4f0ac140 00000400:00000200:12.0:1616513076.944770:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.60@tcp1 of length 224/224 into md 0x865b9f31 [1] + 96160 00000400:00000200:12.0:1616513076.944775:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d84893c2b28 00000400:00000200:12.0:1616513076.944778:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513076.944780:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513076.944785:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84893c2b28 00000800:00000200:12.0:1616513076.944788:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513076.944789:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513076.944791:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513076.944793:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513076.944795:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513076.944798:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513076.944807:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.245@tcp1 : PUT - for me 00000400:00000200:14.0:1616513076.944818:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.245@tcp1 of length 224 into portal 26 MB=0x604192f1ab900 00000400:00000200:14.0:1616513076.944824:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.245@tcp1 of length 224/224 into md 0xf97c6e89 [1] + 0 00000400:00000200:14.0:1616513076.944829:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6770 00000400:00000200:14.0:1616513076.944832:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513076.944835:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513076.944839:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6770 00000800:00000200:14.0:1616513076.944842:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513076.944845:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513076.944847:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513076.944849:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513076.944853:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356071237888 00010000:00000200:0.0:1616513076.944859:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1693356071237888/t0(0) o400->6a416e27-2d97-4254-0b63-05ecd0776242@10.31.128.245@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513076.944866:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696567546176 00000100:00000200:0.0:1616513076.944871:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693356071237888, offset 192 00000400:00000200:0.0:1616513076.944876:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.245@tcp1 00010000:00000200:4.0:1616513076.944879:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc2e86050 x1688696567546176/t0(0) o400->ba91fbd1-fb75-be65-395b-96b16fc10c38@10.31.142.60@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513076.944883:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.245@tcp1 routed destination 00000100:00000200:4.0:1616513076.944893:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696567546176, offset 192 00000400:00000200:0.0:1616513076.944897:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.245@tcp1(10.31.128.245@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513076.944898:0:49113:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:4.0:1616513076.944900:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.60@tcp1 00000800:00000200:0.0:1616513076.944902:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:4.0:1616513076.944907:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.60@tcp1 routed destination 00000800:00000200:2.0:1616513076.944908:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513076.944914:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513076.944916:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513076.944917:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513076.944918:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513076.944920:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513076.944923:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513076.944924:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.60@tcp1(10.31.142.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513076.944928:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513076.944928:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513076.944931:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:2.0:1616513076.944933:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.245@tcp1 : PUT - for me 00000800:00000200:4.0:1616513076.944937:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513076.944939:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513076.944940:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513076.944942:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513076.944942:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.245@tcp1 of length 224 into portal 12 MB=0x604192f1abc40 00000800:00000200:20.0:1616513076.944944:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513076.944947:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513076.944947:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513076.944948:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.245@tcp1 of length 224/224 into md 0xf96450b9 [1] + 64752 00000800:00000200:16.0:1616513076.944949:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513076.944952:0:49113:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513076.944953:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513076.944954:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513076.944956:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513076.944959:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79650 00000800:00000200:20.0:1616513076.944962:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513076.944962:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513076.944962:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513076.944965:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:14.0:1616513076.944966:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:2.0:1616513076.944966:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513076.944969:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513076.944970:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513076.944976:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513076.944979:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb078 00000800:00000200:14.0:1616513076.944982:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513076.945026:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356071238720 00010000:00000200:18.0:1616513076.945038:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464620480 x1693356071238720/t0(0) o400->386b2c53-58b3-40e5-f77e-6cc37390aa99@10.31.128.245@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513076.945053:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693356071238720, offset 192 00000400:00000200:18.0:1616513076.945060:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.245@tcp1 00000400:00000200:18.0:1616513076.945067:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.245@tcp1 routed destination 00000400:00000200:18.0:1616513076.945082:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.245@tcp1(10.31.128.245@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513076.945089:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513076.945095:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.945097:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513076.945099:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513076.945104:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.945109:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513076.945118:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513076.945122:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513076.945126:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513076.945127:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513076.945132:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513076.945135:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cddc0 00000800:00000200:12.0:1616513076.945138:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513076.945342:0:188420:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 12 00000800:00000200:11.2:1616513076.989797:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513076.989854:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513076.989863:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513076.989872:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.93@tcp1 : PUT - for me 00000400:00000200:16.0:1616513076.989882:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.93@tcp1 of length 224 into portal 26 MB=0x5ffdcf90ac080 00000400:00000200:16.0:1616513076.989891:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.93@tcp1 of length 224/224 into md 0xf97c6e99 [1] + 0 00000400:00000200:16.0:1616513076.989896:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c74b8 00000400:00000200:16.0:1616513076.989898:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513076.989901:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513076.989906:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c74b8 00000800:00000200:20.0:1616513076.989907:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.989909:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513076.989912:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513076.989914:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513076.989936:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513076.989970:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699419672704 00010000:00000200:0.0:1616513076.989981:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688699419672704/t0(0) o400->a071fa89-5cde-45b9-aa4b-2cfe171820f5@10.31.138.93@tcp1:33/0 lens 224/224 e 0 to 0 dl 1616513168 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513076.989993:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513076.989996:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699419672704, offset 192 00000400:00000200:0.0:1616513076.990002:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.93@tcp1 00000800:00000200:26.0:1616513076.990003:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513076.990010:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.93@tcp1 routed destination 00000400:00000200:26.0:1616513076.990012:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.93@tcp1 : PUT - for me 00000400:00000200:0.0:1616513076.990029:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.93@tcp1(10.31.138.93@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513076.990036:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.93@tcp1 of length 224 into portal 12 MB=0x5ffdcf90ac240 00000800:00000200:0.0:1616513076.990036:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513076.990042:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.93@tcp1 of length 224/224 into md 0xf96450b9 [1] + 64976 00000800:00000200:0.0:1616513076.990042:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513076.990045:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513076.990047:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513076.990048:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513076.990048:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000100:00000200:26.0:1616513076.990051:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513076.990053:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513076.990057:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:7.2:1616513076.990058:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.990060:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:0.0:1616513076.990060:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513076.990063:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513076.990065:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513076.990069:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513076.990071:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513076.990077:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513076.990081:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79650 00000800:00000200:26.0:1616513076.990085:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513076.990087:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513076.990122:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699419673152 00010000:00000200:18.0:1616513076.990137:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dda00 x1688699419673152/t0(0) o400->d42f061a-5a09-fddb-9d32-58e53a155954@10.31.138.93@tcp1:702/0 lens 224/224 e 0 to 0 dl 1616513082 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513076.990151:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699419673152, offset 192 00000400:00000200:18.0:1616513076.990157:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.93@tcp1 00000400:00000200:18.0:1616513076.990165:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.93@tcp1 routed destination 00000400:00000200:18.0:1616513076.990180:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.93@tcp1(10.31.138.93@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513076.990186:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513076.990192:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513076.990196:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513076.990198:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513076.990203:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513076.990209:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513076.990262:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513076.990268:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513076.990271:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513076.990278:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513076.990282:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd650 00000800:00000200:28.0:1616513076.990286:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513076.990289:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.008412:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.008467:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.008477:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.008486:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.150@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.008495:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.150@tcp1 of length 224 into portal 26 MB=0x5ffd55c524fc0 00000400:00000200:20.0:1616513077.008502:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.150@tcp1 of length 224/224 into md 0xf97c6ea9 [1] + 0 00000400:00000200:20.0:1616513077.008506:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c78f8 00000400:00000200:20.0:1616513077.008510:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.008512:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513077.008517:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c78f8 00000800:00000200:20.0:1616513077.008520:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.008522:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.008522:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.008526:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.008580:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666725568448 00010000:00000200:0.0:1616513077.008591:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688666725568448/t0(0) o400->5aa66f3c-8d03-aedd-6d0f-e79b7bcbcfa0@10.31.132.150@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.008605:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666725568448, offset 192 00000400:00000200:0.0:1616513077.008612:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.150@tcp1 00000400:00000200:0.0:1616513077.008620:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.150@tcp1 routed destination 00000400:00000200:0.0:1616513077.008635:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.150@tcp1(10.31.132.150@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.008641:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.008647:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.008650:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.008652:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.008657:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.008664:0:49120:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513077.008664:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513077.008720:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.008727:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.008731:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.008738:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.008744:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04660 00000800:00000200:2.0:1616513077.008747:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.008750:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.009295:0:49120:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.009350:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.009359:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.009368:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.150@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.009377:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.150@tcp1 of length 224 into portal 12 MB=0x5ffd55c5251c0 00000400:00000200:14.0:1616513077.009384:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.150@tcp1 of length 224/224 into md 0xf96450b9 [1] + 65200 00000400:00000200:14.0:1616513077.009389:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.009391:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513077.009399:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.009402:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.009404:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.009408:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513077.009465:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666725568960 00010000:00000200:18.0:1616513077.009477:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba6300 x1688666725568960/t0(0) o400->3a7857d2-486c-3bf6-e6da-d784e8a9bd48@10.31.132.150@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513077.009492:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666725568960, offset 192 00000400:00000200:18.0:1616513077.009498:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.150@tcp1 00000400:00000200:18.0:1616513077.009506:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.150@tcp1 routed destination 00000400:00000200:18.0:1616513077.009533:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.150@tcp1(10.31.132.150@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513077.009539:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.009544:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.009547:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.009549:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.009553:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.009606:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.009663:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.009669:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.009672:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.009679:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.009683:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdd38 00000800:00000200:28.0:1616513077.009686:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.009689:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.017796:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.017859:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.017868:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513077.017877:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.130@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.017887:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.130@tcp1 of length 224 into portal 26 MB=0x5ffdd440d61c0 00000400:00000200:16.0:1616513077.017893:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.130@tcp1 of length 224/224 into md 0xf97c6ec9 [1] + 0 00000400:00000200:16.0:1616513077.017898:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c73a8 00000400:00000200:16.0:1616513077.017901:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.017903:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513077.017908:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c73a8 00000800:00000200:16.0:1616513077.017911:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.017913:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.017914:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.017920:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.017923:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.017924:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.017932:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.130@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.017942:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.130@tcp1 of length 224 into portal 12 MB=0x5ffdd440d6240 00000400:00000200:20.0:1616513077.017948:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.130@tcp1 of length 224/224 into md 0x865bbec1 [1] + 0 00000400:00000200:20.0:1616513077.017955:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.017958:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513077.017964:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.017968:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.017969:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.017971:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700678136256 00010000:00000200:0.0:1616513077.017982:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688700678136256/t0(0) o400->c60be113-867b-9435-dd1a-6996716e6db1@10.31.142.130@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.017997:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700678136256, offset 192 00000400:00000200:0.0:1616513077.018003:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.130@tcp1 00000400:00000200:0.0:1616513077.018011:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.130@tcp1 routed destination 00000400:00000200:0.0:1616513077.018026:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.130@tcp1(10.31.142.130@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513077.018028:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700678136384 00000800:00000200:0.0:1616513077.018032:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.018038:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:4.0:1616513077.018041:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05100 x1688700678136384/t0(0) o400->b4f84970-6312-7d03-19d7-379735ac3db4@10.31.142.130@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513077.018042:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.018044:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.018049:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513077.018055:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700678136384, offset 192 00000800:00000200:7.2:1616513077.018056:0:49121:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513077.018056:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:4.0:1616513077.018061:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.130@tcp1 00000400:00000200:4.0:1616513077.018069:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.130@tcp1 routed destination 00000400:00000200:4.0:1616513077.018084:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.130@tcp1(10.31.142.130@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513077.018090:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513077.018096:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513077.018099:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513077.018101:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513077.018106:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513077.018110:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513077.018111:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513077.018116:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.018119:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513077.018123:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513077.018126:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513077.018130:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513077.018131:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05870 00000400:00000200:12.0:1616513077.018133:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513077.018134:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513077.018135:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.018138:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513077.018140:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513077.018144:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba990 00000800:00000200:12.0:1616513077.018148:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.039713:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.039768:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.039777:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.039786:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.111@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.039795:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.111@tcp1 of length 224 into portal 26 MB=0x6018929d57f40 00000400:00000200:22.0:1616513077.039801:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.111@tcp1 of length 224/224 into md 0xf97c6ef9 [1] + 0 00000400:00000200:22.0:1616513077.039805:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6220 00000400:00000200:22.0:1616513077.039807:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.039812:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.039816:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6220 00000800:00000200:22.0:1616513077.039818:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.039821:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.039822:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.039827:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513077.039878:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690538484268864 00010000:00000200:0.0:1616513077.039889:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1690538484268864/t0(0) o400->8fc8e258-7cd4-eadd-fbf4-0e2678eeda66@10.31.138.111@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.039902:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690538484268864, offset 192 00000400:00000200:0.0:1616513077.039908:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.111@tcp1 00000400:00000200:0.0:1616513077.039915:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.111@tcp1 routed destination 00000400:00000200:0.0:1616513077.039929:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.111@tcp1(10.31.138.111@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.039935:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.039941:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.039943:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.039945:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513077.039950:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.039950:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.039955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513077.039956:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513077.040005:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.040014:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.040023:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.111@tcp1 : PUT - for me 00000800:00000200:2.0:1616513077.040025:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513077.040031:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.111@tcp1 of length 224 into portal 12 MB=0x6018929d57fc0 00000800:00000200:2.0:1616513077.040043:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513077.040045:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.111@tcp1 of length 224/224 into md 0x865bbec1 [1] + 224 00000800:00000200:26.0:1616513077.040046:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513077.040046:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.040050:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.040052:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:2.0:1616513077.040053:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513077.040059:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513077.040061:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05210 00000800:00000200:28.0:1616513077.040062:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.040062:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513077.040064:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513077.040066:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513077.040122:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690538484268992 00010000:00000200:4.0:1616513077.040135:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e02880 x1690538484268992/t0(0) o400->20e9eb12-2bef-b77b-ae87-9889af8b8b7e@10.31.138.111@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513077.040150:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690538484268992, offset 192 00000400:00000200:4.0:1616513077.040156:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.111@tcp1 00000400:00000200:4.0:1616513077.040163:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.111@tcp1 routed destination 00000400:00000200:4.0:1616513077.040178:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.111@tcp1(10.31.138.111@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513077.040184:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513077.040190:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513077.040193:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513077.040197:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513077.040202:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.040207:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.040260:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.040267:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.040269:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513077.040272:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513077.040276:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.040280:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba440 00000800:00000200:16.0:1616513077.040284:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.040471:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.040482:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.040490:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513077.040492:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513077.040498:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.50@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.040508:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.50@tcp1 of length 224 into portal 26 MB=0x6000108a48ec0 00000400:00000200:14.0:1616513077.040514:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.50@tcp1 of length 224/224 into md 0xf97c6f29 [1] + 0 00000400:00000200:14.0:1616513077.040519:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7100 00000400:00000200:14.0:1616513077.040522:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.040525:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513077.040529:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7100 00000800:00000200:14.0:1616513077.040532:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.040535:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.040537:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513077.040543:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688854300233408 00010000:00000200:0.0:1616513077.040549:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688854300233408/t0(0) o400->04a33636-8dc7-6b33-adad-3af60439b960@10.31.133.50@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.040561:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688854300233408, offset 192 00000400:00000200:0.0:1616513077.040566:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.50@tcp1 00000400:00000200:0.0:1616513077.040572:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.50@tcp1 routed destination 00000400:00000200:0.0:1616513077.040588:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.50@tcp1(10.31.133.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.040594:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.040599:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.040601:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.040603:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.040606:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513077.040610:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.040611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.040620:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513077.040625:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513077.040629:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513077.040629:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513077.040634:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513077.040639:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:26.0:1616513077.040642:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.040882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.040891:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.040896:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513077.040899:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513077.040903:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.50@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.040910:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.50@tcp1 of length 224 into portal 12 MB=0x6000108a49000 00000400:00000200:22.0:1616513077.040915:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.50@tcp1 of length 224/224 into md 0xf96450b9 [1] + 65424 00000400:00000200:22.0:1616513077.040918:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.040920:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.040926:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.040927:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.040929:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513077.040992:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688854300233728 00010000:00000200:18.0:1616513077.041004:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2b180 x1688854300233728/t0(0) o400->3196d1c7-7103-4bfe-1a66-c2d7eec6de2b@10.31.133.50@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513077.041019:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688854300233728, offset 192 00000400:00000200:18.0:1616513077.041025:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.50@tcp1 00000400:00000200:18.0:1616513077.041032:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.50@tcp1 routed destination 00000400:00000200:18.0:1616513077.041048:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.50@tcp1(10.31.133.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513077.041054:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.041060:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.041062:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.041065:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.041069:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.041075:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.041086:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.041091:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.041094:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513077.041096:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513077.041100:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.041104:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc990 00000800:00000200:28.0:1616513077.041108:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.041758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513077.041829:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513077.041838:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513077.041847:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.86@tcp1 : PUT - for me 00000400:00000200:12.0:1616513077.041857:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.86@tcp1 of length 224 into portal 26 MB=0x6018e288c4ec0 00000400:00000200:12.0:1616513077.041863:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.86@tcp1 of length 224/224 into md 0xf97c6f39 [1] + 0 00000400:00000200:12.0:1616513077.041868:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7c28 00000400:00000200:12.0:1616513077.041871:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513077.041873:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513077.041878:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7c28 00000800:00000200:12.0:1616513077.041881:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.041882:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513077.041884:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513077.041887:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.041939:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690559937531584 00010000:00000200:0.0:1616513077.041950:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1690559937531584/t0(0) o400->43f6d86f-a480-4a23-1ce1-24876c221586@10.31.128.86@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.041965:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690559937531584, offset 192 00000400:00000200:0.0:1616513077.041971:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.86@tcp1 00000400:00000200:0.0:1616513077.041978:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.86@tcp1 routed destination 00000400:00000200:0.0:1616513077.041994:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.86@tcp1(10.31.128.86@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.042000:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.042006:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.042009:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.042011:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.042016:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.042021:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513077.042022:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513077.042074:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.042082:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.042084:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513077.042085:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513077.042091:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.042096:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04dd0 00000800:00000200:2.0:1616513077.042100:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.042221:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.042294:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.042303:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.042312:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.86@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.042321:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.86@tcp1 of length 224 into portal 12 MB=0x6018e288c52c0 00000400:00000200:14.0:1616513077.042327:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.86@tcp1 of length 224/224 into md 0xf96450b9 [1] + 65648 00000400:00000200:14.0:1616513077.042334:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.042337:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513077.042343:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.042346:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.042347:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.042351:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513077.042406:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690559937532608 00010000:00000200:18.0:1616513077.042419:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba4380 x1690559937532608/t0(0) o400->b85cea51-6e4a-e0d4-d59d-dd2d8ea9057d@10.31.128.86@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513077.042456:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690559937532608, offset 192 00000400:00000200:18.0:1616513077.042462:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.86@tcp1 00000400:00000200:18.0:1616513077.042470:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.86@tcp1 routed destination 00000400:00000200:18.0:1616513077.042485:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.86@tcp1(10.31.128.86@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513077.042491:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.042497:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.042500:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.042502:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.042506:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.042511:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.042564:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.042571:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.042573:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.042580:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.042584:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccc38 00000800:00000200:20.0:1616513077.042588:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.042591:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.059513:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.059569:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.059578:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513077.059587:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.121@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.059596:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.121@tcp1 of length 224 into portal 26 MB=0x5ffd4b5375380 00000400:00000200:16.0:1616513077.059604:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.121@tcp1 of length 224/224 into md 0xf97c6f69 [1] + 0 00000400:00000200:16.0:1616513077.059609:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7d38 00000400:00000200:16.0:1616513077.059612:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.059615:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513077.059619:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7d38 00000800:00000200:16.0:1616513077.059622:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513077.059623:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.059625:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.059627:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.059643:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663922004864 00010000:00000200:0.0:1616513077.059654:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688663922004864/t0(0) o400->acd89bcf-fb75-efee-1f37-45ccf2203a02@10.31.136.121@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.059668:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663922004864, offset 192 00000400:00000200:0.0:1616513077.059675:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.121@tcp1 00000400:00000200:0.0:1616513077.059683:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.121@tcp1 routed destination 00000400:00000200:0.0:1616513077.059698:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.121@tcp1(10.31.136.121@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.059705:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.059710:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.059713:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.059715:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.059720:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513077.059726:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.059727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.059779:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513077.059785:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513077.059788:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513077.059793:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513077.059798:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb055c8 00000800:00000200:26.0:1616513077.059802:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513077.059804:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.060250:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.060320:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.060330:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.060338:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.121@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.060347:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.121@tcp1 of length 224 into portal 12 MB=0x5ffd4b53754c0 00000400:00000200:20.0:1616513077.060353:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.121@tcp1 of length 224/224 into md 0x865bbec1 [1] + 448 00000400:00000200:20.0:1616513077.060359:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.060361:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513077.060368:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.060371:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513077.060373:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.060378:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513077.060430:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663922005184 00010000:00000200:4.0:1616513077.060455:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01680 x1688663922005184/t0(0) o400->8f739bb0-5615-30e4-2f49-354d1023461c@10.31.136.121@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513077.060469:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663922005184, offset 192 00000400:00000200:4.0:1616513077.060476:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.121@tcp1 00000400:00000200:4.0:1616513077.060483:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.121@tcp1 routed destination 00000400:00000200:4.0:1616513077.060499:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.121@tcp1(10.31.136.121@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513077.060505:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513077.060511:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513077.060513:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513077.060516:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513077.060520:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.060526:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513077.060537:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513077.060542:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513077.060545:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513077.060547:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513077.060552:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513077.060555:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb650 00000800:00000200:12.0:1616513077.060559:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.071843:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513077.071898:0:49127:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513077.071900:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.071910:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.071919:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.127@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.071928:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.127@tcp1 of length 224 into portal 26 MB=0x601fbd179c840 00000400:00000200:28.0:1616513077.071934:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.127@tcp1 of length 224/224 into md 0xf97c6f89 [1] + 0 00000400:00000200:28.0:1616513077.071939:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6908 00000400:00000200:28.0:1616513077.071942:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.071945:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513077.071949:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6908 00000800:00000200:28.0:1616513077.071952:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513077.071956:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513077.071957:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513077.071959:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.071961:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513077.071965:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.071973:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.127@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.071982:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.127@tcp1 of length 224 into portal 12 MB=0x601fbd179c8c0 00000400:00000200:22.0:1616513077.071988:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.127@tcp1 of length 224/224 into md 0xf96450b9 [1] + 65872 00000400:00000200:22.0:1616513077.071992:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.071995:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.072001:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.072004:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.072006:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.072008:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513077.072027:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691030923102272 00010000:00000200:0.0:1616513077.072037:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1691030923102272/t0(0) o400->e1ca7ee2-5079-a933-aa89-81884eca49bf@10.31.143.127@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.072050:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691030923102272, offset 192 00000400:00000200:0.0:1616513077.072057:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.127@tcp1 00000400:00000200:0.0:1616513077.072064:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.127@tcp1 routed destination 00000100:00000200:18.0:1616513077.072069:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691030923102400 00000400:00000200:0.0:1616513077.072078:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.127@tcp1(10.31.143.127@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:18.0:1616513077.072081:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2f080 x1691030923102400/t0(0) o400->d3d80780-1a96-b88b-9860-4383bb67db88@10.31.143.127@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513077.072085:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.072090:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.072093:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:18.0:1616513077.072095:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691030923102400, offset 192 00000800:00000200:0.0:1616513077.072095:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.072099:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513077.072102:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.127@tcp1 00000800:00000200:7.2:1616513077.072105:0:49127:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513077.072106:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513077.072109:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.127@tcp1 routed destination 00000400:00000200:18.0:1616513077.072124:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.127@tcp1(10.31.143.127@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513077.072131:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.072137:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.072139:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.072142:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513077.072144:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.072146:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513077.072148:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513077.072150:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513077.072151:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513077.072155:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513077.072158:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:10.0:1616513077.072161:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513077.072163:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513077.072218:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.072225:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.072227:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.072234:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.072238:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccaa0 00000800:00000200:16.0:1616513077.072242:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.072245:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.104310:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513077.104359:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513077.104367:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.104377:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.104386:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.147@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.104395:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.147@tcp1 of length 224 into portal 26 MB=0x5ffe9a061e440 00000400:00000200:28.0:1616513077.104402:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.147@tcp1 of length 224/224 into md 0xf97c6f99 [1] + 0 00000400:00000200:28.0:1616513077.104406:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6dd0 00000400:00000200:28.0:1616513077.104409:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.104412:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513077.104414:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513077.104416:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6dd0 00000800:00000200:28.0:1616513077.104419:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.104422:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513077.104424:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513077.104424:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:28.0:1616513077.104427:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513077.104446:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.147@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.104454:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.147@tcp1 of length 224 into portal 12 MB=0x5ffe9a061e680 00000400:00000200:22.0:1616513077.104461:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.147@tcp1 of length 224/224 into md 0xf96450b9 [1] + 66096 00000400:00000200:22.0:1616513077.104468:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.104470:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513077.104475:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753766786112 00000800:00000200:22.0:1616513077.104477:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.104480:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.104482:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.104485:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513077.104485:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688753766786112/t0(0) o400->fd5b3cd4-f0a7-06c3-a5e4-0867d9e4c3d6@10.31.141.147@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.104498:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688753766786112, offset 192 00000400:00000200:0.0:1616513077.104503:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.147@tcp1 00000400:00000200:0.0:1616513077.104510:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.147@tcp1 routed destination 00000400:00000200:0.0:1616513077.104522:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.147@tcp1(10.31.141.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.104528:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.104533:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.104536:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.104538:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:18.0:1616513077.104541:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753766786688 00000800:00000200:0.0:1616513077.104542:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.104547:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513077.104548:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:18.0:1616513077.104554:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2bf00 x1688753766786688/t0(0) o400->45775bd3-1f49-bbb9-42dd-eb297751fb0f@10.31.141.147@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:10.0:1616513077.104562:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513077.104567:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:18.0:1616513077.104568:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688753766786688, offset 192 00000400:00000200:10.0:1616513077.104568:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513077.104572:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513077.104575:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.147@tcp1 00000400:00000200:10.0:1616513077.104575:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb046e8 00000400:00000200:18.0:1616513077.104581:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.147@tcp1 routed destination 00000800:00000200:10.0:1616513077.104587:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513077.104588:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:18.0:1616513077.104597:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.147@tcp1(10.31.141.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513077.104605:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.104611:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.104614:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.104616:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.104621:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.104626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.104681:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.104688:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.104691:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.104698:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.104702:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdba0 00000800:00000200:16.0:1616513077.104706:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.104709:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.104719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.104732:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513077.104741:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.104742:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.104751:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.153@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.104761:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.153@tcp1 of length 224 into portal 12 MB=0x6001bc3fc7940 00000400:00000200:20.0:1616513077.104767:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.153@tcp1 of length 224/224 into md 0x865bbec1 [1] + 672 00000400:00000200:20.0:1616513077.104772:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.104775:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513077.104782:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.104785:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.104787:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513077.104853:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969112484160 00010000:00000200:4.0:1616513077.104864:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05a00 x1688969112484160/t0(0) o400->ac23c660-1321-5d64-6cb5-fe3c7fca66f4@10.31.138.153@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513077.104878:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688969112484160, offset 192 00000400:00000200:4.0:1616513077.104884:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.153@tcp1 00000400:00000200:4.0:1616513077.104891:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.153@tcp1 routed destination 00000400:00000200:4.0:1616513077.104905:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.153@tcp1(10.31.138.153@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513077.104911:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:7.2:1616513077.104916:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513077.104916:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513077.104919:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513077.104921:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.104930:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:4.0:1616513077.104931:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.104936:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513077.104939:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:22.0:1616513077.104943:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.153@tcp1 : PUT - for me 00000800:00000200:14.0:1616513077.104945:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:10.0:1616513077.104946:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513077.104947:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513077.104949:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.153@tcp1 of length 224 into portal 26 MB=0x6001bc3fc78c0 00000400:00000200:14.0:1616513077.104954:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513077.104955:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.153@tcp1 of length 224/224 into md 0xf97c6fb9 [1] + 0 00000400:00000200:14.0:1616513077.104958:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb6d8 00000400:00000200:22.0:1616513077.104960:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78660 00000400:00000200:22.0:1616513077.104962:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513077.104962:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513077.104965:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.104969:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78660 00000800:00000200:22.0:1616513077.104971:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.104973:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.104975:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513077.104983:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969112484032 00010000:00000200:0.0:1616513077.104989:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688969112484032/t0(0) o400->195f3336-f575-7031-586d-a5c900dfb339@10.31.138.153@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.105001:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688969112484032, offset 192 00000400:00000200:0.0:1616513077.105006:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.153@tcp1 00000400:00000200:0.0:1616513077.105012:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.153@tcp1 routed destination 00000400:00000200:0.0:1616513077.105025:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.153@tcp1(10.31.138.153@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.105035:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.105040:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.105042:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.105044:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.105047:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513077.105052:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513077.105053:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.105062:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.105066:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.105068:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513077.105070:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513077.105073:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.105077:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05d38 00000800:00000200:16.0:1616513077.105080:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.105882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513077.105937:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513077.105946:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.105947:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.105956:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.130@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.105964:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.130@tcp1 of length 224 into portal 26 MB=0x5ffeb55bcfa40 00000400:00000200:2.0:1616513077.105971:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.130@tcp1 of length 224/224 into md 0xf97c6fc9 [1] + 0 00000400:00000200:2.0:1616513077.105976:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78ee0 00000400:00000200:2.0:1616513077.105979:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.105982:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513077.105986:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78ee0 00000800:00000200:2.0:1616513077.105991:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:7.2:1616513077.105994:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.105994:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.105997:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513077.105997:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688761104398912 00000800:00000200:2.0:1616513077.106000:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513077.106002:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688761104398912/t0(0) o400->f7b204c5-a2a0-1a75-7ef5-0020532eb0e7@10.31.141.130@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513077.106005:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.106006:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.106013:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.130@tcp1 : PUT - for me 00000100:00000200:0.0:1616513077.106013:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688761104398912, offset 192 00000400:00000200:0.0:1616513077.106017:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.130@tcp1 00000400:00000200:2.0:1616513077.106019:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.130@tcp1 of length 224 into portal 12 MB=0x5ffeb55bcfd00 00000400:00000200:0.0:1616513077.106023:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.130@tcp1 routed destination 00000400:00000200:2.0:1616513077.106024:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.130@tcp1 of length 224/224 into md 0xf96450b9 [1] + 66320 00000400:00000200:2.0:1616513077.106028:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.106030:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513077.106036:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.130@tcp1(10.31.141.130@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513077.106040:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.106042:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513077.106042:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513077.106043:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513077.106046:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.106048:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.106050:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.106053:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513077.106057:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.106058:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513077.106064:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513077.106067:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513077.106068:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513077.106071:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513077.106071:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513077.106073:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:10.0:1616513077.106075:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513077.106099:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688761104399616 00010000:00000200:18.0:1616513077.106112:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464620900 x1688761104399616/t0(0) o400->e5c86fdd-03e3-0136-4a84-cde884352732@10.31.141.130@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513077.106126:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688761104399616, offset 192 00000400:00000200:18.0:1616513077.106133:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.130@tcp1 00000400:00000200:18.0:1616513077.106140:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.130@tcp1 routed destination 00000400:00000200:18.0:1616513077.106155:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.130@tcp1(10.31.141.130@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513077.106162:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.106167:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.106170:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.106172:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.106177:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.106243:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.106300:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.106306:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.106309:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.106316:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.106320:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cca18 00000800:00000200:28.0:1616513077.106324:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.106327:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.127103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.127159:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.127168:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.127177:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.19@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.127187:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.19@tcp1 of length 224 into portal 26 MB=0x60183f50cec00 00000400:00000200:22.0:1616513077.127194:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.19@tcp1 of length 224/224 into md 0xf97c7009 [1] + 0 00000400:00000200:22.0:1616513077.127198:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79188 00000400:00000200:22.0:1616513077.127201:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.127204:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.127209:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79188 00000800:00000200:22.0:1616513077.127212:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.127212:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.127214:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.127217:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513077.127274:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516123872256 00010000:00000200:0.0:1616513077.127285:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1690516123872256/t0(0) o400->bc02d223-c595-fc9b-92a5-fb2a7f21e528@10.31.132.19@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.127300:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690516123872256, offset 192 00000400:00000200:0.0:1616513077.127306:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.19@tcp1 00000400:00000200:0.0:1616513077.127314:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.19@tcp1 routed destination 00000400:00000200:0.0:1616513077.127329:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.19@tcp1(10.31.132.19@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.127336:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.127342:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.127344:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.127349:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:7.2:1616513077.127353:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513077.127354:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513077.127360:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513077.127407:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.127416:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513077.127416:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:10.0:1616513077.127421:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:10.0:1616513077.127422:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513077.127424:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.19@tcp1 : PUT - for me 00000800:00000200:2.0:1616513077.127425:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513077.127427:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513077.127430:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04440 00000800:00000200:10.0:1616513077.127433:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513077.127448:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.19@tcp1 of length 224 into portal 12 MB=0x60183f50ced00 00000400:00000200:14.0:1616513077.127454:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.19@tcp1 of length 224/224 into md 0xf96450b9 [1] + 66544 00000400:00000200:14.0:1616513077.127460:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.127462:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513077.127469:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.127472:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.127474:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513077.127530:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516123872512 00010000:00000200:18.0:1616513077.127542:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba6c00 x1690516123872512/t0(0) o400->af228f21-6190-a094-8a85-ab2423e98f6f@10.31.132.19@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513077.127555:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690516123872512, offset 192 00000400:00000200:18.0:1616513077.127562:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.19@tcp1 00000400:00000200:18.0:1616513077.127569:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.19@tcp1 routed destination 00000400:00000200:18.0:1616513077.127583:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.19@tcp1(10.31.132.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513077.127589:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.127594:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.127597:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.127599:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.127603:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.127650:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513077.127706:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513077.127712:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513077.127717:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513077.127724:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513077.127728:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc5d8 00000800:00000200:12.0:1616513077.127732:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513077.127735:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.130340:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513077.130397:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.130407:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.130416:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.83@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.130426:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.83@tcp1 of length 224 into portal 12 MB=0x601979defdf00 00000400:00000200:28.0:1616513077.130453:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.83@tcp1 of length 224/224 into md 0x865bbec1 [1] + 896 00000800:00000200:20.0:1616513077.130456:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513077.130458:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.130461:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513077.130467:0:49133:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513077.130468:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.130471:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.130473:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513077.130489:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690600561696512 00010000:00000200:4.0:1616513077.130501:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6721680 x1690600561696512/t0(0) o400->b4bd54bd-2a26-bb60-1ecf-e8a3354be0cc@10.31.142.83@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513077.130516:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690600561696512, offset 192 00000800:00000200:22.0:1616513077.130522:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513077.130523:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.83@tcp1 00000800:00000200:22.0:1616513077.130531:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:4.0:1616513077.130531:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.83@tcp1 routed destination 00000400:00000200:22.0:1616513077.130540:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.83@tcp1 : PUT - for me 00000400:00000200:4.0:1616513077.130546:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.83@tcp1(10.31.142.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513077.130553:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.130556:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.83@tcp1 of length 224 into portal 26 MB=0x601979defdd00 00000800:00000200:4.0:1616513077.130559:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513077.130562:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513077.130564:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.83@tcp1 of length 224/224 into md 0xf97c7019 [1] + 0 00000800:00000200:4.0:1616513077.130564:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:22.0:1616513077.130569:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78d48 00000800:00000200:4.0:1616513077.130569:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:22.0:1616513077.130572:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.130574:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513077.130575:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:22.0:1616513077.130579:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78d48 00000800:00000200:22.0:1616513077.130582:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513077.130583:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513077.130584:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:10.0:1616513077.130586:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513077.130586:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513077.130588:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513077.130593:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.130597:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb980 00000800:00000200:2.0:1616513077.130601:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513077.130643:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690600561696000 00010000:00000200:0.0:1616513077.130654:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1690600561696000/t0(0) o400->52b4cba4-745e-7451-c158-5d6f8b4045ea@10.31.142.83@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.130668:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690600561696000, offset 192 00000400:00000200:0.0:1616513077.130674:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.83@tcp1 00000400:00000200:0.0:1616513077.130682:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.83@tcp1 routed destination 00000400:00000200:0.0:1616513077.130697:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.83@tcp1(10.31.142.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.130704:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.130710:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.130712:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.130715:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.130719:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.130725:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513077.130726:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513077.130737:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513077.130744:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513077.130746:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513077.130760:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513077.130765:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04908 00000800:00000200:12.0:1616513077.130769:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513077.130772:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.135548:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.135604:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.135613:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513077.135613:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513077.135622:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.237@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.135632:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.237@tcp1 of length 224 into portal 26 MB=0x5ffeac92e13c0 00000400:00000200:14.0:1616513077.135638:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.237@tcp1 of length 224/224 into md 0xf97c7029 [1] + 0 00000400:00000200:14.0:1616513077.135643:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79078 00000400:00000200:14.0:1616513077.135646:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.135648:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513077.135655:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79078 00000800:00000200:14.0:1616513077.135658:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.135660:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.135662:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513077.135730:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688758746223552 00000800:00000200:7.2:1616513077.135738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:0.0:1616513077.135740:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688758746223552/t0(0) o400->191d4072-b844-3e04-6bc9-de65ef7b80f0@10.31.128.237@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.135753:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688758746223552, offset 192 00000400:00000200:0.0:1616513077.135759:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.237@tcp1 00000400:00000200:0.0:1616513077.135767:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.237@tcp1 routed destination 00000400:00000200:0.0:1616513077.135781:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.237@tcp1(10.31.128.237@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.135787:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513077.135791:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.135792:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.135795:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513077.135797:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513077.135800:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.135801:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:0.0:1616513077.135807:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513077.135809:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.237@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.135818:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.237@tcp1 of length 224 into portal 12 MB=0x5ffeac92e1780 00000400:00000200:22.0:1616513077.135824:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.237@tcp1 of length 224/224 into md 0xf96450b9 [1] + 66768 00000400:00000200:22.0:1616513077.135829:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.135832:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.135839:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513077.135841:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513077.135843:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513077.135846:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.135854:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513077.135856:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.135857:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.135864:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.135869:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04cc0 00000800:00000200:2.0:1616513077.135873:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513077.135900:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688758746224512 00010000:00000200:18.0:1616513077.135911:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2a400 x1688758746224512/t0(0) o400->c0c979e5-0103-e26c-8ee4-56eba4bb0b5d@10.31.128.237@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513077.135924:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688758746224512, offset 192 00000400:00000200:18.0:1616513077.135930:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.237@tcp1 00000400:00000200:18.0:1616513077.135937:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.237@tcp1 routed destination 00000400:00000200:18.0:1616513077.135951:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.237@tcp1(10.31.128.237@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513077.135957:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.135962:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.135965:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.135966:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.135971:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.135977:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.136034:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.136041:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.136044:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.136050:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.136054:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccbb0 00000800:00000200:20.0:1616513077.136058:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.136061:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.191719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513077.191768:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513077.191777:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.191789:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.191799:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.64@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.191810:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.64@tcp1 of length 224 into portal 12 MB=0x5ffdfac5f2640 00000400:00000200:14.0:1616513077.191817:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.64@tcp1 of length 224/224 into md 0xf96450b9 [1] + 66992 00000400:00000200:14.0:1616513077.191823:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513077.191824:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:14.0:1616513077.191826:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513077.191834:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513077.191835:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.191838:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.191840:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.191843:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513077.191844:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.64@tcp1 : PUT - for me 00000400:00000200:12.0:1616513077.191854:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.64@tcp1 of length 224 into portal 26 MB=0x5ffdfac5f23c0 00000400:00000200:12.0:1616513077.191861:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.64@tcp1 of length 224/224 into md 0xf97c7039 [1] + 0 00000400:00000200:12.0:1616513077.191866:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b795c8 00000400:00000200:12.0:1616513077.191869:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513077.191872:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:16.0:1616513077.191878:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513077.191878:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b795c8 00000800:00000200:12.0:1616513077.191882:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513077.191885:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513077.191887:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513077.191924:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711018260032 00010000:00000200:18.0:1616513077.191939:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba3a80 x1688711018260032/t0(0) o400->9a1d204c-82d3-ca2a-8341-1175c5670ede@10.31.129.64@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513077.191955:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688711018260032, offset 192 00000100:00000200:0.0:1616513077.191955:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711018259392 00000400:00000200:18.0:1616513077.191961:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.64@tcp1 00010000:00000200:0.0:1616513077.191968:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688711018259392/t0(0) o400->6823b45f-d09d-aea4-400e-8d5236faf74e@10.31.129.64@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513077.191969:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.64@tcp1 routed destination 00000100:00000200:0.0:1616513077.191983:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688711018259392, offset 192 00000400:00000200:18.0:1616513077.191984:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.64@tcp1(10.31.129.64@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513077.191989:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.64@tcp1 00000800:00000200:18.0:1616513077.191991:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513077.191996:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.64@tcp1 routed destination 00000800:00000200:18.0:1616513077.191997:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.192000:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513077.192003:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.192009:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513077.192010:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.64@tcp1(10.31.129.64@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.192016:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.192022:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.192025:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513077.192026:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.192027:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.192032:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.192037:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513077.192039:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513077.192075:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513077.192083:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513077.192085:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513077.192090:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513077.192093:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccdd0 00000800:00000200:10.0:1616513077.192096:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513077.192098:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513077.192099:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.192106:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.192109:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513077.192115:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513077.192116:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.192122:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:20.0:1616513077.192125:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.199579:0:49141:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.199646:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.199656:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.199665:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.41@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.199674:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.41@tcp1 of length 224 into portal 26 MB=0x602be022575c0 00000400:00000200:14.0:1616513077.199681:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.41@tcp1 of length 224/224 into md 0xf97c7059 [1] + 0 00000400:00000200:14.0:1616513077.199686:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78dd0 00000400:00000200:14.0:1616513077.199688:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.199691:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:11.2:1616513077.199695:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513077.199696:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78dd0 00000800:00000200:14.0:1616513077.199699:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.199702:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.199703:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.199706:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513077.199773:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.199783:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513077.199789:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691864963315136 00000400:00000200:16.0:1616513077.199792:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.41@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.199800:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.41@tcp1 of length 224 into portal 12 MB=0x602be02257740 00010000:00000200:0.0:1616513077.199800:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1691864963315136/t0(0) o400->4b14735a-d702-1937-9da7-b494f3058d8c@10.31.136.41@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513077.199808:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.41@tcp1 of length 224/224 into md 0x865bbec1 [1] + 1120 00000400:00000200:16.0:1616513077.199814:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513077.199815:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691864963315136, offset 192 00000100:00000200:16.0:1616513077.199816:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513077.199823:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513077.199825:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.41@tcp1 00000800:00000200:16.0:1616513077.199826:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513077.199827:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.199831:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513077.199833:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.41@tcp1 routed destination 00000400:00000200:0.0:1616513077.199848:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.41@tcp1(10.31.136.41@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.199855:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.199860:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.199863:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.199866:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.199871:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513077.199877:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.199878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513077.199888:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691864963315520 00010000:00000200:4.0:1616513077.199900:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd4800 x1691864963315520/t0(0) o400->b1ca1661-dc10-764f-4080-f2575db818c1@10.31.136.41@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513077.199915:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691864963315520, offset 192 00000400:00000200:4.0:1616513077.199921:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.41@tcp1 00000400:00000200:4.0:1616513077.199929:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.41@tcp1 routed destination 00000800:00000200:10.0:1616513077.199931:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513077.199938:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513077.199941:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513077.199943:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.41@tcp1(10.31.136.41@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513077.199947:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:4.0:1616513077.199950:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:10.0:1616513077.199953:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:4.0:1616513077.199956:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513077.199958:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513077.199961:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513077.199965:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513077.199966:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513077.199968:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.199970:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.200024:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.200031:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.200034:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.200040:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.200044:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba5d8 00000800:00000200:28.0:1616513077.200048:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.200051:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.241926:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.241983:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.241994:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.242003:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.192@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.242014:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.192@tcp1 of length 224 into portal 26 MB=0x605a0746130c0 00000400:00000200:14.0:1616513077.242021:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.192@tcp1 of length 224/224 into md 0xf97c7079 [1] + 0 00000400:00000200:14.0:1616513077.242028:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78e58 00000400:00000200:14.0:1616513077.242032:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.242035:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513077.242039:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513077.242040:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78e58 00000800:00000200:14.0:1616513077.242044:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513077.242047:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513077.242049:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.242051:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513077.242052:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513077.242059:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513077.242060:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.192@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.242069:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.192@tcp1 of length 224 into portal 12 MB=0x605a074613240 00000100:00000200:0.0:1616513077.242070:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036565696704 00000800:00000200:12.0:1616513077.242072:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513077.242076:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.192@tcp1 of length 224/224 into md 0xf96450b9 [1] + 67216 00000800:00000200:12.0:1616513077.242081:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513077.242082:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:0.0:1616513077.242083:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1695036565696704/t0(0) o400->66e841a6-f2e3-e96b-8301-9b9589fa3290@10.31.129.192@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513077.242084:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513077.242084:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513077.242089:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.68@tcp1 : PUT - for me 00000800:00000200:22.0:1616513077.242093:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.242096:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.242098:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513077.242098:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.68@tcp1 of length 224 into portal 26 MB=0x5ffdbdbc00bc0 00000100:00000200:0.0:1616513077.242098:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036565696704, offset 192 00000400:00000200:12.0:1616513077.242105:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.68@tcp1 of length 224/224 into md 0xf97c7089 [1] + 0 00000400:00000200:0.0:1616513077.242106:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.192@tcp1 00000400:00000200:12.0:1616513077.242109:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b797e8 00000400:00000200:12.0:1616513077.242112:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:0.0:1616513077.242113:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.192@tcp1 routed destination 00000100:00000200:12.0:1616513077.242114:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:18.0:1616513077.242120:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036565697088 00000400:00000200:12.0:1616513077.242120:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b797e8 00000800:00000200:12.0:1616513077.242123:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513077.242125:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513077.242127:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513077.242131:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.192@tcp1(10.31.129.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:18.0:1616513077.242134:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a29680 x1695036565697088/t0(0) o400->cbfd5633-79d2-70ac-f94d-fe38bed42bb5@10.31.129.192@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513077.242138:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.242144:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.242147:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:18.0:1616513077.242149:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036565697088, offset 192 00000800:00000200:0.0:1616513077.242149:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.242150:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694633270208 00000400:00000200:18.0:1616513077.242155:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.192@tcp1 00000800:00000200:0.0:1616513077.242155:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.242160:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513077.242163:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.192@tcp1 routed destination 00000100:00000200:0.0:1616513077.242164:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:15.0:1616513077.242166:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688694633270208/t0(0) o400->0e40655a-6265-2e67-eea5-93550d7afd9f@10.31.141.68@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513077.242175:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513077.242179:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.192@tcp1(10.31.129.192@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:15.0:1616513077.242184:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694633270208, offset 192 00000800:00000200:18.0:1616513077.242186:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:15.0:1616513077.242190:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.68@tcp1 00000800:00000200:18.0:1616513077.242192:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.242195:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.242198:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:15.0:1616513077.242199:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.68@tcp1 routed destination 00000800:00000200:18.0:1616513077.242203:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:15.0:1616513077.242216:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.68@tcp1(10.31.141.68@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.242223:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.242231:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513077.242233:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513077.242233:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513077.242234:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513077.242236:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:28.0:1616513077.242243:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.242244:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:14.0:1616513077.242244:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (25)-- 00000800:00000200:2.0:1616513077.242245:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:7.2:1616513077.242248:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513077.242248:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.242251:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.68@tcp1 : PUT - for me 00000100:00000200:15.0:1616513077.242252:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513077.242255:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.242260:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.68@tcp1 of length 224 into portal 12 MB=0x5ffdbdbc00cc0 00000400:00000200:2.0:1616513077.242261:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04aa0 00000800:00000200:10.0:1616513077.242262:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:28.0:1616513077.242267:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.68@tcp1 of length 224/224 into md 0x865bbec1 [1] + 1344 00000800:00000200:2.0:1616513077.242268:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:10.0:1616513077.242269:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513077.242271:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513077.242272:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:10.0:1616513077.242272:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.242274:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:10.0:1616513077.242279:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513077.242284:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:10.0:1616513077.242285:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6330 00000800:00000200:28.0:1616513077.242287:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513077.242289:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:10.0:1616513077.242289:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513077.242296:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513077.242298:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513077.242300:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513077.242300:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513077.242301:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694633270464 00000400:00000200:20.0:1616513077.242307:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.242311:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdcb0 00000800:00000200:20.0:1616513077.242315:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513077.242315:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6726780 x1688694633270464/t0(0) o400->52153b4f-769f-32cb-f1b5-1a3198716f03@10.31.141.68@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513077.242325:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:8.0:1616513077.242326:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694633270464, offset 192 00000400:00000200:8.0:1616513077.242331:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.68@tcp1 00000800:00000200:12.0:1616513077.242332:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513077.242337:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513077.242337:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.68@tcp1 routed destination 00000800:00000200:16.0:1616513077.242339:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513077.242349:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.68@tcp1(10.31.141.68@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513077.242354:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513077.242357:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.166@tcp1 : PUT - for me 00000800:00000200:8.0:1616513077.242359:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513077.242361:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513077.242363:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513077.242365:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.166@tcp1 of length 224 into portal 12 MB=0x5ffdddf9b6b80 00000800:00000200:8.0:1616513077.242367:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513077.242370:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.166@tcp1 of length 224/224 into md 0x865bbec1 [1] + 1568 00000800:00000200:11.2:1616513077.242372:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513077.242374:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513077.242376:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513077.242381:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513077.242383:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:12.0:1616513077.242383:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513077.242387:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513077.242387:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513077.242389:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513077.242389:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703287913344 00000400:00000200:28.0:1616513077.242390:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:8.0:1616513077.242394:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c665a00 x1688703287913344/t0(0) o400->e67f01be-6f87-0321-28c3-27fbf72125da@10.31.136.166@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513077.242395:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.242399:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f717e8 00000800:00000200:28.0:1616513077.242402:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513077.242402:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703287913344, offset 192 00000400:00000200:8.0:1616513077.242407:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.166@tcp1 00000400:00000200:8.0:1616513077.242412:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.166@tcp1 routed destination 00000400:00000200:8.0:1616513077.242422:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.166@tcp1(10.31.136.166@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513077.242426:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513077.242430:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513077.242431:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513077.242433:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513077.242451:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.242454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.242463:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.242467:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513077.242469:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513077.242471:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513077.242475:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.242478:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71650 00000800:00000200:2.0:1616513077.242478:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513077.242481:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.242483:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513077.242486:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.166@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.242495:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.166@tcp1 of length 224 into portal 26 MB=0x5ffdddf9b6b00 00000400:00000200:2.0:1616513077.242502:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.166@tcp1 of length 224/224 into md 0xf97c70a9 [1] + 0 00000400:00000200:2.0:1616513077.242506:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78770 00000400:00000200:2.0:1616513077.242509:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.242512:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513077.242516:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78770 00000800:00000200:2.0:1616513077.242519:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.242522:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.242523:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.242531:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703287913216 00010000:00000200:15.0:1616513077.242537:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688703287913216/t0(0) o400->1f2a1932-e235-d1c4-01a1-27c4b1083114@10.31.136.166@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.242550:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703287913216, offset 192 00000400:00000200:15.0:1616513077.242555:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.166@tcp1 00000400:00000200:15.0:1616513077.242563:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.166@tcp1 routed destination 00000400:00000200:15.0:1616513077.242576:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.166@tcp1(10.31.136.166@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.242582:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.242587:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.242590:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.242591:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.242595:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.242599:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.242600:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513077.242609:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.242614:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.242617:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513077.242617:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513077.242624:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.242630:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7ed0 00000800:00000200:16.0:1616513077.242634:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.252104:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.252160:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.252169:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.252179:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.139@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.252191:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.139@tcp1 of length 224 into portal 26 MB=0x60393c7865ac0 00000400:00000200:20.0:1616513077.252198:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.139@tcp1 of length 224/224 into md 0xf97c70c9 [1] + 0 00000400:00000200:20.0:1616513077.252203:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b787f8 00000400:00000200:20.0:1616513077.252205:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.252208:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513077.252213:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513077.252213:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b787f8 00000800:00000200:20.0:1616513077.252216:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513077.252219:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513077.252221:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513077.252224:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513077.252224:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513077.252233:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.139@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.252242:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.139@tcp1 of length 224 into portal 12 MB=0x60393c7865cc0 00000400:00000200:28.0:1616513077.252248:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.139@tcp1 of length 224/224 into md 0x865bbec1 [1] + 1792 00000400:00000200:28.0:1616513077.252254:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.252256:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513077.252264:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.252269:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.252271:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513077.252279:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783102811328 00010000:00000200:16.0:1616513077.252290:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6727500 x1692783102811328/t0(0) o400->de9a2935-ff2e-45bc-ccc4-2776583df8e2@10.31.140.139@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513077.252291:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783102810816 00010000:00000200:15.0:1616513077.252301:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1692783102810816/t0(0) o400->db68f1dd-49f3-e889-6fc1-33316f964506@10.31.140.139@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513077.252303:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783102811328, offset 192 00000400:00000200:16.0:1616513077.252310:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.139@tcp1 00000100:00000200:15.0:1616513077.252315:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783102810816, offset 192 00000400:00000200:16.0:1616513077.252318:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.139@tcp1 routed destination 00000400:00000200:15.0:1616513077.252321:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.139@tcp1 00000400:00000200:15.0:1616513077.252329:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.139@tcp1 routed destination 00000400:00000200:16.0:1616513077.252334:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.139@tcp1(10.31.140.139@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513077.252341:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:15.0:1616513077.252344:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.139@tcp1(10.31.140.139@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.252351:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513077.252352:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513077.252355:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513077.252357:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.252357:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.252360:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.252361:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:15.0:1616513077.252362:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.252367:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513077.252368:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513077.252371:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.252373:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513077.252380:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.252385:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.252387:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513077.252390:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513077.252392:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.252397:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6dd0 00000800:00000200:16.0:1616513077.252400:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513077.252423:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513077.252430:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513077.252432:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513077.252454:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513077.252458:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ee0 00000800:00000200:10.0:1616513077.252462:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513077.252465:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.277523:0:49150:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.277582:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.277592:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.277601:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.58@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.277611:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.58@tcp1 of length 224 into portal 26 MB=0x5ffe33711db00 00000400:00000200:14.0:1616513077.277618:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.58@tcp1 of length 224/224 into md 0xf97c70d9 [1] + 0 00000400:00000200:14.0:1616513077.277622:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78198 00000400:00000200:14.0:1616513077.277625:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.277629:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513077.277634:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78198 00000800:00000200:2.0:1616513077.277636:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.277637:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.277640:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.277642:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.277702:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726230129408 00010000:00000200:15.0:1616513077.277714:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688726230129408/t0(0) o400->6f025779-7e0c-829c-a870-7a2802db8565@10.31.129.58@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513077.277721:0:49150:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513077.277728:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726230129408, offset 192 00000800:00000200:10.0:1616513077.277732:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:15.0:1616513077.277734:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.58@tcp1 00000800:00000200:10.0:1616513077.277740:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:15.0:1616513077.277743:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.58@tcp1 routed destination 00000400:00000200:10.0:1616513077.277748:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.58@tcp1 : PUT - for me 00000400:00000200:15.0:1616513077.277760:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.58@tcp1(10.31.129.58@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.277767:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:10.0:1616513077.277770:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.58@tcp1 of length 224 into portal 12 MB=0x5ffe33711df40 00000800:00000200:15.0:1616513077.277774:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:10.0:1616513077.277776:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.58@tcp1 of length 224/224 into md 0xf96450b9 [1] + 67440 00000800:00000200:15.0:1616513077.277777:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513077.277779:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:10.0:1616513077.277781:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513077.277783:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:15.0:1616513077.277785:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513077.277786:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:10.0:1616513077.277790:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:7.2:1616513077.277791:0:49150:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:15.0:1616513077.277793:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513077.277794:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:10.0:1616513077.277796:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513077.277798:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.277802:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513077.277803:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513077.277805:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513077.277810:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513077.277814:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7d38 00000800:00000200:10.0:1616513077.277817:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513077.277855:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726230130496 00010000:00000200:18.0:1616513077.277868:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdb180 x1688726230130496/t0(0) o400->5ab3dee9-5948-c363-ce0d-ce3816bd7bb3@10.31.129.58@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513077.277883:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726230130496, offset 192 00000400:00000200:18.0:1616513077.277889:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.58@tcp1 00000400:00000200:18.0:1616513077.277897:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.58@tcp1 routed destination 00000400:00000200:18.0:1616513077.277913:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.58@tcp1(10.31.129.58@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513077.277919:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.277925:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.277928:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.277930:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.277935:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.277942:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.277996:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.278003:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.278006:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.278012:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.278016:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cda90 00000800:00000200:20.0:1616513077.278019:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.278022:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.313021:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.313079:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.313091:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.313101:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.146@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.313110:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.146@tcp1 of length 224 into portal 26 MB=0x603d126ba0fc0 00000400:00000200:22.0:1616513077.313117:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.146@tcp1 of length 224/224 into md 0xf97c7109 [1] + 0 00000400:00000200:22.0:1616513077.313122:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78880 00000400:00000200:22.0:1616513077.313125:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.313128:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.313133:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78880 00000800:00000200:2.0:1616513077.313136:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.313137:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.313140:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.313143:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.313146:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.313147:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.313156:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.146@tcp1 : PUT - for me 00000100:00000200:15.0:1616513077.313159:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693046693040064 00000400:00000200:2.0:1616513077.313165:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.146@tcp1 of length 224 into portal 12 MB=0x603d126ba1080 00010000:00000200:15.0:1616513077.313171:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1693046693040064/t0(0) o400->2ae12e89-f988-ee60-1e48-e11d8b0de4bd@10.242.104.146@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513077.313171:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.146@tcp1 of length 224/224 into md 0xf96450b9 [1] + 67664 00000400:00000200:2.0:1616513077.313176:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.313179:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513077.313186:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693046693040064, offset 192 00000800:00000200:2.0:1616513077.313187:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.313190:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.313192:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513077.313193:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.146@tcp1 00000400:00000200:15.0:1616513077.313201:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.146@tcp1 routed destination 00000400:00000200:15.0:1616513077.313217:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.146@tcp1(10.242.104.146@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.313224:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.313231:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.313234:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.313236:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.313242:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.313246:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.313249:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513077.313264:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693046693040256 00010000:00000200:18.0:1616513077.313277:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464625a00 x1693046693040256/t0(0) o400->ed4952b1-a035-dcc5-30a4-03c672a4cf1c@10.242.104.146@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513077.313292:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693046693040256, offset 192 00000800:00000200:10.0:1616513077.313297:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513077.313298:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.146@tcp1 00000800:00000200:10.0:1616513077.313303:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513077.313306:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.146@tcp1 routed destination 00000400:00000200:10.0:1616513077.313306:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513077.313307:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513077.313312:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513077.313318:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7100 00000400:00000200:18.0:1616513077.313321:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.146@tcp1(10.242.104.146@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513077.313321:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513077.313328:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.313334:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.313336:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.313340:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.313346:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.313352:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.313364:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.313370:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.313373:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.313379:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.313382:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc440 00000800:00000200:20.0:1616513077.313386:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.313389:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.320349:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.320404:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.320413:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.320422:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.154@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.320431:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.154@tcp1 of length 224 into portal 26 MB=0x5ffdd3080e7c0 00000400:00000200:14.0:1616513077.320452:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.154@tcp1 of length 224/224 into md 0xf97c7129 [1] + 0 00000400:00000200:14.0:1616513077.320457:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79870 00000800:00000200:2.0:1616513077.320458:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513077.320461:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.320464:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513077.320469:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79870 00000800:00000200:2.0:1616513077.320469:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513077.320472:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513077.320475:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513077.320477:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.320478:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.154@tcp1 : PUT - for me 00000800:00000200:14.0:1616513077.320480:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513077.320486:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.154@tcp1 of length 224 into portal 12 MB=0x5ffdd3080e840 00000400:00000200:2.0:1616513077.320493:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.154@tcp1 of length 224/224 into md 0xf96450b9 [1] + 67888 00000400:00000200:2.0:1616513077.320498:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.320501:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513077.320507:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.320513:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.320515:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.320523:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700350162880 00010000:00000200:15.0:1616513077.320534:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688700350162880/t0(0) o400->6c773198-3fed-e441-6043-1d7691807034@10.31.141.154@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.320548:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700350162880, offset 192 00000400:00000200:15.0:1616513077.320555:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.154@tcp1 00000400:00000200:15.0:1616513077.320563:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.154@tcp1 routed destination 00000100:00000200:18.0:1616513077.320571:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700350163008 00000400:00000200:15.0:1616513077.320579:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.154@tcp1(10.31.141.154@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:18.0:1616513077.320583:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464627500 x1688700350163008/t0(0) o400->26225ad4-39ff-9374-0e81-b3e9692f3453@10.31.141.154@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.320586:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.320592:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:18.0:1616513077.320598:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700350163008, offset 192 00000800:00000200:15.0:1616513077.320598:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.320600:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513077.320604:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.154@tcp1 00000800:00000200:15.0:1616513077.320605:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.320610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513077.320611:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.154@tcp1 routed destination 00000100:00000200:15.0:1616513077.320613:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513077.320621:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513077.320627:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.154@tcp1(10.31.141.154@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513077.320628:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513077.320631:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513077.320632:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.320634:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:10.0:1616513077.320638:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513077.320640:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513077.320642:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:10.0:1616513077.320644:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c73a8 00000800:00000200:18.0:1616513077.320645:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513077.320647:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513077.320650:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.320656:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.320665:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.320668:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.320670:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.320674:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.320677:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd320 00000800:00000200:20.0:1616513077.320680:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.320682:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.320691:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.320699:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.320704:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513077.320708:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.320710:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.320712:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.320713:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.374784:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.374840:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.374850:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.374859:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.161@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.374869:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.161@tcp1 of length 224 into portal 12 MB=0x5ffd50557dd00 00000400:00000200:22.0:1616513077.374876:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.161@tcp1 of length 224/224 into md 0xf96450b9 [1] + 68112 00000400:00000200:22.0:1616513077.374881:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.374884:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.374891:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.374894:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513077.374895:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.374899:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.374941:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513077.374956:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665266314496 00010000:00000200:18.0:1616513077.374968:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a29f80 x1688665266314496/t0(0) o400->c22d82b6-27c4-7579-8d53-4a4a5afb9660@10.31.132.161@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513077.374983:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665266314496, offset 192 00000400:00000200:18.0:1616513077.374990:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.161@tcp1 00000800:00000200:20.0:1616513077.374996:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513077.374998:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.161@tcp1 routed destination 00000800:00000200:20.0:1616513077.375005:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.375013:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.161@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.375013:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.161@tcp1(10.31.132.161@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513077.375019:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513077.375021:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.161@tcp1 of length 224 into portal 26 MB=0x5ffd50557dc80 00000800:00000200:18.0:1616513077.375026:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513077.375027:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.161@tcp1 of length 224/224 into md 0xf97c7149 [1] + 0 00000800:00000200:18.0:1616513077.375029:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513077.375031:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79ba0 00000800:00000200:18.0:1616513077.375031:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513077.375034:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.375036:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513077.375036:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.375041:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513077.375042:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79ba0 00000800:00000200:20.0:1616513077.375044:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.375047:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.375049:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.375051:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513077.375096:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.375105:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:15.0:1616513077.375108:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665266314368 00000400:00000200:14.0:1616513077.375108:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513077.375109:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513077.375114:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.375118:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd100 00010000:00000200:15.0:1616513077.375120:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688665266314368/t0(0) o400->239c28cd-ca5f-57b0-5cae-1559efb2a0be@10.31.132.161@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513077.375129:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.375134:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665266314368, offset 192 00000400:00000200:15.0:1616513077.375141:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.161@tcp1 00000400:00000200:15.0:1616513077.375150:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.161@tcp1 routed destination 00000400:00000200:15.0:1616513077.375166:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.161@tcp1(10.31.132.161@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.375173:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.375179:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.375182:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.375184:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.375190:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.375195:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.375197:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513077.375249:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.375257:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.375260:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.375267:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.375273:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c74b8 00000800:00000200:28.0:1616513077.375277:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.375280:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.376411:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.376499:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.376508:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.376517:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.84@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.376527:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.84@tcp1 of length 224 into portal 26 MB=0x5f90afab2ed00 00000400:00000200:20.0:1616513077.376533:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.84@tcp1 of length 224/224 into md 0xf97c7159 [1] + 0 00000400:00000200:20.0:1616513077.376538:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78220 00000400:00000200:20.0:1616513077.376541:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.376543:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513077.376548:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78220 00000800:00000200:24.0:1616513077.376552:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.376552:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.376554:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.376559:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.376562:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.376563:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513077.376571:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.84@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.376581:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.84@tcp1 of length 224 into portal 12 MB=0x5f90afab2ed80 00000400:00000200:24.0:1616513077.376587:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.84@tcp1 of length 224/224 into md 0x865bbec1 [1] + 2016 00000400:00000200:24.0:1616513077.376593:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.376595:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513077.376603:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.376606:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.376608:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.376611:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681200434572544 00010000:00000200:15.0:1616513077.376622:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1681200434572544/t0(0) o400->375e34b7-4338-6bab-8fdb-69f7769d167d@10.31.131.84@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.376637:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681200434572544, offset 192 00000400:00000200:15.0:1616513077.376643:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.84@tcp1 00000400:00000200:15.0:1616513077.376652:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.84@tcp1 routed destination 00000400:00000200:15.0:1616513077.376667:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.84@tcp1(10.31.131.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513077.376669:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681200434572672 00000800:00000200:15.0:1616513077.376673:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.376680:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:16.0:1616513077.376682:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc1b00 x1681200434572672/t0(0) o400->830205a1-9caa-01b4-8baa-e82753d73811@10.31.131.84@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.376683:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.376685:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.376690:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.376695:0:49161:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513077.376697:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681200434572672, offset 192 00000100:00000200:15.0:1616513077.376697:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513077.376704:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.84@tcp1 00000400:00000200:16.0:1616513077.376712:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.84@tcp1 routed destination 00000400:00000200:16.0:1616513077.376727:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.84@tcp1(10.31.131.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513077.376734:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513077.376740:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513077.376742:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.376745:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513077.376749:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513077.376750:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.376755:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513077.376756:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.376760:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.376766:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513077.376768:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.376772:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513077.376773:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7e48 00000400:00000200:16.0:1616513077.376774:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513077.376777:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513077.376777:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513077.376780:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513077.376780:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513077.376783:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42dd0 00000800:00000200:16.0:1616513077.376786:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.389957:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513077.390012:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.390021:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.390030:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.50@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.390039:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.50@tcp1 of length 224 into portal 26 MB=0x6022b34926940 00000400:00000200:28.0:1616513077.390046:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.50@tcp1 of length 224/224 into md 0xf97c7169 [1] + 0 00000400:00000200:28.0:1616513077.390051:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b798f8 00000400:00000200:28.0:1616513077.390054:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.390057:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513077.390062:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b798f8 00000800:00000200:28.0:1616513077.390065:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.390065:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.390068:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513077.390069:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.390130:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691234449123648 00010000:00000200:15.0:1616513077.390141:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1691234449123648/t0(0) o400->1e8937dc-6c5b-c3f8-309f-4e7a8ed80a11@10.31.141.50@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.390155:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691234449123648, offset 192 00000400:00000200:15.0:1616513077.390162:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.50@tcp1 00000400:00000200:15.0:1616513077.390170:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.50@tcp1 routed destination 00000400:00000200:15.0:1616513077.390185:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.50@tcp1(10.31.141.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.390192:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.390198:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.390201:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.390203:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.390208:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513077.390214:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.390214:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.390269:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.390276:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513077.390280:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513077.390286:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513077.390295:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6f68 00000800:00000200:18.0:1616513077.390299:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.390302:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.390374:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.390430:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.390452:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.390460:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.50@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.390471:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.50@tcp1 of length 224 into portal 12 MB=0x6022b34926b80 00000400:00000200:20.0:1616513077.390477:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.50@tcp1 of length 224/224 into md 0x865bbec1 [1] + 2240 00000400:00000200:20.0:1616513077.390483:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.390485:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513077.390485:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.390494:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.390497:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.390499:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513077.390516:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691234449124224 00010000:00000200:24.0:1616513077.390528:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05e80 x1691234449124224/t0(0) o400->6ca5da16-d378-57d1-87af-5d020e3bbc5f@10.31.141.50@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513077.390542:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691234449124224, offset 192 00000400:00000200:24.0:1616513077.390549:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.50@tcp1 00000400:00000200:24.0:1616513077.390556:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.50@tcp1 routed destination 00000400:00000200:24.0:1616513077.390571:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.50@tcp1(10.31.141.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513077.390578:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513077.390584:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513077.390586:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513077.390590:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.390595:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.390600:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.390612:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.390615:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513077.390617:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513077.390620:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513077.390623:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513077.390626:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2990 00000800:00000200:24.0:1616513077.390629:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.394524:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513077.394578:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.394587:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.394596:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.20@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.394606:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.20@tcp1 of length 224 into portal 26 MB=0x5f93bd259ec00 00000400:00000200:2.0:1616513077.394613:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.20@tcp1 of length 224/224 into md 0xf97c7189 [1] + 0 00000400:00000200:2.0:1616513077.394618:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79298 00000400:00000200:2.0:1616513077.394621:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.394624:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513077.394628:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79298 00000800:00000200:22.0:1616513077.394632:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.394632:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.394634:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513077.394639:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.394642:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513077.394642:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513077.394651:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.20@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.394661:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.20@tcp1 of length 224 into portal 12 MB=0x5f93bd259ec80 00000400:00000200:22.0:1616513077.394667:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.20@tcp1 of length 224/224 into md 0xf96450b9 [1] + 68336 00000400:00000200:22.0:1616513077.394673:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.394675:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.394683:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.394686:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.394690:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.394693:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681410211048448 00010000:00000200:15.0:1616513077.394704:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1681410211048448/t0(0) o400->df323de8-2364-e4a6-cf51-a9db4177b645@10.31.132.20@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.394718:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681410211048448, offset 192 00000400:00000200:15.0:1616513077.394724:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.20@tcp1 00000400:00000200:15.0:1616513077.394733:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.20@tcp1 routed destination 00000100:00000200:26.0:1616513077.394749:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681410211048576 00000400:00000200:15.0:1616513077.394749:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.20@tcp1(10.31.132.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.394755:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.394762:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:26.0:1616513077.394763:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2c800 x1681410211048576/t0(0) o400->c661d6d2-5bd1-fd0a-dbd2-7aca331b514d@10.31.132.20@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.394765:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.394767:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.394772:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:26.0:1616513077.394777:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681410211048576, offset 192 00000800:00000200:7.2:1616513077.394777:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.394779:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513077.394784:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.20@tcp1 00000400:00000200:26.0:1616513077.394792:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.20@tcp1 routed destination 00000400:00000200:26.0:1616513077.394808:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.20@tcp1(10.31.132.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.394814:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.394820:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.394823:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.394825:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.394830:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513077.394831:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.394838:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513077.394842:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513077.394842:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513077.394848:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.394854:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6c38 00000800:00000200:14.0:1616513077.394858:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.394876:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.394931:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.394937:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.394940:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.394946:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.394950:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:16.0:1616513077.394953:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.394956:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.396566:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513077.396619:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.396627:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.396635:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.214@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.396644:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.214@tcp1 of length 224 into portal 26 MB=0x6026e66c70180 00000400:00000200:18.0:1616513077.396650:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.214@tcp1 of length 224/224 into md 0xf97c7199 [1] + 0 00000400:00000200:18.0:1616513077.396655:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78aa0 00000400:00000200:18.0:1616513077.396657:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.396660:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513077.396664:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78aa0 00000800:00000200:18.0:1616513077.396667:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.396669:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.396671:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.396678:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513077.396681:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.396682:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.396691:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.214@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.396700:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.214@tcp1 of length 224 into portal 12 MB=0x6026e66c70200 00000400:00000200:22.0:1616513077.396706:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.214@tcp1 of length 224/224 into md 0xf96450b9 [1] + 68560 00000400:00000200:22.0:1616513077.396711:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.396714:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.396720:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.396723:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.396725:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.396739:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691523054240128 00010000:00000200:15.0:1616513077.396749:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1691523054240128/t0(0) o400->0e289489-261f-8ecc-e12d-dff858342ac4@10.31.142.214@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.396762:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691523054240128, offset 192 00000400:00000200:15.0:1616513077.396769:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.214@tcp1 00000400:00000200:15.0:1616513077.396777:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.214@tcp1 routed destination 00000100:00000200:26.0:1616513077.396783:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691523054240256 00000400:00000200:15.0:1616513077.396791:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.214@tcp1(10.31.142.214@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:26.0:1616513077.396795:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a28000 x1691523054240256/t0(0) o400->1b3937ce-405d-e9ae-6a26-7a253744a752@10.31.142.214@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.396797:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.396803:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.396806:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.396808:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:26.0:1616513077.396809:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691523054240256, offset 192 00000800:00000200:15.0:1616513077.396812:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513077.396816:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.214@tcp1 00000800:00000200:7.2:1616513077.396817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.396819:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513077.396824:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.214@tcp1 routed destination 00000400:00000200:26.0:1616513077.396839:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.214@tcp1(10.31.142.214@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.396851:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.396857:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.396860:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.396862:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.396867:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513077.396870:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.396878:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513077.396880:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513077.396881:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513077.396887:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.396893:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c77e8 00000800:00000200:14.0:1616513077.396897:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.396913:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.396969:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.396975:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.396978:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.396985:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.396989:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aedd0 00000800:00000200:28.0:1616513077.396993:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.396996:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.400538:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.400597:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.400638:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513077.400646:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.106@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.400655:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.106@tcp1 of length 224 into portal 26 MB=0x6002099eea380 00000400:00000200:16.0:1616513077.400661:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.106@tcp1 of length 224/224 into md 0xf97c71b9 [1] + 0 00000400:00000200:16.0:1616513077.400666:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78550 00000400:00000200:16.0:1616513077.400671:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.400674:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513077.400678:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78550 00000800:00000200:20.0:1616513077.400679:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.400681:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.400684:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.400685:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.400741:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688989881770880 00010000:00000200:15.0:1616513077.400752:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688989881770880/t0(0) o400->6cbaa7d4-30a5-f20d-5984-2bd79db47b25@10.31.142.106@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.400765:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688989881770880, offset 192 00000400:00000200:15.0:1616513077.400771:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.106@tcp1 00000400:00000200:15.0:1616513077.400778:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.106@tcp1 routed destination 00000400:00000200:15.0:1616513077.400792:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.106@tcp1(10.31.142.106@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.400798:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.400804:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.400807:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.400809:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.400814:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513077.400820:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.400820:0:49163:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.400875:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.400882:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.400885:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.400892:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.400898:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7210 00000800:00000200:2.0:1616513077.400902:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.400905:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.401109:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513077.401162:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.401171:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.401180:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.106@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.401188:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.106@tcp1 of length 224 into portal 12 MB=0x6002099eea800 00000400:00000200:28.0:1616513077.401195:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.106@tcp1 of length 224/224 into md 0x865bbec1 [1] + 2464 00000400:00000200:28.0:1616513077.401202:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.401205:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513077.401217:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.401219:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.401222:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.401224:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.401240:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688989881772032 00010000:00000200:0.0:1616513077.401252:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6726300 x1688989881772032/t0(0) o400->17d075a1-4ea0-b2e8-05d6-5bedce035b86@10.31.142.106@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.401266:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688989881772032, offset 192 00000400:00000200:0.0:1616513077.401272:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.106@tcp1 00000400:00000200:0.0:1616513077.401279:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.106@tcp1 routed destination 00000400:00000200:0.0:1616513077.401294:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.106@tcp1(10.31.142.106@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.401300:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.401306:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.401308:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.401310:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.401315:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.401320:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.401331:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.401336:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.401339:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513077.401340:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513077.401345:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.401349:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05540 00000800:00000200:20.0:1616513077.401353:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.405007:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513077.405073:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.405083:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.405092:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.200@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.405100:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.200@tcp1 of length 224 into portal 26 MB=0x604a29db4cdc0 00000400:00000200:18.0:1616513077.405107:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.200@tcp1 of length 224/224 into md 0xf97c71d9 [1] + 0 00000400:00000200:18.0:1616513077.405112:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78088 00000400:00000200:18.0:1616513077.405115:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.405118:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513077.405122:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78088 00000800:00000200:18.0:1616513077.405125:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.405128:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.405128:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.405135:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513077.405138:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.405138:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.405147:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.200@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.405156:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.200@tcp1 of length 224 into portal 12 MB=0x604a29db4ce40 00000400:00000200:14.0:1616513077.405162:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.200@tcp1 of length 224/224 into md 0xf96450b9 [1] + 68784 00000400:00000200:14.0:1616513077.405167:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.405170:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513077.405176:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.405181:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513077.405183:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946337349056 00000800:00000200:14.0:1616513077.405183:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:15.0:1616513077.405194:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1693946337349056/t0(0) o400->b7e33ddf-725d-98a0-2087-1f9fde2139ac@10.31.140.200@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.405207:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946337349056, offset 192 00000400:00000200:15.0:1616513077.405213:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.200@tcp1 00000400:00000200:15.0:1616513077.405220:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.200@tcp1 routed destination 00000100:00000200:26.0:1616513077.405240:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946337349184 00000400:00000200:15.0:1616513077.405241:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.200@tcp1(10.31.140.200@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.405251:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:26.0:1616513077.405253:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba2880 x1693946337349184/t0(0) o400->8334e65b-e357-3264-a729-38dd5b0c433f@10.31.140.200@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.405258:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.405261:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.405263:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:26.0:1616513077.405274:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946337349184, offset 192 00000800:00000200:15.0:1616513077.405274:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.405279:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:26.0:1616513077.405280:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.200@tcp1 00000100:00000200:15.0:1616513077.405281:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513077.405288:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.200@tcp1 routed destination 00000400:00000200:26.0:1616513077.405303:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.200@tcp1(10.31.140.200@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.405309:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.405315:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.405318:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.405320:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.405325:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.405331:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513077.405334:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.405341:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513077.405344:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513077.405345:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513077.405345:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513077.405350:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513077.405351:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:24.0:1616513077.405353:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513077.405357:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7a08 00000400:00000200:24.0:1616513077.405359:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513077.405361:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513077.405363:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae3b8 00000800:00000200:24.0:1616513077.405367:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.405369:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.416633:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.416691:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.416711:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513077.416719:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.119@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.416728:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.119@tcp1 of length 224 into portal 26 MB=0x5fffd24d3cf40 00000400:00000200:16.0:1616513077.416734:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.119@tcp1 of length 224/224 into md 0xf97c71e9 [1] + 0 00000400:00000200:16.0:1616513077.416739:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79dc0 00000400:00000200:16.0:1616513077.416741:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.416744:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513077.416748:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79dc0 00000800:00000200:16.0:1616513077.416751:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.416754:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.416755:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.416758:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.416768:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.416776:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.119@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.416784:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.119@tcp1 of length 224 into portal 12 MB=0x5fffd24d3cfc0 00000400:00000200:20.0:1616513077.416790:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.119@tcp1 of length 224/224 into md 0x865bbec1 [1] + 2688 00000400:00000200:20.0:1616513077.416795:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.416797:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513077.416804:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.416807:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.416809:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.416811:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.416812:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688837593222976 00010000:00000200:15.0:1616513077.416822:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688837593222976/t0(0) o400->bf40910f-d50a-c4bd-8e6c-18b6bfa3c886@10.31.138.119@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.416835:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688837593222976, offset 192 00000400:00000200:15.0:1616513077.416841:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.119@tcp1 00000400:00000200:15.0:1616513077.416848:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.119@tcp1 routed destination 00000400:00000200:15.0:1616513077.416862:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.119@tcp1(10.31.138.119@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513077.416868:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688837593223104 00000800:00000200:15.0:1616513077.416869:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.416874:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.416877:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.416879:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513077.416880:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06300 x1688837593223104/t0(0) o400->82d0a267-4c32-2c88-bfb4-265761426ae3@10.31.138.119@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.416884:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513077.416891:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.416891:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513077.416893:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688837593223104, offset 192 00000400:00000200:0.0:1616513077.416899:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.119@tcp1 00000400:00000200:0.0:1616513077.416906:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.119@tcp1 routed destination 00000400:00000200:0.0:1616513077.416920:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.119@tcp1(10.31.138.119@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.416927:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.416932:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.416935:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.416939:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.416944:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513077.416946:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513077.416949:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513077.416953:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.416957:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.416963:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.416970:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7760 00000800:00000200:2.0:1616513077.416974:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.416976:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513077.417002:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.417009:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.417011:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513077.417013:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513077.417017:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.417021:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05cb0 00000800:00000200:28.0:1616513077.417025:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.424972:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513077.425025:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.425033:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513077.425041:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.57@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.425049:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.57@tcp1 of length 224 into portal 26 MB=0x5ffdc5cd58f40 00000400:00000200:24.0:1616513077.425055:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.57@tcp1 of length 224/224 into md 0xf97c7219 [1] + 0 00000400:00000200:24.0:1616513077.425059:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79210 00000400:00000200:24.0:1616513077.425062:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.425064:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513077.425068:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79210 00000800:00000200:24.0:1616513077.425071:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.425073:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513077.425075:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.425077:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.425133:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696798940992 00010000:00000200:15.0:1616513077.425145:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688696798940992/t0(0) o400->99438664-5f28-b96f-07ae-68740715ad65@10.31.142.57@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.425159:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696798940992, offset 192 00000400:00000200:15.0:1616513077.425165:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.57@tcp1 00000800:00000200:7.2:1616513077.425170:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:15.0:1616513077.425174:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.57@tcp1 routed destination 00000400:00000200:15.0:1616513077.425189:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.57@tcp1(10.31.142.57@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.425196:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.425202:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.425204:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.425208:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513077.425213:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:15.0:1616513077.425220:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513077.425226:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.425236:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.425247:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.57@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.425256:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.57@tcp1 of length 224 into portal 12 MB=0x5ffdc5cd58fc0 00000400:00000200:18.0:1616513077.425263:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.57@tcp1 of length 224/224 into md 0xf96450b9 [1] + 69008 00000400:00000200:18.0:1616513077.425267:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.425270:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513077.425277:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513077.425279:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513077.425280:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513077.425285:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513077.425286:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513077.425288:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513077.425289:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513077.425295:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513077.425301:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7cb0 00000800:00000200:22.0:1616513077.425305:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.425340:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696798941120 00010000:00000200:26.0:1616513077.425351:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d5a00 x1688696798941120/t0(0) o400->1d7106cb-7ff3-d106-5310-09fda606b1a1@10.31.142.57@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.425365:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696798941120, offset 192 00000400:00000200:26.0:1616513077.425371:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.57@tcp1 00000400:00000200:26.0:1616513077.425379:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.57@tcp1 routed destination 00000400:00000200:26.0:1616513077.425393:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.57@tcp1(10.31.142.57@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.425399:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.425405:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.425408:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.425410:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.425414:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.425419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.425472:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.425480:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.425482:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513077.425488:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513077.425489:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.425493:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aff58 00000800:00000200:16.0:1616513077.425497:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.443929:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513077.443986:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.443996:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513077.444005:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.138@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.444014:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.138@tcp1 of length 224 into portal 26 MB=0x5ffd4af2ee200 00000400:00000200:24.0:1616513077.444021:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.138@tcp1 of length 224/224 into md 0xf97c7229 [1] + 0 00000400:00000200:24.0:1616513077.444028:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b796d8 00000400:00000200:24.0:1616513077.444030:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.444033:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513077.444038:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b796d8 00000800:00000200:20.0:1616513077.444040:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.444041:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.444044:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513077.444046:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.444101:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663820788224 00010000:00000200:15.0:1616513077.444111:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688663820788224/t0(0) o400->bf423f73-118d-7a21-54b3-c08f136d9a60@10.31.136.138@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.444124:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663820788224, offset 192 00000400:00000200:15.0:1616513077.444130:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.138@tcp1 00000400:00000200:15.0:1616513077.444137:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.138@tcp1 routed destination 00000400:00000200:15.0:1616513077.444151:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.138@tcp1(10.31.136.138@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.444158:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.444164:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.444166:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.444168:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.444173:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.444175:0:49167:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:15.0:1616513077.444179:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513077.444231:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.444241:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.444250:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.138@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.444259:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.138@tcp1 of length 224 into portal 12 MB=0x5ffd4af2ee280 00000400:00000200:14.0:1616513077.444266:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.138@tcp1 of length 224/224 into md 0xf96450b9 [1] + 69232 00000400:00000200:14.0:1616513077.444271:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.444274:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513077.444281:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513077.444284:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513077.444286:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513077.444293:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513077.444295:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513077.444296:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513077.444298:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513077.444305:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513077.444311:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7430 00000800:00000200:18.0:1616513077.444314:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.444352:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663820788352 00010000:00000200:26.0:1616513077.444364:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba4c80 x1688663820788352/t0(0) o400->9c14398a-c883-84b9-2ad3-1a7fbf70df07@10.31.136.138@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.444380:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663820788352, offset 192 00000400:00000200:26.0:1616513077.444387:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.138@tcp1 00000400:00000200:26.0:1616513077.444394:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.138@tcp1 routed destination 00000400:00000200:26.0:1616513077.444409:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.138@tcp1(10.31.136.138@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.444416:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.444422:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.444425:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.444427:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.444432:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.444440:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.444494:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.444501:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.444504:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.444510:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.444514:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aff58 00000800:00000200:28.0:1616513077.444518:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.444521:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.459550:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.459607:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.459616:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.459625:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.459635:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 224 into portal 26 MB=0x603d0322a88c0 00000400:00000200:20.0:1616513077.459642:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.147@tcp1 of length 224/224 into md 0xf97c7239 [1] + 0 00000400:00000200:20.0:1616513077.459647:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79d38 00000400:00000200:20.0:1616513077.459652:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.459655:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513077.459658:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513077.459659:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79d38 00000800:00000200:20.0:1616513077.459663:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.459665:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.459667:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.459726:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042589993152 00010000:00000200:15.0:1616513077.459737:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1693042589993152/t0(0) o400->f703e995-2e46-683a-1126-38f035bae8c7@10.242.104.147@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.459751:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693042589993152, offset 192 00000400:00000200:15.0:1616513077.459757:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:15.0:1616513077.459767:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:15.0:1616513077.459782:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.459788:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.459795:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.459797:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.459800:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.459805:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513077.459812:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.459812:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.459869:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.459877:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.459880:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.459887:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.459894:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6cc0 00000800:00000200:2.0:1616513077.459897:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.459900:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.474192:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513077.474193:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.474211:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.474220:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.474229:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.134@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.474239:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.134@tcp1 of length 224 into portal 12 MB=0x5ffe46473c380 00000400:00000200:18.0:1616513077.474245:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.134@tcp1 of length 224/224 into md 0xf96450b9 [1] + 69456 00000400:00000200:18.0:1616513077.474251:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.474254:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513077.474265:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.474268:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.474270:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513077.474274:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.474274:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513077.474283:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.474292:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.134@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.474301:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.134@tcp1 of length 224 into portal 26 MB=0x5ffe46473c100 00000400:00000200:28.0:1616513077.474307:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.134@tcp1 of length 224/224 into md 0xf97c7249 [1] + 0 00000400:00000200:28.0:1616513077.474312:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79f58 00000400:00000200:28.0:1616513077.474315:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.474318:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513077.474323:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79f58 00000800:00000200:28.0:1616513077.474326:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.474329:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.474329:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:26.0:1616513077.474332:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731286487936 00000800:00000200:28.0:1616513077.474333:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:26.0:1616513077.474345:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d2400 x1688731286487936/t0(0) o400->8fd75fbc-d628-a0d2-c848-6e73673429b7@10.31.140.134@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513077.474356:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731286487296 00000100:00000200:26.0:1616513077.474361:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688731286487936, offset 192 00000400:00000200:26.0:1616513077.474367:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.134@tcp1 00010000:00000200:15.0:1616513077.474367:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688731286487296/t0(0) o400->ab5b6fc9-6d43-88f6-d131-67217a3bc8f0@10.31.140.134@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513077.474375:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.134@tcp1 routed destination 00000100:00000200:15.0:1616513077.474382:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688731286487296, offset 192 00000400:00000200:15.0:1616513077.474388:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.134@tcp1 00000400:00000200:26.0:1616513077.474391:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.134@tcp1(10.31.140.134@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:15.0:1616513077.474396:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.134@tcp1 routed destination 00000800:00000200:26.0:1616513077.474398:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.474405:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.474408:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.474410:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513077.474411:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.134@tcp1(10.31.140.134@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.474415:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:15.0:1616513077.474418:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513077.474420:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.474424:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.474427:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.474428:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.474446:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.474448:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.474453:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513077.474460:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.474468:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.474471:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.474478:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.474484:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6ee0 00000800:00000200:24.0:1616513077.474487:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513077.474487:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.474490:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513077.474494:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513077.474497:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513077.474503:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513077.474507:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae3b8 00000800:00000200:24.0:1616513077.474512:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.474515:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.475482:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.475535:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.475544:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.475553:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.188@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.475564:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.188@tcp1 of length 224 into portal 26 MB=0x5ffd344c11a80 00000400:00000200:22.0:1616513077.475571:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.188@tcp1 of length 224/224 into md 0xf97c7269 [1] + 0 00000400:00000200:22.0:1616513077.475576:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79cb0 00000400:00000200:22.0:1616513077.475578:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.475581:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.475587:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79cb0 00000800:00000200:18.0:1616513077.475589:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.475592:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.475595:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.475596:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.475674:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513077.475682:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657740241536 00010000:00000200:15.0:1616513077.475693:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688657740241536/t0(0) o400->eb59cafc-c871-e4df-3992-13bc4c568c3c@10.31.128.188@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.475706:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657740241536, offset 192 00000400:00000200:15.0:1616513077.475711:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.188@tcp1 00000400:00000200:15.0:1616513077.475720:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.188@tcp1 routed destination 00000400:00000200:15.0:1616513077.475734:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.188@tcp1(10.31.128.188@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.475740:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513077.475743:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.475746:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.475748:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:15.0:1616513077.475751:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513077.475753:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.475756:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:15.0:1616513077.475762:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513077.475763:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.188@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.475772:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.188@tcp1 of length 224 into portal 12 MB=0x5ffd344c11e80 00000400:00000200:16.0:1616513077.475779:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.188@tcp1 of length 224/224 into md 0x865bbec1 [1] + 2912 00000400:00000200:16.0:1616513077.475784:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.475788:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513077.475795:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513077.475796:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513077.475798:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513077.475803:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513077.475805:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.475806:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.475808:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.475816:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.475822:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6198 00000800:00000200:28.0:1616513077.475826:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.475866:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657740242560 00010000:00000200:0.0:1616513077.475879:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd0900 x1688657740242560/t0(0) o400->3f9da855-d875-197b-e359-73deaa02c555@10.31.128.188@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.475894:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657740242560, offset 192 00000400:00000200:0.0:1616513077.475900:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.188@tcp1 00000400:00000200:0.0:1616513077.475908:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.188@tcp1 routed destination 00000400:00000200:0.0:1616513077.475923:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.188@tcp1(10.31.128.188@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.475930:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.475936:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.475939:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.475941:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.475946:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.475951:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.476005:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.476012:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.476014:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.476021:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.476025:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb058f8 00000800:00000200:2.0:1616513077.476029:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.476032:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.491482:0:49172:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513077.491538:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513077.491542:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513077.491548:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.491557:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.128.29@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.491567:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.128.29@tcp1 of length 224 into portal 26 MB=0x5f9c6b38734c0 00000400:00000200:18.0:1616513077.491573:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.128.29@tcp1 of length 224/224 into md 0xf97c7289 [1] + 0 00000400:00000200:18.0:1616513077.491578:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79a90 00000400:00000200:18.0:1616513077.491583:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.491586:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513077.491591:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513077.491591:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79a90 00000800:00000200:18.0:1616513077.491594:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.491597:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.491599:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513077.491614:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.491623:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.491633:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.128.29@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.491642:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.128.29@tcp1 of length 224 into portal 12 MB=0x5f9c6b3873540 00000400:00000200:20.0:1616513077.491648:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.128.29@tcp1 of length 224/224 into md 0x865bbec1 [1] + 3136 00000400:00000200:20.0:1616513077.491653:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.491656:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513077.491656:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682006694376640 00000800:00000200:20.0:1616513077.491662:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.491665:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.491667:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:15.0:1616513077.491667:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1682006694376640/t0(0) o400->46722202-aa9c-fb86-449a-3cb56026d7a0@10.242.128.29@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513077.491670:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.491681:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1682006694376640, offset 192 00000400:00000200:15.0:1616513077.491687:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.128.29@tcp1 00000400:00000200:15.0:1616513077.491695:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.128.29@tcp1 routed destination 00000400:00000200:15.0:1616513077.491711:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.128.29@tcp1(10.242.128.29@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.491717:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.491723:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.491726:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.491728:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.491733:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.491738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.491740:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513077.491747:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682006694376768 00010000:00000200:0.0:1616513077.491760:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06780 x1682006694376768/t0(0) o400->3906a3cd-55b3-05d4-2204-9ec6c7b7676e@10.242.128.29@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.491774:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1682006694376768, offset 192 00000400:00000200:0.0:1616513077.491781:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.128.29@tcp1 00000400:00000200:0.0:1616513077.491789:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.128.29@tcp1 routed destination 00000800:00000200:28.0:1616513077.491791:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.491798:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.491801:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513077.491804:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.128.29@tcp1(10.242.128.29@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513077.491808:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513077.491811:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513077.491814:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c75c8 00000800:00000200:0.0:1616513077.491817:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513077.491818:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513077.491820:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513077.491821:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513077.491822:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.491827:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.491831:0:49172:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.491885:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.491892:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.491895:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.491902:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.491906:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb044c8 00000800:00000200:2.0:1616513077.491910:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.491913:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.494421:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.494475:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.494485:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.494494:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.245@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.494503:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.245@tcp1 of length 224 into portal 12 MB=0x60182ea767700 00000400:00000200:20.0:1616513077.494510:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.245@tcp1 of length 224/224 into md 0x865bbec1 [1] + 3360 00000400:00000200:20.0:1616513077.494515:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.494517:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513077.494524:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.494527:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.494528:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.494533:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.494589:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511651272448 00010000:00000200:0.0:1616513077.494601:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03600 x1690511651272448/t0(0) o400->69351bdf-d12e-17e5-d51e-a9d520be9cb8@10.31.130.245@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.494615:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511651272448, offset 192 00000400:00000200:0.0:1616513077.494623:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.245@tcp1 00000400:00000200:0.0:1616513077.494631:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.245@tcp1 routed destination 00000400:00000200:0.0:1616513077.494646:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.245@tcp1(10.31.130.245@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.494653:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.494659:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.494661:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.494663:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.494668:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.494673:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.494728:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.494735:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.494738:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513077.494743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513077.494744:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.494748:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04c38 00000800:00000200:28.0:1616513077.494752:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.494755:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513077.494803:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.494813:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.494822:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.245@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.494831:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.245@tcp1 of length 224 into portal 26 MB=0x60182ea767280 00000400:00000200:22.0:1616513077.494837:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.245@tcp1 of length 224/224 into md 0xf97c72a9 [1] + 0 00000400:00000200:22.0:1616513077.494842:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79ed0 00000400:00000200:22.0:1616513077.494845:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.494848:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.494852:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79ed0 00000800:00000200:22.0:1616513077.494856:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.494856:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.494858:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.494860:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.494917:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511651271296 00010000:00000200:15.0:1616513077.494928:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1690511651271296/t0(0) o400->62971c6d-d858-814f-3ddd-948c9bec19c4@10.31.130.245@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.494942:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511651271296, offset 192 00000400:00000200:15.0:1616513077.494949:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.245@tcp1 00000400:00000200:15.0:1616513077.494957:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.245@tcp1 routed destination 00000400:00000200:15.0:1616513077.494973:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.245@tcp1(10.31.130.245@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.494979:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.494986:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.494989:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.495002:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.495006:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.495011:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.495013:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513077.495064:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.495072:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.495075:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.495082:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.495090:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6e58 00000800:00000200:2.0:1616513077.495093:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.495096:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.521198:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.521254:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513077.521255:0:49175:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513077.521263:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513077.521273:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.132@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.521282:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.132@tcp1 of length 224 into portal 12 MB=0x604a4706b1080 00000400:00000200:16.0:1616513077.521289:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.132@tcp1 of length 224/224 into md 0x865bbec1 [1] + 3584 00000400:00000200:16.0:1616513077.521294:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.521296:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513077.521303:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.521306:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.521308:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.521310:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513077.521311:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.521320:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.521329:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.132@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.521338:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.132@tcp1 of length 224 into portal 26 MB=0x604a4706b0e40 00000400:00000200:18.0:1616513077.521345:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.132@tcp1 of length 224/224 into md 0xf97c72b9 [1] + 0 00000400:00000200:18.0:1616513077.521349:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78440 00000400:00000200:18.0:1616513077.521352:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.521355:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513077.521359:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78440 00000800:00000200:18.0:1616513077.521363:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.521365:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513077.521367:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513077.521367:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954167476352 00000800:00000200:18.0:1616513077.521370:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513077.521380:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd0d80 x1693954167476352/t0(0) o400->443aeadb-8993-870b-e21d-8d94182658ac@10.31.141.132@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.521394:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954167476352, offset 192 00000400:00000200:0.0:1616513077.521401:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.132@tcp1 00000400:00000200:0.0:1616513077.521408:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.132@tcp1 routed destination 00000100:00000200:15.0:1616513077.521422:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954167475776 00000400:00000200:0.0:1616513077.521423:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.132@tcp1(10.31.141.132@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.521430:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:15.0:1616513077.521432:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1693954167475776/t0(0) o400->35613f7f-2aa2-1fd2-2300-1e55d0d536c0@10.31.141.132@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513077.521450:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.521453:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.521455:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.521459:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954167475776, offset 192 00000800:00000200:0.0:1616513077.521460:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.521465:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513077.521468:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.132@tcp1 00000400:00000200:15.0:1616513077.521477:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.132@tcp1 routed destination 00000400:00000200:15.0:1616513077.521491:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.132@tcp1(10.31.141.132@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.521498:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.521504:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.521508:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.521510:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.521515:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.521519:0:49175:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513077.521520:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:15.0:1616513077.521522:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513077.521527:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.521529:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.521536:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.521540:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000800:00000200:28.0:1616513077.521544:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.521547:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513077.521573:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.521580:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.521583:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.521590:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.521597:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c64c8 00000800:00000200:2.0:1616513077.521600:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.521603:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.541242:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.541298:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.541308:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513077.541317:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.130@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.541326:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.130@tcp1 of length 224 into portal 26 MB=0x5ffdb8cc28e00 00000400:00000200:16.0:1616513077.541333:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.130@tcp1 of length 224/224 into md 0xf97c72d9 [1] + 0 00000400:00000200:16.0:1616513077.541338:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79320 00000400:00000200:16.0:1616513077.541341:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.541344:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513077.541348:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79320 00000800:00000200:16.0:1616513077.541351:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.541352:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.541354:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.541356:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.541414:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693308034560 00010000:00000200:15.0:1616513077.541425:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688693308034560/t0(0) o400->f8aec801-8a5c-9255-3c4a-70c7b15d3540@10.31.132.130@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.541452:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693308034560, offset 192 00000400:00000200:15.0:1616513077.541459:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.130@tcp1 00000400:00000200:15.0:1616513077.541467:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.130@tcp1 routed destination 00000400:00000200:15.0:1616513077.541483:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.130@tcp1(10.31.132.130@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.541489:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.541495:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.541498:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.541500:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.541505:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.541510:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.541512:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513077.541563:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.541570:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.541574:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.541580:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.541586:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6000 00000800:00000200:28.0:1616513077.541590:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.541593:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.542120:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.542176:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.542185:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.542193:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.130@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.542202:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.130@tcp1 of length 224 into portal 12 MB=0x5ffdb8cc28f40 00000400:00000200:22.0:1616513077.542208:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.130@tcp1 of length 224/224 into md 0xf96450b9 [1] + 69680 00000400:00000200:22.0:1616513077.542213:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.542216:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.542224:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.542226:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.542228:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.542230:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.542289:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693308034880 00010000:00000200:26.0:1616513077.542302:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a28900 x1688693308034880/t0(0) o400->739e6e72-d24a-449e-61ac-bb02d86ea722@10.31.132.130@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.542317:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693308034880, offset 192 00000400:00000200:26.0:1616513077.542323:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.130@tcp1 00000400:00000200:26.0:1616513077.542331:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.130@tcp1 routed destination 00000400:00000200:26.0:1616513077.542349:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.130@tcp1(10.31.132.130@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.542355:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513077.542361:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.542364:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513077.542366:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.542372:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.542377:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.542390:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.542397:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.542400:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.542406:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.542410:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aedd0 00000800:00000200:2.0:1616513077.542414:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.542417:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.545347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.545403:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.545412:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.545421:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.152@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.545430:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.152@tcp1 of length 224 into portal 26 MB=0x5ffda39038780 00000400:00000200:20.0:1616513077.545451:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.152@tcp1 of length 224/224 into md 0xf97c72f9 [1] + 0 00000800:00000200:16.0:1616513077.545456:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513077.545458:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b786e8 00000400:00000200:20.0:1616513077.545461:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.545463:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513077.545468:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b786e8 00000800:00000200:20.0:1616513077.545471:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.545474:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.545476:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.545532:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688687608039296 00010000:00000200:15.0:1616513077.545543:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688687608039296/t0(0) o400->48e52b54-bbbc-c798-54e9-a64610a8dd5d@10.31.129.152@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.545558:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688687608039296, offset 192 00000400:00000200:15.0:1616513077.545564:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.152@tcp1 00000400:00000200:15.0:1616513077.545572:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.152@tcp1 routed destination 00000400:00000200:15.0:1616513077.545588:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.152@tcp1(10.31.129.152@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.545595:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.545601:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.545603:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.545606:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.545611:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.545615:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.545618:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513077.545668:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.545675:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.545678:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.545685:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.545694:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c67f8 00000800:00000200:28.0:1616513077.545697:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.545700:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.545707:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.545718:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.545725:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.545726:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513077.545736:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.152@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.545746:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.152@tcp1 of length 224 into portal 12 MB=0x5ffda39038bc0 00000400:00000200:16.0:1616513077.545752:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.152@tcp1 of length 224/224 into md 0x865bbec1 [1] + 3808 00000400:00000200:16.0:1616513077.545757:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.545760:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513077.545767:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.545770:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.545772:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.545842:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688687608040384 00010000:00000200:0.0:1616513077.545854:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd3a80 x1688687608040384/t0(0) o400->8876b5c1-539e-b6e4-91bf-225642d13241@10.31.129.152@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.545867:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688687608040384, offset 192 00000400:00000200:0.0:1616513077.545873:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.152@tcp1 00000400:00000200:0.0:1616513077.545880:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.152@tcp1 routed destination 00000400:00000200:0.0:1616513077.545894:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.152@tcp1(10.31.129.152@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.545900:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.545906:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.545910:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.545912:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.545917:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.545922:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.545973:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.545979:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513077.545982:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513077.545988:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513077.545992:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a90 00000800:00000200:22.0:1616513077.545995:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.545998:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.578154:0:49181:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513077.578210:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.578219:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.578229:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.22@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.578238:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.22@tcp1 of length 224 into portal 26 MB=0x60214c2cded00 00000400:00000200:2.0:1616513077.578245:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.22@tcp1 of length 224/224 into md 0xf97c7319 [1] + 0 00000400:00000200:2.0:1616513077.578249:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b785d8 00000400:00000200:2.0:1616513077.578252:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.578255:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513077.578261:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b785d8 00000800:00000200:2.0:1616513077.578264:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.578266:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513077.578268:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.578271:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.578326:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691138051140864 00010000:00000200:15.0:1616513077.578338:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1691138051140864/t0(0) o400->d7ab2d5b-097e-8e02-769e-d10a916b5cee@10.31.129.22@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.578352:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691138051140864, offset 192 00000400:00000200:15.0:1616513077.578359:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.22@tcp1 00000400:00000200:15.0:1616513077.578367:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.22@tcp1 routed destination 00000400:00000200:15.0:1616513077.578383:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.22@tcp1(10.31.129.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.578390:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.578396:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.578399:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.578401:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.578407:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513077.578414:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513077.578439:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.578495:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.578504:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.578513:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.22@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.578522:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.22@tcp1 of length 224 into portal 12 MB=0x60214c2cdf180 00000400:00000200:28.0:1616513077.578528:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.22@tcp1 of length 224/224 into md 0x865bbec1 [1] + 4032 00000400:00000200:28.0:1616513077.578533:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.578536:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:22.0:1616513077.578542:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925088, portal 4 00000800:00000200:28.0:1616513077.578543:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513077.578546:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513077.578548:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:22.0:1616513077.578548:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925088, offset 0 00000800:00000200:28.0:1616513077.578550:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.578554:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513077.578555:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513077.578555:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:28.0:1616513077.578557:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513077.578561:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:28.0:1616513077.578562:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.578567:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6880 00000800:00000200:28.0:1616513077.578570:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513077.578575:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513077.578582:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:22.0:1616513077.578589:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513077.578592:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:22.0:1616513077.578594:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:22.0:1616513077.578599:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:0.0:1616513077.578607:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691138051142016 00010000:00000200:0.0:1616513077.578621:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6724380 x1691138051142016/t0(0) o400->2ea66dcd-6795-1518-5b87-7fe7c900fcd5@10.31.129.22@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.578635:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691138051142016, offset 192 00000400:00000200:0.0:1616513077.578642:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.22@tcp1 00000400:00000200:0.0:1616513077.578649:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.22@tcp1 routed destination 00000800:00000200:1.2:1616513077.578655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:0.0:1616513077.578665:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.22@tcp1(10.31.129.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.578671:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:1.0:1616513077.578677:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:0.0:1616513077.578677:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.578680:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.578682:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:1.0:1616513077.578684:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513077.578687:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513077.578687:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.578692:0:49181:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:1.0:1616513077.578695:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513077.578703:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9641200 x1693067725925088/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513084 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513077.578704:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.578709:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513077.578711:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513077.578716:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a770 00000400:00000200:22.0:1616513077.578717:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513077.578720:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb045d8 00000800:00000200:1.0:1616513077.578721:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:22.0:1616513077.578723:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:1.0:1616513077.578724:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:23.0:1616513077.578725:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925104, portal 4 00000800:00000200:22.0:1616513077.578726:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513077.578731:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925104, offset 0 00000400:00000200:23.0:1616513077.578739:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:23.0:1616513077.578745:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:23.0:1616513077.578758:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513077.578765:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:23.0:1616513077.578772:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513077.578775:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:23.0:1616513077.578778:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:23.0:1616513077.578783:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:1.2:1616513077.578814:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513077.578824:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:27.2:1616513077.578830:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:1.0:1616513077.578831:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513077.578834:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513077.578839:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513077.578848:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626ae0 00000800:00000200:27.0:1616513077.578851:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:1.0:1616513077.578855:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffaf5 [1] + 192 00000800:00000200:27.0:1616513077.578856:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513077.578859:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:15.0:1616513077.578861:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:1.0:1616513077.578861:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:27.0:1616513077.578866:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:1.0:1616513077.578866:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9641200 x1693067725925088/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513084 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513077.578872:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be46780 x1693067725925104/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513084 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513077.578881:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:27.0:1616513077.578884:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3edd0 00000800:00000200:1.0:1616513077.578884:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513077.578886:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:27.0:1616513077.578888:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:22.0:1616513077.578888:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9641200 x1693067725925088/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513084 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513077.578900:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9641200 x1693067725925088/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513084 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513077.578909:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:27.2:1616513077.578998:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513077.579008:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513077.579015:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513077.579023:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513077.579031:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626af0 00000400:00000200:27.0:1616513077.579037:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371663bd [1] + 192 00000400:00000200:27.0:1616513077.579042:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513077.579046:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be46780 x1693067725925104/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513084 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513077.579060:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513077.579061:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513077.579063:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513077.579065:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:23.0:1616513077.579068:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be46780 x1693067725925104/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513084 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513077.579081:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be46780 x1693067725925104/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513084 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513077.579090:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eee0 00000800:00000200:7.2:1616513077.584988:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.585001:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.585010:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.585019:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.26@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.585029:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.26@tcp1 of length 224 into portal 26 MB=0x60182dce9a940 00000400:00000200:14.0:1616513077.585036:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.26@tcp1 of length 224/224 into md 0xf97c7329 [1] + 0 00000400:00000200:14.0:1616513077.585041:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78f68 00000400:00000200:14.0:1616513077.585043:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.585046:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513077.585051:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78f68 00000800:00000200:14.0:1616513077.585054:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.585055:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.585057:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.585060:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.585115:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511423940928 00010000:00000200:15.0:1616513077.585126:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1690511423940928/t0(0) o400->1f8e106a-2640-c097-abaa-7c8155f69311@10.31.133.26@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.585140:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511423940928, offset 192 00000400:00000200:15.0:1616513077.585146:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.26@tcp1 00000400:00000200:15.0:1616513077.585154:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.26@tcp1 routed destination 00000400:00000200:15.0:1616513077.585170:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.26@tcp1(10.31.133.26@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.585176:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.585182:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.585185:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.585187:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.585192:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513077.585198:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.585207:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513077.585238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513077.585261:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.585270:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.585279:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.26@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.585288:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.26@tcp1 of length 224 into portal 12 MB=0x60182dce9aa40 00000400:00000200:22.0:1616513077.585294:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.26@tcp1 of length 224/224 into md 0xf96450b9 [1] + 69904 00000400:00000200:22.0:1616513077.585299:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.585302:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.585309:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.585312:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513077.585314:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513077.585315:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.585319:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513077.585321:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.585324:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.585331:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.585337:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6d48 00000800:00000200:16.0:1616513077.585340:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.585343:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513077.585373:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511423941184 00010000:00000200:26.0:1616513077.585385:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a28d80 x1690511423941184/t0(0) o400->51e7ebd4-2159-0460-e9a1-564de6334465@10.31.133.26@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.585400:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511423941184, offset 192 00000400:00000200:26.0:1616513077.585406:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.26@tcp1 00000400:00000200:26.0:1616513077.585414:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.26@tcp1 routed destination 00000400:00000200:26.0:1616513077.585429:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.26@tcp1(10.31.133.26@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.585448:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513077.585454:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.585457:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513077.585459:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.585464:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.585468:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.585479:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.585484:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.585487:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513077.585489:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513077.585495:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.585499:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:2.0:1616513077.585503:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.595689:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.595743:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.595753:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.595761:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.103@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.595771:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.103@tcp1 of length 224 into portal 26 MB=0x6012ea414d800 00000400:00000200:22.0:1616513077.595778:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.103@tcp1 of length 224/224 into md 0xf97c7339 [1] + 0 00000400:00000200:22.0:1616513077.595783:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79b18 00000400:00000200:22.0:1616513077.595785:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.595788:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.595793:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79b18 00000800:00000200:22.0:1616513077.595796:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.595796:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.595799:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.595800:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.595863:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690149693216768 00010000:00000200:15.0:1616513077.595874:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1690149693216768/t0(0) o400->65bc7e38-8bde-4996-455e-cb1e5e546d21@10.31.140.103@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.595888:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690149693216768, offset 192 00000400:00000200:15.0:1616513077.595896:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.103@tcp1 00000400:00000200:15.0:1616513077.595905:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.103@tcp1 routed destination 00000800:00000200:11.2:1616513077.595916:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:15.0:1616513077.595921:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.103@tcp1(10.31.140.103@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.595928:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.595934:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.595936:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.595939:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:15.0:1616513077.595943:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:15.0:1616513077.595950:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513077.595973:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.595982:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.595991:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.103@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.596001:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.103@tcp1 of length 224 into portal 12 MB=0x6012ea414d880 00000400:00000200:28.0:1616513077.596007:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.103@tcp1 of length 224/224 into md 0x865bbec1 [1] + 4256 00000400:00000200:28.0:1616513077.596012:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.596015:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513077.596021:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513077.596024:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513077.596025:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.596028:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.596036:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.596040:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.596046:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.596052:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7650 00000800:00000200:20.0:1616513077.596056:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.596059:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.596085:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690149693216896 00010000:00000200:0.0:1616513077.596098:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6722880 x1690149693216896/t0(0) o400->9a20d158-9b39-0a86-4038-c956d4168beb@10.31.140.103@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.596112:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690149693216896, offset 192 00000400:00000200:0.0:1616513077.596119:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.103@tcp1 00000400:00000200:0.0:1616513077.596127:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.103@tcp1 routed destination 00000400:00000200:0.0:1616513077.596142:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.103@tcp1(10.31.140.103@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.596149:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.596155:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.596158:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.596160:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.596165:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.596171:0:49183:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.596223:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.596230:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513077.596232:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513077.596238:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.596242:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb054b8 00000800:00000200:14.0:1616513077.596246:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.596249:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.604064:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513077.604119:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.604129:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.604138:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.89@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.604148:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.89@tcp1 of length 224 into portal 26 MB=0x6011233ddf440 00000400:00000200:18.0:1616513077.604154:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.89@tcp1 of length 224/224 into md 0xf97c7349 [1] + 0 00000400:00000200:18.0:1616513077.604159:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78cc0 00000400:00000200:18.0:1616513077.604162:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.604165:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513077.604169:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78cc0 00000800:00000200:18.0:1616513077.604173:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.604177:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513077.604179:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.604182:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.604234:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690027551487040 00000800:00000200:7.2:1616513077.604249:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:15.0:1616513077.604255:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1690027551487040/t0(0) o400->eaeb7bff-f746-4605-4f16-346b05b96b4c@10.31.128.89@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.604268:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690027551487040, offset 192 00000400:00000200:15.0:1616513077.604274:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.89@tcp1 00000400:00000200:15.0:1616513077.604282:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.89@tcp1 routed destination 00000400:00000200:15.0:1616513077.604297:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.89@tcp1(10.31.128.89@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.604303:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513077.604305:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.604309:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.604311:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.604313:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513077.604314:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.604318:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513077.604322:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.89@tcp1 : PUT - for me 00000100:00000200:15.0:1616513077.604324:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513077.604331:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.89@tcp1 of length 224 into portal 12 MB=0x6011233ddf940 00000400:00000200:14.0:1616513077.604337:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.89@tcp1 of length 224/224 into md 0xf96450b9 [1] + 70128 00000400:00000200:14.0:1616513077.604341:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.604344:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513077.604350:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.604353:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.604354:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.604357:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.604363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513077.604413:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690027551488320 00000800:00000200:24.0:1616513077.604422:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:26.0:1616513077.604425:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba3600 x1690027551488320/t0(0) o400->9f85c469-e852-2661-1421-da2e88923684@10.31.128.89@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513077.604429:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513077.604432:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513077.604452:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:26.0:1616513077.604453:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690027551488320, offset 192 00000400:00000200:24.0:1616513077.604458:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6aa0 00000400:00000200:26.0:1616513077.604459:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.89@tcp1 00000800:00000200:24.0:1616513077.604462:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.604465:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513077.604468:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.89@tcp1 routed destination 00000400:00000200:26.0:1616513077.604483:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.89@tcp1(10.31.128.89@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.604489:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513077.604495:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.604498:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513077.604502:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.604507:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.604512:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.604521:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.604525:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513077.604527:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513077.604530:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513077.604532:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513077.604535:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aebb0 00000800:00000200:18.0:1616513077.604538:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.610773:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.610834:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.610843:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.610852:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.194@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.610862:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.194@tcp1 of length 224 into portal 26 MB=0x5ffd4a268b2c0 00000400:00000200:20.0:1616513077.610869:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.194@tcp1 of length 224/224 into md 0xf97c7359 [1] + 0 00000400:00000200:20.0:1616513077.610874:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78990 00000400:00000200:20.0:1616513077.610877:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.610880:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513077.610884:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78990 00000800:00000200:20.0:1616513077.610887:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.610887:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.610890:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.610892:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.610947:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663606473408 00000800:00000200:11.2:1616513077.610954:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:15.0:1616513077.610958:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688663606473408/t0(0) o400->4533f9c1-7b8f-203d-7f51-cc40acdb0664@10.31.132.194@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.610971:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663606473408, offset 192 00000400:00000200:15.0:1616513077.610976:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.194@tcp1 00000400:00000200:15.0:1616513077.610984:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.194@tcp1 routed destination 00000400:00000200:15.0:1616513077.610999:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.194@tcp1(10.31.132.194@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.611010:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513077.611014:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.611015:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.611018:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:15.0:1616513077.611020:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513077.611024:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.611024:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:15.0:1616513077.611031:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513077.611032:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.194@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.611042:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.194@tcp1 of length 224 into portal 12 MB=0x5ffd4a268b380 00000400:00000200:24.0:1616513077.611048:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.194@tcp1 of length 224/224 into md 0x865bbec1 [1] + 4480 00000400:00000200:24.0:1616513077.611053:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.611056:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513077.611063:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513077.611067:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513077.611068:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:24.0:1616513077.611075:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513077.611076:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513077.611079:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513077.611080:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513077.611086:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.611093:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6bb0 00000800:00000200:28.0:1616513077.611097:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.611127:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663606473600 00010000:00000200:0.0:1616513077.611140:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc6300 x1688663606473600/t0(0) o400->8bf97b2c-67b6-f0e3-567c-d750fe1d597d@10.31.132.194@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.611165:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663606473600, offset 192 00000400:00000200:0.0:1616513077.611171:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.194@tcp1 00000400:00000200:0.0:1616513077.611179:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.194@tcp1 routed destination 00000400:00000200:0.0:1616513077.611194:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.194@tcp1(10.31.132.194@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.611201:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.611207:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.611210:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.611212:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.611217:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.611224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.611285:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.611292:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.611294:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.611301:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.611305:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a08 00000800:00000200:2.0:1616513077.611309:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.611312:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.622925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.622979:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.622987:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.622995:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.59@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.623003:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.59@tcp1 of length 224 into portal 26 MB=0x60479fc9a0f80 00000400:00000200:22.0:1616513077.623010:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.59@tcp1 of length 224/224 into md 0xf97c7369 [1] + 0 00000400:00000200:22.0:1616513077.623014:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79e48 00000400:00000200:22.0:1616513077.623016:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.623019:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.623024:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79e48 00000800:00000200:22.0:1616513077.623026:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.623029:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.623030:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.623033:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.623043:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.623052:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.59@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.623061:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.59@tcp1 of length 224 into portal 12 MB=0x60479fc9a11c0 00000400:00000200:18.0:1616513077.623067:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.59@tcp1 of length 224/224 into md 0xf96450b9 [1] + 70352 00000400:00000200:18.0:1616513077.623072:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.623075:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513077.623082:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.623085:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513077.623087:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513077.623088:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693771835772800 00000800:00000200:18.0:1616513077.623091:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:15.0:1616513077.623099:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1693771835772800/t0(0) o400->b6f6f604-3062-9bee-2b51-15ec2c32d6f6@10.31.143.59@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.623113:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693771835772800, offset 192 00000400:00000200:15.0:1616513077.623120:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.59@tcp1 00000400:00000200:15.0:1616513077.623127:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.59@tcp1 routed destination 00000400:00000200:15.0:1616513077.623143:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.59@tcp1(10.31.143.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513077.623146:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693771835773376 00000800:00000200:15.0:1616513077.623149:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.623155:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:26.0:1616513077.623158:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebba80 x1693771835773376/t0(0) o400->b76f1f92-eefb-c9d3-91c5-cd4fcbcf082b@10.31.143.59@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.623158:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.623160:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.623165:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513077.623172:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513077.623173:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693771835773376, offset 192 00000400:00000200:26.0:1616513077.623179:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.59@tcp1 00000400:00000200:26.0:1616513077.623187:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.59@tcp1 routed destination 00000400:00000200:26.0:1616513077.623201:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.59@tcp1(10.31.143.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.623208:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513077.623211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.623214:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.623217:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513077.623219:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.623224:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.623237:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513077.623279:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.623286:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.623289:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.623296:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.623302:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6088 00000800:00000200:20.0:1616513077.623306:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.623309:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513077.623312:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.623318:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513077.623320:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513077.623323:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513077.623326:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.623330:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af430 00000800:00000200:14.0:1616513077.623333:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.644278:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513077.644335:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.644344:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513077.644353:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.103@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.644363:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.103@tcp1 of length 224 into portal 26 MB=0x5ffd3fa6154c0 00000400:00000200:24.0:1616513077.644370:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.103@tcp1 of length 224/224 into md 0xf97c7399 [1] + 0 00000400:00000200:24.0:1616513077.644374:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79430 00000400:00000200:24.0:1616513077.644377:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.644380:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513077.644385:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79430 00000800:00000200:24.0:1616513077.644388:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.644389:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.644390:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513077.644393:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.644439:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513077.644461:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660787418304 00010000:00000200:15.0:1616513077.644472:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688660787418304/t0(0) o400->d64c77bd-166b-993f-aa8e-9fb51bb24760@10.31.128.103@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.644485:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660787418304, offset 192 00000400:00000200:15.0:1616513077.644491:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.103@tcp1 00000400:00000200:15.0:1616513077.644499:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.103@tcp1 routed destination 00000800:00000200:20.0:1616513077.644512:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513077.644514:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.103@tcp1(10.31.128.103@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.644520:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513077.644523:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.644526:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.644529:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513077.644532:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.103@tcp1 : PUT - for me 00000800:00000200:15.0:1616513077.644533:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:15.0:1616513077.644538:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:20.0:1616513077.644542:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.103@tcp1 of length 224 into portal 12 MB=0x5ffd3fa615780 00000100:00000200:15.0:1616513077.644544:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513077.644548:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.103@tcp1 of length 224/224 into md 0x865bbec1 [1] + 4704 00000400:00000200:20.0:1616513077.644553:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.644556:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513077.644564:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513077.644567:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513077.644569:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:20.0:1616513077.644576:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513077.644577:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513077.644579:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.644580:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.644587:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.644593:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6088 00000100:00000200:0.0:1616513077.644596:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660787419008 00000800:00000200:16.0:1616513077.644597:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513077.644609:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03180 x1688660787419008/t0(0) o400->d9b0981c-b72d-46f9-685f-7c30ce889248@10.31.128.103@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.644623:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660787419008, offset 192 00000400:00000200:0.0:1616513077.644630:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.103@tcp1 00000400:00000200:0.0:1616513077.644638:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.103@tcp1 routed destination 00000400:00000200:0.0:1616513077.644653:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.103@tcp1(10.31.128.103@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.644660:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.644669:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.644671:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.644674:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.644678:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.644684:0:49188:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.644741:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.644748:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.644750:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.644757:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.644761:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05188 00000800:00000200:2.0:1616513077.644765:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.644768:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.656601:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.656655:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.656665:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513077.656668:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513077.656674:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.80@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.656683:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.80@tcp1 of length 224 into portal 12 MB=0x5ffd4721dd680 00000400:00000200:22.0:1616513077.656690:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.80@tcp1 of length 224/224 into md 0xf96450b9 [1] + 70576 00000400:00000200:22.0:1616513077.656695:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.656700:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.656706:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.656709:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.656711:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.656766:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662796260992 00010000:00000200:26.0:1616513077.656778:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2ad00 x1688662796260992/t0(0) o400->8769f895-4206-8732-d211-1da53d024f2f@10.31.138.80@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.656791:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662796260992, offset 192 00000400:00000200:26.0:1616513077.656798:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.80@tcp1 00000400:00000200:26.0:1616513077.656804:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.80@tcp1 routed destination 00000400:00000200:26.0:1616513077.656818:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.80@tcp1(10.31.138.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.656824:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.656830:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.656832:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.656834:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.656838:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.656845:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.656902:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.656911:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513077.656920:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.80@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.656929:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.80@tcp1 of length 224 into portal 26 MB=0x5ffd4721dd600 00000400:00000200:24.0:1616513077.656936:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.80@tcp1 of length 224/224 into md 0xf97c73a9 [1] + 0 00000400:00000200:24.0:1616513077.656941:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79760 00000400:00000200:24.0:1616513077.656944:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.656946:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513077.656951:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79760 00000800:00000200:24.0:1616513077.656954:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513077.656955:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513077.656957:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513077.656962:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513077.656963:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513077.656965:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.656965:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.656972:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.656976:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af540 00000800:00000200:20.0:1616513077.656980:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.657016:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662796260864 00010000:00000200:15.0:1616513077.657027:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688662796260864/t0(0) o400->6c90a941-ac48-cdce-f298-5d613b3fed28@10.31.138.80@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.657042:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662796260864, offset 192 00000400:00000200:15.0:1616513077.657048:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.80@tcp1 00000400:00000200:15.0:1616513077.657056:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.80@tcp1 routed destination 00000400:00000200:15.0:1616513077.657071:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.80@tcp1(10.31.138.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.657078:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.657084:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.657087:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.657089:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.657095:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.657100:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.657101:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513077.657112:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.657119:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.657122:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.657129:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.657136:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6aa0 00000800:00000200:2.0:1616513077.657139:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.657142:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.661945:0:49190:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.662000:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.662009:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.662018:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.25@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.662027:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.25@tcp1 of length 224 into portal 26 MB=0x6048d6d289640 00000400:00000200:14.0:1616513077.662034:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.25@tcp1 of length 224/224 into md 0xf97c73c9 [1] + 0 00000400:00000200:14.0:1616513077.662039:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b794b8 00000400:00000200:14.0:1616513077.662041:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.662044:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513077.662049:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b794b8 00000800:00000200:14.0:1616513077.662052:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.662053:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.662055:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.662059:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.662062:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.662063:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.662072:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.25@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.662081:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.25@tcp1 of length 224 into portal 12 MB=0x6048d6d2898c0 00000400:00000200:22.0:1616513077.662087:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.25@tcp1 of length 224/224 into md 0xf96450b9 [1] + 70800 00000400:00000200:22.0:1616513077.662092:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.662094:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.662102:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.662105:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.662106:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.662112:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855328540224 00010000:00000200:15.0:1616513077.662123:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1693855328540224/t0(0) o400->f0326a22-2e6f-7f24-0e8c-f2409588e321@10.31.141.25@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.662136:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855328540224, offset 192 00000400:00000200:15.0:1616513077.662142:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.25@tcp1 00000400:00000200:15.0:1616513077.662150:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.25@tcp1 routed destination 00000400:00000200:15.0:1616513077.662164:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.25@tcp1(10.31.141.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513077.662166:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855328540864 00000800:00000200:15.0:1616513077.662171:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.662176:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:26.0:1616513077.662179:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2c380 x1693855328540864/t0(0) o400->7df73bc3-f10d-72ba-47d2-815e5e0a98e8@10.31.141.25@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.662179:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.662181:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.662185:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513077.662192:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513077.662193:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855328540864, offset 192 00000400:00000200:26.0:1616513077.662202:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.25@tcp1 00000400:00000200:26.0:1616513077.662210:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.25@tcp1 routed destination 00000400:00000200:26.0:1616513077.662225:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.25@tcp1(10.31.141.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.662232:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513077.662232:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.662238:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.662241:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513077.662243:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.662248:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.662252:0:49190:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513077.662289:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.662296:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.662299:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.662305:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513077.662307:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513077.662311:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6d48 00000800:00000200:18.0:1616513077.662314:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513077.662315:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513077.662316:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513077.662318:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513077.662318:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513077.662323:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513077.662327:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afb18 00000800:00000200:18.0:1616513077.662331:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.683553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.683609:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.683619:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.683628:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.55@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.683637:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.55@tcp1 of length 224 into portal 26 MB=0x60184259ef640 00000400:00000200:20.0:1616513077.683644:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.55@tcp1 of length 224/224 into md 0xf97c73d9 [1] + 0 00000400:00000200:20.0:1616513077.683649:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78ff0 00000400:00000200:20.0:1616513077.683652:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:7.2:1616513077.683652:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513077.683655:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513077.683659:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78ff0 00000800:00000200:28.0:1616513077.683662:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.683662:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.683665:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.683667:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513077.683707:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.683717:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:15.0:1616513077.683722:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516938749504 00000400:00000200:2.0:1616513077.683726:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.55@tcp1 : PUT - for me 00010000:00000200:15.0:1616513077.683732:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1690516938749504/t0(0) o400->2f6fbc30-5eb7-5918-b7d5-6cb43d738cbb@10.31.131.55@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513077.683735:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.55@tcp1 of length 224 into portal 12 MB=0x60184259ef6c0 00000400:00000200:2.0:1616513077.683741:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.55@tcp1 of length 224/224 into md 0xf96450b9 [1] + 71024 00000400:00000200:2.0:1616513077.683746:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:15.0:1616513077.683747:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690516938749504, offset 192 00000100:00000200:2.0:1616513077.683749:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:15.0:1616513077.683754:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.55@tcp1 00000800:00000200:2.0:1616513077.683755:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.683758:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:15.0:1616513077.683761:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.55@tcp1 routed destination 00000800:00000200:2.0:1616513077.683762:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.683765:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513077.683776:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.55@tcp1(10.31.131.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.683782:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.683787:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.683790:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.683792:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.683797:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.683802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.683803:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513077.683821:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516938749632 00010000:00000200:26.0:1616513077.683833:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621200 x1690516938749632/t0(0) o400->dea4b7dc-db52-dfaf-ab7a-763281d58862@10.31.131.55@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.683848:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690516938749632, offset 192 00000400:00000200:26.0:1616513077.683855:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.55@tcp1 00000800:00000200:16.0:1616513077.683855:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513077.683862:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.55@tcp1 routed destination 00000800:00000200:16.0:1616513077.683862:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.683866:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.683872:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513077.683878:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.55@tcp1(10.31.131.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513077.683878:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c67f8 00000800:00000200:16.0:1616513077.683882:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513077.683885:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513077.683887:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513077.683891:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.683893:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513077.683895:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.683900:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.683905:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.683956:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.683965:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513077.683967:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513077.683974:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.683977:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afc28 00000800:00000200:14.0:1616513077.683981:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.683983:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.709316:0:49195:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513077.709374:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.709383:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.709392:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.144@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.709401:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.144@tcp1 of length 224 into portal 26 MB=0x60183ab25f480 00000400:00000200:2.0:1616513077.709407:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.144@tcp1 of length 224/224 into md 0xf97c73f9 [1] + 0 00000400:00000200:2.0:1616513077.709412:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78330 00000400:00000200:2.0:1616513077.709415:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.709418:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513077.709423:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78330 00000800:00000200:2.0:1616513077.709426:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.709428:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.709429:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513077.709432:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.709488:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514883998848 00010000:00000200:15.0:1616513077.709499:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1690514883998848/t0(0) o400->f50d9faa-bd12-4944-c0c5-2a5f32df1d1d@10.31.131.144@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.709513:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514883998848, offset 192 00000400:00000200:15.0:1616513077.709520:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.144@tcp1 00000400:00000200:15.0:1616513077.709528:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.144@tcp1 routed destination 00000400:00000200:15.0:1616513077.709543:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.144@tcp1(10.31.131.144@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.709550:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.709556:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.709559:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.709561:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.709566:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513077.709573:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513077.709611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.709667:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.709674:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.709677:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.709684:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.709690:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c64c8 00000800:00000200:28.0:1616513077.709694:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.709697:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.709702:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.709710:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.144@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.709718:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.144@tcp1 of length 224 into portal 12 MB=0x60183ab25f840 00000800:00000200:20.0:1616513077.709720:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513077.709724:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.144@tcp1 of length 224/224 into md 0x865bbec1 [1] + 4928 00000400:00000200:28.0:1616513077.709729:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.709731:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513077.709737:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.709740:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.709742:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.709801:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514883999808 00010000:00000200:0.0:1616513077.709814:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6727080 x1690514883999808/t0(0) o400->0fa46b74-449f-677c-35d6-a7019c645484@10.31.131.144@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.709828:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514883999808, offset 192 00000400:00000200:0.0:1616513077.709835:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.144@tcp1 00000400:00000200:0.0:1616513077.709842:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.144@tcp1 routed destination 00000400:00000200:0.0:1616513077.709858:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.144@tcp1(10.31.131.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.709867:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.709873:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.709878:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.709880:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.709885:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.709891:0:49195:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.709903:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.709908:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513077.709911:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513077.709917:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.709921:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb056d8 00000800:00000200:14.0:1616513077.709925:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.709927:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.723664:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513077.723672:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513077.723721:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.723730:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513077.723739:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.82@tcp1 : PUT - for me 00000800:00000200:22.0:1616513077.723739:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513077.723748:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.82@tcp1 of length 224 into portal 26 MB=0x5ffd3f8916b80 00000800:00000200:22.0:1616513077.723749:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513077.723756:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.82@tcp1 of length 224/224 into md 0xf97c7409 [1] + 0 00000400:00000200:22.0:1616513077.723758:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.169@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.723761:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78908 00000400:00000200:24.0:1616513077.723765:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.723768:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.723768:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.169@tcp1 of length 224 into portal 26 MB=0x5ffd71b1ea880 00000400:00000200:24.0:1616513077.723772:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78908 00000800:00000200:20.0:1616513077.723774:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.723775:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513077.723777:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.169@tcp1 of length 224/224 into md 0xf97c7429 [1] + 0 00000800:00000200:24.0:1616513077.723778:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.723780:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513077.723782:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b783b8 00000400:00000200:22.0:1616513077.723785:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.723788:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.723792:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b783b8 00000800:00000200:22.0:1616513077.723795:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.723795:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.723798:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.723800:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.723845:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660757015424 00000100:00000200:24.0:1616513077.723848:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674221598848 00010000:00000200:15.0:1616513077.723856:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688660757015424/t0(0) o400->8b7ad990-5ce8-beb7-b3fd-819b9e412d81@10.31.129.82@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:24.0:1616513077.723859:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688674221598848/t0(0) o400->154e96c4-79fd-24b0-4e61-13df5d28e25a@10.31.129.169@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.723871:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660757015424, offset 192 00000100:00000200:24.0:1616513077.723873:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688674221598848, offset 192 00000400:00000200:15.0:1616513077.723878:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.82@tcp1 00000400:00000200:24.0:1616513077.723879:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.169@tcp1 00000400:00000200:15.0:1616513077.723887:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.82@tcp1 routed destination 00000400:00000200:24.0:1616513077.723888:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.169@tcp1 routed destination 00000400:00000200:24.0:1616513077.723904:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.169@tcp1(10.31.129.169@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:15.0:1616513077.723906:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.82@tcp1(10.31.129.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513077.723908:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513077.723911:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.723914:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513077.723917:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513077.723920:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.723920:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513077.723922:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:15.0:1616513077.723923:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.723925:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513077.723926:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:15.0:1616513077.723930:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513077.723935:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.723935:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.723938:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513077.723962:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.723972:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513077.723975:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:28.0:1616513077.723980:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.82@tcp1 : PUT - for me 00000800:00000200:16.0:1616513077.723982:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513077.723985:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513077.723986:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.723989:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.82@tcp1 of length 224 into portal 12 MB=0x5ffd3f8916f80 00000800:00000200:14.0:1616513077.723989:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513077.723992:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.723995:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.82@tcp1 of length 224/224 into md 0x865bbec1 [1] + 5152 00000800:00000200:14.0:1616513077.723996:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513077.723999:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.724001:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.724003:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3430 00000400:00000200:14.0:1616513077.724005:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:28.0:1616513077.724006:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513077.724007:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513077.724011:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c75c8 00000800:00000200:28.0:1616513077.724012:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513077.724014:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513077.724016:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513077.724017:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513077.724018:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.724078:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660757016448 00010000:00000200:0.0:1616513077.724090:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6723a80 x1688660757016448/t0(0) o400->131e644f-01ab-9f1e-847a-103283f31928@10.31.129.82@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.724105:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660757016448, offset 192 00000400:00000200:0.0:1616513077.724111:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.82@tcp1 00000400:00000200:0.0:1616513077.724118:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.82@tcp1 routed destination 00000400:00000200:0.0:1616513077.724134:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.82@tcp1(10.31.129.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.724141:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.724147:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.724149:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.724152:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.724157:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.724162:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.724171:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.724175:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513077.724177:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513077.724179:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513077.724183:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513077.724186:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04550 00000800:00000200:24.0:1616513077.724189:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.724794:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.724804:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.724810:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513077.724813:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513077.724818:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.169@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.724827:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.169@tcp1 of length 224 into portal 12 MB=0x5ffd71b1eacc0 00000400:00000200:16.0:1616513077.724833:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.169@tcp1 of length 224/224 into md 0x865bbec1 [1] + 5376 00000400:00000200:16.0:1616513077.724838:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.724841:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513077.724847:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.724850:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.724851:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.724858:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674221599936 00010000:00000200:0.0:1616513077.724865:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1200 x1688674221599936/t0(0) o400->15856cfd-8546-8907-7543-e4b49f34fc41@10.31.129.169@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.724876:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688674221599936, offset 192 00000400:00000200:0.0:1616513077.724881:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.169@tcp1 00000400:00000200:0.0:1616513077.724887:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.169@tcp1 routed destination 00000400:00000200:0.0:1616513077.724900:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.169@tcp1(10.31.129.169@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.724906:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.724911:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.724914:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.724915:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.724919:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.724924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.725017:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.725023:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.725026:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.725033:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.725037:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb042a8 00000800:00000200:2.0:1616513077.725041:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.725044:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.742389:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.742448:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.742460:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513077.742462:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513077.742468:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.84@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.742478:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.84@tcp1 of length 224 into portal 12 MB=0x5ffd447867cc0 00000400:00000200:20.0:1616513077.742484:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.84@tcp1 of length 224/224 into md 0x865bbec1 [1] + 5600 00000400:00000200:20.0:1616513077.742489:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.742492:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513077.742498:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.742502:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.742504:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.742558:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513077.742562:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662081699008 00000800:00000200:14.0:1616513077.742570:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513077.742574:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e00000 x1688662081699008/t0(0) o400->c4a0c36c-3a18-2b21-ded1-8acdcc701792@10.31.138.84@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513077.742578:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.742586:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.84@tcp1 : PUT - for me 00000100:00000200:0.0:1616513077.742589:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662081699008, offset 192 00000400:00000200:14.0:1616513077.742594:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.84@tcp1 of length 224 into portal 26 MB=0x5ffd447867c40 00000400:00000200:0.0:1616513077.742595:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.84@tcp1 00000400:00000200:14.0:1616513077.742600:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.84@tcp1 of length 224/224 into md 0xf97c7439 [1] + 0 00000400:00000200:0.0:1616513077.742603:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.84@tcp1 routed destination 00000400:00000200:14.0:1616513077.742604:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b782a8 00000400:00000200:14.0:1616513077.742607:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.742609:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513077.742613:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b782a8 00000800:00000200:14.0:1616513077.742616:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513077.742617:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.84@tcp1(10.31.138.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513077.742618:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.742620:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.742622:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513077.742624:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.742629:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.742632:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.742634:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.742639:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.742644:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.742697:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:15.0:1616513077.742698:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662081698880 00000800:00000200:28.0:1616513077.742704:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.742708:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:15.0:1616513077.742708:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688662081698880/t0(0) o400->23b59926-cb95-0b48-ef13-261f9cb58e7f@10.31.138.84@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513077.742715:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.742719:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb043b8 00000100:00000200:15.0:1616513077.742722:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662081698880, offset 192 00000800:00000200:28.0:1616513077.742723:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.742726:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:15.0:1616513077.742727:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.84@tcp1 00000400:00000200:15.0:1616513077.742735:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.84@tcp1 routed destination 00000400:00000200:15.0:1616513077.742749:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.84@tcp1(10.31.138.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.742755:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.742762:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.742764:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.742766:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.742771:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.742776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.742777:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513077.742828:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.742836:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.742839:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.742846:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.742852:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6ee0 00000800:00000200:2.0:1616513077.742856:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.742859:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.743145:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513077.743154:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.743161:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513077.743164:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513077.743170:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.16@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.743178:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.16@tcp1 of length 224 into portal 26 MB=0x5ffd5a3d37b00 00000400:00000200:24.0:1616513077.743184:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.16@tcp1 of length 224/224 into md 0xf97c7469 [1] + 0 00000400:00000200:24.0:1616513077.743189:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78000 00000400:00000200:24.0:1616513077.743193:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.743196:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513077.743205:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78000 00000800:00000200:24.0:1616513077.743208:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.743211:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.743213:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.743220:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667925216000 00010000:00000200:15.0:1616513077.743226:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688667925216000/t0(0) o400->a0006b45-174b-c084-e50f-67c44d22187c@10.31.136.16@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.743238:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667925216000, offset 192 00000400:00000200:15.0:1616513077.743243:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.16@tcp1 00000400:00000200:15.0:1616513077.743249:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.16@tcp1 routed destination 00000400:00000200:15.0:1616513077.743263:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.16@tcp1(10.31.136.16@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.743268:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.743274:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.743278:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.743280:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.743283:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513077.743288:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513077.743288:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.743298:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.743302:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.743304:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513077.743308:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513077.743310:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.743315:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7430 00000800:00000200:28.0:1616513077.743318:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.744100:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.744110:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.744115:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513077.744117:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513077.744123:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.16@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.744129:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.16@tcp1 of length 224 into portal 12 MB=0x5ffd5a3d37cc0 00000400:00000200:20.0:1616513077.744134:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.16@tcp1 of length 224/224 into md 0x865bbec1 [1] + 5824 00000400:00000200:20.0:1616513077.744138:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.744140:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513077.744145:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.744147:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.744149:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513077.744207:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667925216448 00010000:00000200:0.0:1616513077.744218:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07500 x1688667925216448/t0(0) o400->b22d1901-054f-fdc0-11bf-1476ab68d0dd@10.31.136.16@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.744232:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667925216448, offset 192 00000400:00000200:0.0:1616513077.744238:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.16@tcp1 00000400:00000200:0.0:1616513077.744245:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.16@tcp1 routed destination 00000400:00000200:0.0:1616513077.744259:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.16@tcp1(10.31.136.16@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.744266:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513077.744271:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.744274:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513077.744276:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513077.744281:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.744288:0:49199:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.744343:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.744349:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513077.744352:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513077.744359:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513077.744363:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05980 00000800:00000200:18.0:1616513077.744367:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.744370:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.760788:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513077.760843:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.760853:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.760861:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.43@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.760870:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.43@tcp1 of length 224 into portal 26 MB=0x5ffd3d58f11c0 00000400:00000200:2.0:1616513077.760877:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.43@tcp1 of length 224/224 into md 0xf97c7479 [1] + 0 00000400:00000200:2.0:1616513077.760883:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79980 00000400:00000200:2.0:1616513077.760886:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.760889:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513077.760894:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79980 00000800:00000200:22.0:1616513077.760896:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.760897:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.760900:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.760901:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.760959:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660169658816 00010000:00000200:15.0:1616513077.760970:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688660169658816/t0(0) o400->844f4560-e5d6-b1c6-16d3-dc6dc4795480@10.31.129.43@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.760984:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660169658816, offset 192 00000400:00000200:15.0:1616513077.760991:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.43@tcp1 00000400:00000200:15.0:1616513077.760998:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.43@tcp1 routed destination 00000400:00000200:15.0:1616513077.761014:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.43@tcp1(10.31.129.43@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.761021:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.761027:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.761030:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.761032:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.761037:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513077.761044:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513077.761082:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.761139:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.761146:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.761149:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.761156:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.761162:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7760 00000800:00000200:16.0:1616513077.761166:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.761169:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.761208:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513077.761263:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.761272:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.761282:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.43@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.761290:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.43@tcp1 of length 224 into portal 12 MB=0x5ffd3d58f1540 00000400:00000200:18.0:1616513077.761296:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.43@tcp1 of length 224/224 into md 0xf96450b9 [1] + 71248 00000400:00000200:18.0:1616513077.761301:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.761304:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513077.761310:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.761313:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.761315:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513077.761320:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.761374:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660169659712 00010000:00000200:26.0:1616513077.761387:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebde80 x1688660169659712/t0(0) o400->6d1e9908-772e-8d83-4b03-1f972bb0c395@10.31.129.43@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.761402:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660169659712, offset 192 00000400:00000200:26.0:1616513077.761408:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.43@tcp1 00000400:00000200:26.0:1616513077.761415:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.43@tcp1 routed destination 00000400:00000200:26.0:1616513077.761430:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.43@tcp1(10.31.129.43@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.761450:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513077.761457:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.761461:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513077.761463:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.761468:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.761472:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.761483:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.761488:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513077.761491:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513077.761492:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513077.761497:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513077.761501:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afcb0 00000800:00000200:22.0:1616513077.761507:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.777607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.777619:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.777628:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.777637:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.777645:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x60399133d7b80 00000400:00000200:14.0:1616513077.777652:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0xf96450b9 [1] + 71472 00000400:00000200:14.0:1616513077.777657:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.777660:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513077.777667:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.777669:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.777672:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.777674:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.777730:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805847939968 00010000:00000200:26.0:1616513077.777812:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba6780 x1692805847939968/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:703/0 lens 336/336 e 0 to 0 dl 1616513083 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513077.777827:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805847939968, offset 192 00000400:00000200:26.0:1616513077.777833:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:26.0:1616513077.777840:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:26.0:1616513077.777855:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.777861:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.777867:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.777870:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.777872:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.777877:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.777925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.777980:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.777987:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513077.777990:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513077.777996:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513077.778000:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeaa0 00000800:00000200:24.0:1616513077.778004:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.778007:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.781448:0:49203:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513077.781501:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513077.781526:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.781535:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.781544:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.150@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.781553:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.150@tcp1 of length 224 into portal 12 MB=0x6018a100658c0 00000800:00000200:16.0:1616513077.781557:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513077.781560:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.150@tcp1 of length 224/224 into md 0xf96450b9 [1] + 71808 00000400:00000200:2.0:1616513077.781565:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513077.781566:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000100:00000200:2.0:1616513077.781568:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513077.781574:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513077.781576:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.150@tcp1 : PUT - for me 00000800:00000200:2.0:1616513077.781577:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513077.781579:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.781582:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513077.781585:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.150@tcp1 of length 224 into portal 26 MB=0x6018a100654c0 00000400:00000200:16.0:1616513077.781592:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.150@tcp1 of length 224/224 into md 0xf97c7499 [1] + 0 00000400:00000200:16.0:1616513077.781597:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78bb0 00000400:00000200:16.0:1616513077.781600:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.781602:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513077.781609:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78bb0 00000800:00000200:28.0:1616513077.781612:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.781612:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.781615:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.781617:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513077.781647:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690542346229952 00010000:00000200:26.0:1616513077.781658:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623180 x1690542346229952/t0(0) o400->5c0f51f1-3fb2-763b-8052-a740919ce07e@10.31.131.150@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513077.781671:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690542346228928 00000100:00000200:26.0:1616513077.781672:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690542346229952, offset 192 00000400:00000200:26.0:1616513077.781679:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.150@tcp1 00010000:00000200:15.0:1616513077.781682:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1690542346228928/t0(0) o400->3f8f0302-ae1e-6450-f7e0-941fbf126c4b@10.31.131.150@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513077.781686:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.150@tcp1 routed destination 00000100:00000200:15.0:1616513077.781695:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690542346228928, offset 192 00000400:00000200:26.0:1616513077.781699:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.150@tcp1(10.31.131.150@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:15.0:1616513077.781702:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.150@tcp1 00000800:00000200:26.0:1616513077.781706:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:15.0:1616513077.781710:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.150@tcp1 routed destination 00000800:00000200:26.0:1616513077.781711:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.781714:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513077.781716:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.781720:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.781724:0:49203:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:15.0:1616513077.781725:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.150@tcp1(10.31.131.150@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.781731:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513077.781735:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513077.781737:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.781739:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.781741:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513077.781741:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513077.781744:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:15.0:1616513077.781746:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513077.781751:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513077.781752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.781753:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513077.781755:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae5d8 00000800:00000200:14.0:1616513077.781759:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.781762:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513077.781763:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.781770:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513077.781773:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513077.781780:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513077.781786:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7210 00000800:00000200:24.0:1616513077.781789:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.781793:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.801533:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.801591:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.801601:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.801610:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.143@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.801620:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.143@tcp1 of length 224 into portal 26 MB=0x6002a01114640 00000400:00000200:22.0:1616513077.801626:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.143@tcp1 of length 224/224 into md 0xf97c74a9 [1] + 0 00000400:00000200:22.0:1616513077.801631:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b793a8 00000400:00000200:22.0:1616513077.801634:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.801637:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.801641:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b793a8 00000800:00000200:22.0:1616513077.801645:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.801646:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.801647:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.801650:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.801706:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689030266799680 00010000:00000200:15.0:1616513077.801718:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1689030266799680/t0(0) o400->9366b67b-5caa-36ff-9d4e-33407e6a7dde@10.31.142.143@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.801732:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689030266799680, offset 192 00000400:00000200:15.0:1616513077.801738:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.143@tcp1 00000400:00000200:15.0:1616513077.801746:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.143@tcp1 routed destination 00000400:00000200:15.0:1616513077.801761:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.143@tcp1(10.31.142.143@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.801768:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.801774:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.801777:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.801779:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.801784:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.801789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.801791:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513077.801799:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.801806:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513077.801809:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513077.801815:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.801821:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6c38 00000800:00000200:14.0:1616513077.801825:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.801827:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.802445:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513077.802456:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.802464:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513077.802466:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513077.802473:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.143@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.802482:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.143@tcp1 of length 224 into portal 12 MB=0x6002a01115600 00000400:00000200:2.0:1616513077.802488:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.143@tcp1 of length 224/224 into md 0xf96450b9 [1] + 72032 00000400:00000200:2.0:1616513077.802494:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.802496:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513077.802503:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.802506:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.802508:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.802564:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689030266803712 00010000:00000200:26.0:1616513077.802576:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464625100 x1689030266803712/t0(0) o400->d7913ba8-3cb4-2fd1-f45d-28df3e6c2e72@10.31.142.143@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.802600:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689030266803712, offset 192 00000400:00000200:26.0:1616513077.802606:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.143@tcp1 00000400:00000200:26.0:1616513077.802613:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.143@tcp1 routed destination 00000400:00000200:26.0:1616513077.802627:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.143@tcp1(10.31.142.143@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.802633:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.802639:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.802641:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.802643:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.802648:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.802694:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.802750:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.802757:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.802760:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513077.802766:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.802770:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae000 00000800:00000200:28.0:1616513077.802774:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.802777:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.823238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.823295:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.823316:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.823324:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.155@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.823333:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.155@tcp1 of length 224 into portal 12 MB=0x60182aa9f9280 00000400:00000200:14.0:1616513077.823339:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.155@tcp1 of length 224/224 into md 0xf96450b9 [1] + 72256 00000400:00000200:14.0:1616513077.823344:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.823347:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513077.823353:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.823353:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.823356:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.823359:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.823412:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510580224640 00010000:00000200:26.0:1616513077.823422:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba1680 x1690510580224640/t0(0) o400->1fbe63ca-2e7e-d284-d0f2-c48a41d00ff4@10.31.138.155@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.823447:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510580224640, offset 192 00000800:00000200:11.2:1616513077.823449:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513077.823452:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.155@tcp1 00000400:00000200:26.0:1616513077.823459:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.155@tcp1 routed destination 00000400:00000200:26.0:1616513077.823472:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.155@tcp1(10.31.138.155@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.823478:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513077.823483:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.823485:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513077.823487:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.823491:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.823497:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513077.823509:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.823518:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513077.823527:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.155@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.823535:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.155@tcp1 of length 224 into portal 26 MB=0x60182aa9f91c0 00000400:00000200:24.0:1616513077.823543:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.155@tcp1 of length 224/224 into md 0xf97c74b9 [1] + 0 00000400:00000200:24.0:1616513077.823547:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b784c8 00000400:00000200:24.0:1616513077.823550:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.823553:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513077.823558:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b784c8 00000800:00000200:24.0:1616513077.823561:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.823563:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513077.823565:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513077.823568:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513077.823568:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513077.823575:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513077.823578:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513077.823581:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513077.823584:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513077.823588:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af760 00000800:00000200:22.0:1616513077.823592:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.823623:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510580224448 00010000:00000200:15.0:1616513077.823634:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1690510580224448/t0(0) o400->2a4e814c-df6b-0054-fdf9-ca12731f1223@10.31.138.155@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.823648:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510580224448, offset 192 00000400:00000200:15.0:1616513077.823654:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.155@tcp1 00000400:00000200:15.0:1616513077.823663:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.155@tcp1 routed destination 00000400:00000200:15.0:1616513077.823678:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.155@tcp1(10.31.138.155@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.823685:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.823691:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.823694:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.823696:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.823701:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.823706:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.823708:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513077.823761:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.823768:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.823771:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513077.823774:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513077.823778:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.823785:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7e48 00000800:00000200:28.0:1616513077.823788:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.826606:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513077.826664:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.826674:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513077.826683:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.59@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.826692:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.59@tcp1 of length 224 into portal 26 MB=0x5ffdec91f5480 00000400:00000200:24.0:1616513077.826699:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.59@tcp1 of length 224/224 into md 0xf97c74c9 [1] + 0 00000400:00000200:24.0:1616513077.826704:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79540 00000400:00000200:24.0:1616513077.826707:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.826709:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513077.826714:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79540 00000800:00000200:24.0:1616513077.826717:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.826719:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513077.826720:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513077.826726:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513077.826729:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.826730:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.826739:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.59@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.826747:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.59@tcp1 of length 224 into portal 12 MB=0x5ffdec91f5500 00000400:00000200:20.0:1616513077.826754:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.59@tcp1 of length 224/224 into md 0x865bbec1 [1] + 6048 00000400:00000200:20.0:1616513077.826759:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.826762:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513077.826768:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.826772:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.826774:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.826779:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707205649536 00010000:00000200:15.0:1616513077.826790:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688707205649536/t0(0) o400->3442cd57-12a5-f527-f235-5bbc1d7ad7c1@10.31.140.59@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.826805:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688707205649536, offset 192 00000400:00000200:15.0:1616513077.826813:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.59@tcp1 00000400:00000200:15.0:1616513077.826821:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.59@tcp1 routed destination 00000100:00000200:0.0:1616513077.826830:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707205649664 00000400:00000200:15.0:1616513077.826836:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.59@tcp1(10.31.140.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.826843:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:0.0:1616513077.826843:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07980 x1688707205649664/t0(0) o400->72f35e61-026a-cb6b-b02b-e038b49403bb@10.31.140.59@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.826849:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.826851:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.826854:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513077.826856:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688707205649664, offset 192 00000800:00000200:15.0:1616513077.826859:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513077.826862:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.59@tcp1 00000800:00000200:7.2:1616513077.826864:0:49208:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.826866:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513077.826869:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.59@tcp1 routed destination 00000400:00000200:0.0:1616513077.826885:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.59@tcp1(10.31.140.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.826891:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.826896:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.826899:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.826901:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.826906:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.826911:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.826918:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.826925:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513077.826928:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513077.826935:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513077.826941:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c73a8 00000800:00000200:18.0:1616513077.826944:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.826947:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513077.826964:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.826971:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.826974:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513077.826974:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513077.826980:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.826984:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05298 00000800:00000200:28.0:1616513077.826989:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.835949:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513077.836012:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.836029:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.836040:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.56@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.836049:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.56@tcp1 of length 224 into portal 26 MB=0x5ffd4db4c1280 00000400:00000200:2.0:1616513077.836056:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.56@tcp1 of length 224/224 into md 0xf97c74f9 [1] + 0 00000400:00000200:2.0:1616513077.836060:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78a18 00000400:00000200:2.0:1616513077.836063:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.836066:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513077.836071:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78a18 00000800:00000200:2.0:1616513077.836074:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.836075:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.836077:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513077.836081:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.836085:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513077.836085:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513077.836094:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.56@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.836103:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.56@tcp1 of length 224 into portal 12 MB=0x5ffd4db4c1300 00000400:00000200:22.0:1616513077.836109:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.56@tcp1 of length 224/224 into md 0xf96450b9 [1] + 72480 00000400:00000200:22.0:1616513077.836114:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.836117:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.836124:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.836128:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.836129:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.836135:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664560898688 00010000:00000200:15.0:1616513077.836146:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688664560898688/t0(0) o400->01932d3f-136b-7eb5-1876-e7ed2f2e9e13@10.31.138.56@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.836160:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664560898688, offset 192 00000400:00000200:15.0:1616513077.836167:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.56@tcp1 00000400:00000200:15.0:1616513077.836176:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.56@tcp1 routed destination 00000100:00000200:26.0:1616513077.836191:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664560898816 00000400:00000200:15.0:1616513077.836191:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.56@tcp1(10.31.138.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.836198:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:26.0:1616513077.836203:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec200480 x1688664560898816/t0(0) o400->879ad243-79ea-e08a-9a44-dcdfca0eed93@10.31.138.56@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.836204:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.836207:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.836209:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.836214:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:26.0:1616513077.836218:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664560898816, offset 192 00000800:00000200:7.2:1616513077.836219:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.836221:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513077.836224:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.56@tcp1 00000400:00000200:26.0:1616513077.836232:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.56@tcp1 routed destination 00000400:00000200:26.0:1616513077.836248:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.56@tcp1(10.31.138.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.836256:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.836263:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.836265:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.836267:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513077.836270:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513077.836272:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513077.836277:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513077.836280:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513077.836281:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513077.836287:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.836292:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7d38 00000800:00000200:14.0:1616513077.836296:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.836318:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.836374:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.836389:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.836392:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.836399:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.836403:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afd38 00000800:00000200:16.0:1616513077.836407:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.836410:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.840341:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513077.840400:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.840422:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513077.840424:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513077.840430:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.160@tcp1 : PUT - for me 00000400:00000200:24.0:1616513077.840451:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.160@tcp1 of length 224 into portal 26 MB=0x6041916de1e40 00000400:00000200:24.0:1616513077.840457:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.160@tcp1 of length 224/224 into md 0xf97c7509 [1] + 0 00000400:00000200:24.0:1616513077.840462:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b79c28 00000400:00000200:24.0:1616513077.840465:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.840468:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513077.840472:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b79c28 00000800:00000200:24.0:1616513077.840475:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.840477:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.840479:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.840534:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693355664612928 00010000:00000200:15.0:1616513077.840545:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1693355664612928/t0(0) o400->188ccebc-516f-f2dd-9382-38529c120ec2@10.31.129.160@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.840558:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693355664612928, offset 192 00000400:00000200:15.0:1616513077.840564:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.160@tcp1 00000400:00000200:15.0:1616513077.840571:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.160@tcp1 routed destination 00000400:00000200:15.0:1616513077.840586:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.160@tcp1(10.31.129.160@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.840592:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.840600:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.840602:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.840604:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.840609:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513077.840615:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.840652:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.840708:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.840715:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513077.840718:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513077.840725:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513077.840731:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7ed0 00000800:00000200:18.0:1616513077.840735:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.840738:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.840743:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.840751:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.160@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.840759:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.160@tcp1 of length 224 into portal 12 MB=0x6041916de2240 00000800:00000200:22.0:1616513077.840761:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513077.840765:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.160@tcp1 of length 224/224 into md 0xf96450b9 [1] + 72704 00000400:00000200:18.0:1616513077.840770:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.840772:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513077.840778:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.840780:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.840782:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.840838:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693355664613952 00010000:00000200:26.0:1616513077.840848:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb8000 x1693355664613952/t0(0) o400->fb51cf43-7dc7-3389-7d92-526e5002de21@10.31.129.160@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.840861:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693355664613952, offset 192 00000400:00000200:26.0:1616513077.840866:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.160@tcp1 00000400:00000200:26.0:1616513077.840873:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.160@tcp1 routed destination 00000400:00000200:26.0:1616513077.840886:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.160@tcp1(10.31.129.160@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.840892:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.840897:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.840900:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.840901:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.840905:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.840911:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.840964:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.840971:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.840973:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.840980:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.840984:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeee0 00000800:00000200:16.0:1616513077.840988:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.840991:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.846161:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513077.846213:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513077.846215:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.846225:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.846233:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.212@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.846243:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.212@tcp1 of length 224 into portal 26 MB=0x5ffe251c59040 00000400:00000200:14.0:1616513077.846250:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.212@tcp1 of length 224/224 into md 0xf97c7519 [1] + 0 00000400:00000200:14.0:1616513077.846255:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8ee1b78b28 00000400:00000200:14.0:1616513077.846258:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.846260:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513077.846265:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8ee1b78b28 00000800:00000200:14.0:1616513077.846268:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513077.846269:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513077.846271:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.846273:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.846275:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513077.846279:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.846287:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.212@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.846302:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.212@tcp1 of length 224 into portal 12 MB=0x5ffe251c59300 00000400:00000200:28.0:1616513077.846308:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.212@tcp1 of length 224/224 into md 0x865bbec1 [1] + 6272 00000400:00000200:28.0:1616513077.846314:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.846316:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513077.846322:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.846323:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.846326:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513077.846328:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.846329:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688722383147072 00010000:00000200:15.0:1616513077.846340:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688722383147072/t0(0) o400->b18206e6-850d-5a04-b3cb-c95d914cfac9@10.31.142.212@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513077.846349:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688722383147776 00000100:00000200:15.0:1616513077.846353:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688722383147072, offset 192 00000400:00000200:15.0:1616513077.846359:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.212@tcp1 00010000:00000200:0.0:1616513077.846362:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6720480 x1688722383147776/t0(0) o400->fe5c02c4-c70f-41ab-d875-2a0861cfaf55@10.31.142.212@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513077.846367:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.212@tcp1 routed destination 00000100:00000200:0.0:1616513077.846376:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688722383147776, offset 192 00000400:00000200:15.0:1616513077.846381:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.212@tcp1(10.31.142.212@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513077.846383:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.212@tcp1 00000800:00000200:15.0:1616513077.846387:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513077.846391:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.212@tcp1 routed destination 00000800:00000200:15.0:1616513077.846393:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.846396:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.846398:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.846402:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513077.846406:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.212@tcp1(10.31.142.212@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513077.846407:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.846409:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513077.846413:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.846419:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.846422:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.846424:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.846429:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.846437:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.846460:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.846467:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513077.846471:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513077.846477:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513077.846483:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c73a8 00000800:00000200:18.0:1616513077.846487:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.846490:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513077.846492:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.846499:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.846502:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.846508:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.846512:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04198 00000800:00000200:16.0:1616513077.846516:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.846519:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.847405:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.847416:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.847422:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.847430:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.77@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.847452:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.77@tcp1 of length 224 into portal 26 MB=0x601832739b740 00000400:00000200:14.0:1616513077.847457:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.77@tcp1 of length 224/224 into md 0xf97c7539 [1] + 0 00000400:00000200:14.0:1616513077.847461:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04000 00000400:00000200:14.0:1616513077.847463:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.847466:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513077.847470:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04000 00000800:00000200:14.0:1616513077.847472:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.847474:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.847476:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.847478:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.847484:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512670701376 00000800:00000200:22.0:1616513077.847485:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.847485:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:15.0:1616513077.847490:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1690512670701376/t0(0) o400->8d93cd5b-9850-4d3c-5960-3b02dd221f89@10.31.138.77@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513077.847492:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.77@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.847498:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.77@tcp1 of length 224 into portal 12 MB=0x601832739b7c0 00000100:00000200:15.0:1616513077.847502:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512670701376, offset 192 00000400:00000200:14.0:1616513077.847502:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.77@tcp1 of length 224/224 into md 0xf96450b9 [1] + 72928 00000400:00000200:14.0:1616513077.847506:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:15.0:1616513077.847508:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.77@tcp1 00000100:00000200:14.0:1616513077.847508:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513077.847513:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:15.0:1616513077.847514:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.77@tcp1 routed destination 00000800:00000200:14.0:1616513077.847515:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513077.847517:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513077.847528:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.77@tcp1(10.31.138.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.847534:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.847539:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.847541:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.847543:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.847547:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.847551:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.847553:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513077.847564:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.847571:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:26.0:1616513077.847575:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512670701504 00000800:00000200:22.0:1616513077.847575:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513077.847575:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.847581:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:26.0:1616513077.847586:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba1f80 x1690512670701504/t0(0) o400->b49a1535-39ec-02e4-e70b-a7ba7578d742@10.31.138.77@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513077.847588:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6c38 00000800:00000200:2.0:1616513077.847591:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.847598:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512670701504, offset 192 00000400:00000200:26.0:1616513077.847603:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.77@tcp1 00000400:00000200:26.0:1616513077.847611:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.77@tcp1 routed destination 00000400:00000200:26.0:1616513077.847624:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.77@tcp1(10.31.138.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.847629:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.847635:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.847637:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.847639:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.847643:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.847689:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.847752:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.847758:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513077.847761:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513077.847767:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513077.847771:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af8f8 00000800:00000200:24.0:1616513077.847774:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.847777:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.863853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.863913:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.863934:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513077.863943:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.82@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.863951:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.82@tcp1 of length 224 into portal 26 MB=0x603e51c7a3440 00000400:00000200:16.0:1616513077.863958:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.82@tcp1 of length 224/224 into md 0xf97c7559 [1] + 0 00000400:00000200:16.0:1616513077.863963:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05b18 00000400:00000200:16.0:1616513077.863965:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.863968:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513077.863972:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513077.863974:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:16.0:1616513077.863977:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.863979:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513077.863981:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.863984:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.863985:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.863993:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.82@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.864002:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.82@tcp1 of length 224 into portal 12 MB=0x603e51c7a34c0 00000400:00000200:20.0:1616513077.864008:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.82@tcp1 of length 224/224 into md 0x865bbec1 [1] + 6496 00000400:00000200:20.0:1616513077.864014:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.864016:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513077.864023:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.864026:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.864028:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.864034:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693132420428864 00010000:00000200:15.0:1616513077.864043:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1693132420428864/t0(0) o400->22cd91a9-fe9d-3132-6e5e-80a5c383b899@10.31.133.82@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.864056:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693132420428864, offset 192 00000400:00000200:15.0:1616513077.864061:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.82@tcp1 00000400:00000200:15.0:1616513077.864069:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.82@tcp1 routed destination 00000400:00000200:15.0:1616513077.864082:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.82@tcp1(10.31.133.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513077.864086:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693132420428992 00000800:00000200:15.0:1616513077.864088:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.864094:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.864096:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.864098:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513077.864098:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07080 x1693132420428992/t0(0) o400->f5bb4f7e-4dc4-7a09-b861-7361018ba59b@10.31.133.82@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.864102:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.864108:0:49212:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.864109:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513077.864111:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693132420428992, offset 192 00000400:00000200:0.0:1616513077.864117:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.82@tcp1 00000800:00000200:18.0:1616513077.864121:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513077.864124:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.82@tcp1 routed destination 00000800:00000200:18.0:1616513077.864128:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513077.864131:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513077.864137:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:0.0:1616513077.864139:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.82@tcp1(10.31.133.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513077.864143:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7760 00000800:00000200:0.0:1616513077.864145:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513077.864146:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.864149:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513077.864150:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.864153:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.864157:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.864161:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.864167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.864221:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.864228:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513077.864231:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513077.864232:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513077.864237:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.864241:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb047f8 00000800:00000200:28.0:1616513077.864245:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:23.0:1616513077.870601:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:23.0:1616513077.870609:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:7.2:1616513077.871586:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.871598:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.871608:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.871617:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.76@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.871627:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.76@tcp1 of length 224 into portal 26 MB=0x5ffd52bbc0000 00000400:00000200:22.0:1616513077.871634:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.76@tcp1 of length 224/224 into md 0xf97c7579 [1] + 0 00000400:00000200:22.0:1616513077.871639:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04220 00000400:00000200:22.0:1616513077.871642:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.871645:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.871649:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:22.0:1616513077.871652:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.871653:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.871655:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.871660:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.871663:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.871664:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.871672:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.76@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.871681:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.76@tcp1 of length 224 into portal 12 MB=0x5ffd52bbc0140 00000400:00000200:2.0:1616513077.871687:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.76@tcp1 of length 224/224 into md 0xf96450b9 [1] + 73152 00000400:00000200:2.0:1616513077.871693:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.871695:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513077.871702:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.871705:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.871707:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.871712:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665910411264 00010000:00000200:15.0:1616513077.871722:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688665910411264/t0(0) o400->32d5e6b4-472f-b172-6658-5fcae5819bc3@10.31.138.76@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.871735:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665910411264, offset 192 00000400:00000200:15.0:1616513077.871741:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.76@tcp1 00000400:00000200:15.0:1616513077.871749:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.76@tcp1 routed destination 00000400:00000200:15.0:1616513077.871764:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.76@tcp1(10.31.138.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513077.871767:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665910411584 00000800:00000200:15.0:1616513077.871770:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.871776:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.871778:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:26.0:1616513077.871779:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464627080 x1688665910411584/t0(0) o400->d7e86985-8c15-0304-1cd4-cee205e5d12a@10.31.138.76@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.871780:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.871785:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.871789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.871792:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513077.871794:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665910411584, offset 192 00000400:00000200:26.0:1616513077.871801:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.76@tcp1 00000400:00000200:26.0:1616513077.871809:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.76@tcp1 routed destination 00000400:00000200:26.0:1616513077.871824:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.76@tcp1(10.31.138.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.871847:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.871852:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.871854:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.871856:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.871860:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513077.871863:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.871870:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.871873:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513077.871873:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513077.871880:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.871885:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6ee0 00000800:00000200:14.0:1616513077.871889:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.871906:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.871962:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.871968:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.871971:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.871978:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.871982:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae7f8 00000800:00000200:20.0:1616513077.871986:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.871989:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.875120:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513077.875176:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.875185:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513077.875194:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.196@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.875204:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.196@tcp1 of length 224 into portal 26 MB=0x604a299c81d40 00000400:00000200:16.0:1616513077.875211:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.196@tcp1 of length 224/224 into md 0xf97c7589 [1] + 0 00000400:00000200:16.0:1616513077.875217:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05430 00000400:00000200:16.0:1616513077.875219:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.875222:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513077.875227:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05430 00000800:00000200:16.0:1616513077.875230:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513077.875232:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.875233:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.875239:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.875242:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.875243:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.875252:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.196@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.875261:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.196@tcp1 of length 224 into portal 12 MB=0x604a299c81dc0 00000400:00000200:28.0:1616513077.875267:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.196@tcp1 of length 224/224 into md 0x865bbec1 [1] + 6720 00000400:00000200:28.0:1616513077.875273:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.875275:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513077.875282:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.875285:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.875287:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.875303:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946271505728 00010000:00000200:15.0:1616513077.875313:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1693946271505728/t0(0) o400->4565f98e-e283-edd1-0e20-691e79f2effd@10.31.140.196@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.875327:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946271505728, offset 192 00000400:00000200:15.0:1616513077.875334:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.196@tcp1 00000400:00000200:15.0:1616513077.875342:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.196@tcp1 routed destination 00000100:00000200:0.0:1616513077.875347:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946271505856 00000400:00000200:15.0:1616513077.875356:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.196@tcp1(10.31.140.196@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513077.875359:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6721f80 x1693946271505856/t0(0) o400->8c5cb75b-cf4d-d235-16e7-390ef0a31c08@10.31.140.196@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.875362:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.875368:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.875371:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513077.875372:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946271505856, offset 192 00000800:00000200:15.0:1616513077.875373:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.875378:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513077.875378:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.196@tcp1 00000800:00000200:7.2:1616513077.875382:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.875384:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513077.875385:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.196@tcp1 routed destination 00000400:00000200:0.0:1616513077.875400:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.196@tcp1(10.31.140.196@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.875406:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.875412:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.875414:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.875416:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.875421:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.875426:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513077.875433:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.875450:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513077.875453:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513077.875460:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513077.875465:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c64c8 00000800:00000200:18.0:1616513077.875469:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.875472:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513077.875479:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.875486:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513077.875489:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513077.875491:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513077.875496:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513077.875500:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04b28 00000800:00000200:24.0:1616513077.875504:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.901782:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.901841:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.901851:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.901860:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.140@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.901869:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.140@tcp1 of length 224 into portal 26 MB=0x60195330060c0 00000400:00000200:20.0:1616513077.901876:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.140@tcp1 of length 224/224 into md 0xf97c7599 [1] + 0 00000400:00000200:20.0:1616513077.901881:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04088 00000400:00000200:20.0:1616513077.901884:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.901887:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513077.901892:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04088 00000800:00000200:28.0:1616513077.901895:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.901895:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.901898:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.901900:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.901964:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590177681600 00010000:00000200:15.0:1616513077.901974:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1690590177681600/t0(0) o400->c51b5703-5efe-a7e9-d90d-75ee66aae609@10.31.141.140@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.901986:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690590177681600, offset 192 00000400:00000200:15.0:1616513077.901992:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.140@tcp1 00000400:00000200:15.0:1616513077.901999:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.140@tcp1 routed destination 00000400:00000200:15.0:1616513077.902012:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.140@tcp1(10.31.141.140@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.902018:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.902024:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.902027:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.902029:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.902033:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513077.902039:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.902040:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.902095:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.902103:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513077.902106:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513077.902113:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513077.902119:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6d48 00000800:00000200:22.0:1616513077.902123:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.902126:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.902132:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.902140:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.140@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.902148:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.140@tcp1 of length 224 into portal 12 MB=0x6019533006480 00000800:00000200:14.0:1616513077.902150:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513077.902154:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.140@tcp1 of length 224/224 into md 0xf96450b9 [1] + 73376 00000400:00000200:22.0:1616513077.902160:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.902162:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513077.902169:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.902171:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513077.902173:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.902236:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590177682560 00010000:00000200:26.0:1616513077.902249:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec200000 x1690590177682560/t0(0) o400->fc18fffc-6bb2-c4db-d47c-6d4ec8b49a02@10.31.141.140@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.902264:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690590177682560, offset 192 00000400:00000200:26.0:1616513077.902270:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.140@tcp1 00000400:00000200:26.0:1616513077.902278:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.140@tcp1 routed destination 00000400:00000200:26.0:1616513077.902294:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.140@tcp1(10.31.141.140@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.902300:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.902306:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.902309:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.902311:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.902316:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.902321:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513077.902377:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.902384:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513077.902386:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513077.902393:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513077.902397:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af320 00000800:00000200:16.0:1616513077.902401:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513077.902404:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.907057:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513077.907111:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513077.907120:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513077.907129:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.186@tcp1 : PUT - for me 00000400:00000200:28.0:1616513077.907138:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.186@tcp1 of length 224 into portal 26 MB=0x5fff356420000 00000400:00000200:28.0:1616513077.907145:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.186@tcp1 of length 224/224 into md 0xf97c75b9 [1] + 0 00000400:00000200:28.0:1616513077.907150:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04a18 00000400:00000200:28.0:1616513077.907152:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513077.907157:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513077.907162:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:28.0:1616513077.907165:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.907165:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.907168:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513077.907170:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.907227:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795472855040 00010000:00000200:15.0:1616513077.907238:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688795472855040/t0(0) o400->e6ad0d3b-e5a5-c651-64b4-345d704569b8@10.31.131.186@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.907252:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688795472855040, offset 192 00000800:00000200:11.2:1616513077.907253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:15.0:1616513077.907259:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.186@tcp1 00000400:00000200:15.0:1616513077.907267:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.186@tcp1 routed destination 00000400:00000200:15.0:1616513077.907282:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.186@tcp1(10.31.131.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.907293:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.907300:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.907303:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.907305:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513077.907306:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.907310:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513077.907315:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:15.0:1616513077.907316:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.907316:0:49216:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513077.907324:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.186@tcp1 : PUT - for me 00000400:00000200:16.0:1616513077.907333:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.186@tcp1 of length 224 into portal 12 MB=0x5fff3564200c0 00000400:00000200:16.0:1616513077.907339:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.186@tcp1 of length 224/224 into md 0x865bbec1 [1] + 6944 00000400:00000200:16.0:1616513077.907344:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513077.907349:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513077.907356:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513077.907359:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513077.907360:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513077.907364:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513077.907383:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.907390:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.907393:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.907400:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.907406:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6088 00000800:00000200:2.0:1616513077.907411:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.907414:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513077.907450:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795472855232 00010000:00000200:0.0:1616513077.907462:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1680 x1688795472855232/t0(0) o400->f785a59f-ed71-6e4e-60aa-d74f848479c1@10.31.131.186@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513077.907474:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688795472855232, offset 192 00000400:00000200:0.0:1616513077.907479:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.186@tcp1 00000400:00000200:0.0:1616513077.907486:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.186@tcp1 routed destination 00000400:00000200:0.0:1616513077.907500:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.186@tcp1(10.31.131.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513077.907505:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513077.907511:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513077.907513:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513077.907515:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513077.907519:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.907524:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.907534:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.907540:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.907543:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513077.907545:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513077.907549:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.907555:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05c28 00000800:00000200:20.0:1616513077.907559:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.930855:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.930911:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.930930:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.930939:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.96@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.930950:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.96@tcp1 of length 224 into portal 26 MB=0x5ffdcebf6c840 00000400:00000200:22.0:1616513077.930957:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.96@tcp1 of length 224/224 into md 0xf97c75d9 [1] + 0 00000400:00000200:22.0:1616513077.930961:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05f58 00000400:00000200:22.0:1616513077.930964:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.930966:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513077.930970:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513077.930971:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05f58 00000800:00000200:22.0:1616513077.930974:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513077.930976:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513077.930978:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.930981:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.930981:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.930989:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.96@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.930998:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.96@tcp1 of length 224 into portal 12 MB=0x5ffdcebf6c8c0 00000400:00000200:18.0:1616513077.931005:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.96@tcp1 of length 224/224 into md 0xf96450b9 [1] + 73600 00000400:00000200:18.0:1616513077.931010:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.931013:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513077.931020:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.931023:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.931025:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.931038:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699200260160 00010000:00000200:15.0:1616513077.931050:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688699200260160/t0(0) o400->62aae506-6222-ee27-d690-51998a52c69b@10.31.138.96@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.931064:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699200260160, offset 192 00000400:00000200:15.0:1616513077.931070:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.96@tcp1 00000400:00000200:15.0:1616513077.931078:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.96@tcp1 routed destination 00000100:00000200:26.0:1616513077.931084:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699200260288 00000400:00000200:15.0:1616513077.931094:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.96@tcp1(10.31.138.96@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:26.0:1616513077.931097:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb8900 x1688699200260288/t0(0) o400->b721ae50-5685-2502-165e-325b0a6a7bee@10.31.138.96@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513077.931100:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.931107:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.931110:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:26.0:1616513077.931112:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699200260288, offset 192 00000800:00000200:15.0:1616513077.931112:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.931116:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513077.931118:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.96@tcp1 00000800:00000200:7.2:1616513077.931121:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.931124:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513077.931126:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.96@tcp1 routed destination 00000400:00000200:26.0:1616513077.931143:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.96@tcp1(10.31.138.96@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.931150:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.931156:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.931159:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.931161:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.931166:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.931173:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513077.931174:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.931182:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.931185:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513077.931185:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513077.931192:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513077.931198:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c64c8 00000800:00000200:14.0:1616513077.931202:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513077.931228:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.931234:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513077.931237:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513077.931244:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513077.931248:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af980 00000800:00000200:24.0:1616513077.931252:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.931255:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.958231:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513077.958288:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513077.958298:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513077.958307:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.17@tcp1 : PUT - for me 00000400:00000200:2.0:1616513077.958316:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.17@tcp1 of length 224 into portal 26 MB=0x5ffe22e0f50c0 00000400:00000200:2.0:1616513077.958324:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.17@tcp1 of length 224/224 into md 0xf97c75f9 [1] + 0 00000400:00000200:2.0:1616513077.958329:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05078 00000400:00000200:2.0:1616513077.958333:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513077.958336:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513077.958341:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05078 00000800:00000200:18.0:1616513077.958344:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.958344:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.958347:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513077.958352:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513077.958355:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513077.958355:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513077.958364:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.17@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.958375:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.17@tcp1 of length 224 into portal 12 MB=0x5ffe22e0f5380 00000400:00000200:18.0:1616513077.958382:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.17@tcp1 of length 224/224 into md 0xf96450b9 [1] + 73824 00000400:00000200:18.0:1616513077.958387:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.958390:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513077.958397:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.958400:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.958402:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.958407:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688721784000704 00010000:00000200:15.0:1616513077.958418:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688721784000704/t0(0) o400->77d03434-d4bd-ec9c-ed46-ff41edaadfb0@10.31.140.17@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.958432:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688721784000704, offset 192 00000400:00000200:15.0:1616513077.958452:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.17@tcp1 00000400:00000200:15.0:1616513077.958459:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.17@tcp1 routed destination 00000100:00000200:26.0:1616513077.958462:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688721784001408 00010000:00000200:26.0:1616513077.958475:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebda00 x1688721784001408/t0(0) o400->e0a41f4d-c105-200c-c254-91e1dfa4bac4@10.31.140.17@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513077.958475:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.17@tcp1(10.31.140.17@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.958482:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.958489:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:26.0:1616513077.958490:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688721784001408, offset 192 00000800:00000200:15.0:1616513077.958492:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.958496:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:26.0:1616513077.958497:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.17@tcp1 00000800:00000200:15.0:1616513077.958501:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513077.958505:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.17@tcp1 routed destination 00000800:00000200:7.2:1616513077.958506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.958509:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513077.958520:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.17@tcp1(10.31.140.17@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.958527:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.958533:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.958536:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.958538:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.958543:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.958551:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513077.958560:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.958568:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513077.958569:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513077.958571:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513077.958572:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513077.958576:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513077.958578:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513077.958579:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513077.958584:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7760 00000400:00000200:28.0:1616513077.958586:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513077.958588:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513077.958590:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae6e8 00000800:00000200:28.0:1616513077.958594:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513077.958597:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.961520:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513077.961572:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513077.961582:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513077.961591:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.100@tcp1 : PUT - for me 00000400:00000200:22.0:1616513077.961599:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.100@tcp1 of length 224 into portal 26 MB=0x60004c07c1500 00000400:00000200:22.0:1616513077.961606:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.100@tcp1 of length 224/224 into md 0xf97c7609 [1] + 0 00000400:00000200:22.0:1616513077.961611:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05ed0 00000400:00000200:22.0:1616513077.961614:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513077.961617:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513077.961621:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:22.0:1616513077.961624:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513077.961627:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513077.961627:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513077.961632:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.961685:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688870269490432 00010000:00000200:15.0:1616513077.961696:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688870269490432/t0(0) o400->ee6428df-93be-05be-2222-f4208fae0932@10.31.138.100@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.961711:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688870269490432, offset 192 00000400:00000200:15.0:1616513077.961717:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.100@tcp1 00000400:00000200:15.0:1616513077.961725:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.100@tcp1 routed destination 00000400:00000200:15.0:1616513077.961740:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.100@tcp1(10.31.138.100@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.961747:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.961753:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.961756:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.961758:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.961763:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513077.961767:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513077.961770:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513077.961822:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513077.961829:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513077.961832:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513077.961839:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513077.961845:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c73a8 00000800:00000200:2.0:1616513077.961849:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513077.961852:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513077.961862:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513077.961873:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.961880:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.961881:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.961890:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.100@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.961898:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.100@tcp1 of length 224 into portal 12 MB=0x60004c07c1640 00000400:00000200:18.0:1616513077.961905:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.100@tcp1 of length 224/224 into md 0xf96450b9 [1] + 74048 00000400:00000200:18.0:1616513077.961910:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.961913:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513077.961919:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.961922:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.961924:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.961984:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688870269490752 00010000:00000200:26.0:1616513077.961996:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9200 x1688870269490752/t0(0) o400->2da0006f-fe94-259e-c8da-b2111a59d063@10.31.138.100@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.962011:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688870269490752, offset 192 00000400:00000200:26.0:1616513077.962018:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.100@tcp1 00000400:00000200:26.0:1616513077.962025:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.100@tcp1 routed destination 00000400:00000200:26.0:1616513077.962040:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.100@tcp1(10.31.138.100@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.962047:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.962053:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.962055:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.962059:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.962064:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.962071:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.962127:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.962135:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513077.962138:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513077.962144:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513077.962148:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae220 00000800:00000200:24.0:1616513077.962152:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.962155:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513077.962161:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513077.962163:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.962165:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513077.962167:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.962169:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513077.983902:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513077.983957:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.983967:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513077.983975:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.97@tcp1 : PUT - for me 00000400:00000200:20.0:1616513077.983985:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.97@tcp1 of length 224 into portal 26 MB=0x5ffdcca8fe6c0 00000400:00000200:20.0:1616513077.983993:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.97@tcp1 of length 224/224 into md 0xf97c7629 [1] + 0 00000400:00000200:20.0:1616513077.983997:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05e48 00000400:00000200:20.0:1616513077.984000:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.984003:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513077.984008:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05e48 00000800:00000200:24.0:1616513077.984011:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.984011:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.984014:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513077.984016:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513077.984073:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698639869632 00010000:00000200:15.0:1616513077.984083:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688698639869632/t0(0) o400->a344fab1-3a52-9736-ac08-8a95339fb151@10.31.138.97@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.984098:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698639869632, offset 192 00000400:00000200:15.0:1616513077.984104:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.97@tcp1 00000400:00000200:15.0:1616513077.984112:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.97@tcp1 routed destination 00000400:00000200:15.0:1616513077.984127:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.97@tcp1(10.31.138.97@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.984133:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513077.984139:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.984142:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513077.984144:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513077.984149:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513077.984155:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513077.984155:0:49226:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513077.985455:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513077.985461:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513077.985464:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513077.985471:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513077.985476:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c73a8 00000800:00000200:30.0:1616513077.985480:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513077.985483:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513077.985489:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513077.985496:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.97@tcp1 : PUT - for me 00000400:00000200:30.0:1616513077.985504:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.97@tcp1 of length 224 into portal 12 MB=0x5ffdcca8fe7c0 00000800:00000200:14.0:1616513077.985506:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513077.985510:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.97@tcp1 of length 224/224 into md 0xf96450b9 [1] + 74272 00000400:00000200:30.0:1616513077.985515:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513077.985517:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513077.985523:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513077.985525:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513077.985526:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.985585:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698639869888 00010000:00000200:26.0:1616513077.985597:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a79200 x1688698639869888/t0(0) o400->85715015-38da-8cd1-32ba-1b5be63ee997@10.31.138.97@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.985611:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698639869888, offset 192 00000400:00000200:26.0:1616513077.985616:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.97@tcp1 00000400:00000200:26.0:1616513077.985624:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.97@tcp1 routed destination 00000400:00000200:26.0:1616513077.985639:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.97@tcp1(10.31.138.97@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.985645:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.985651:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.985653:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513077.985655:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513077.985660:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.985666:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513077.985719:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.985726:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.985728:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.985735:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.985739:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae770 00000800:00000200:20.0:1616513077.985743:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.985746:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.994556:0:49227:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513077.994614:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513077.994623:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513077.994632:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.43@tcp1 : PUT - for me 00000400:00000200:18.0:1616513077.994642:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.43@tcp1 of length 224 into portal 12 MB=0x5ffd427eab580 00000400:00000200:18.0:1616513077.994648:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.43@tcp1 of length 224/224 into md 0xf96450b9 [1] + 74496 00000400:00000200:18.0:1616513077.994653:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513077.994656:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513077.994663:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513077.994666:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.994669:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513077.994673:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513077.994727:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661551396224 00000800:00000200:11.2:1616513077.994732:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:26.0:1616513077.994740:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9b00 x1688661551396224/t0(0) o400->898675e6-5bf9-b591-3d5f-a6d192f38232@10.31.138.43@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.994754:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661551396224, offset 192 00000400:00000200:26.0:1616513077.994760:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.43@tcp1 00000400:00000200:26.0:1616513077.994768:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.43@tcp1 routed destination 00000800:00000200:24.0:1616513077.994793:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513077.994795:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.43@tcp1(10.31.138.43@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513077.994801:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513077.994802:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513077.994807:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.994810:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513077.994811:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.43@tcp1 : PUT - for me 00000800:00000200:26.0:1616513077.994812:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513077.994817:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513077.994820:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.43@tcp1 of length 224 into portal 26 MB=0x5ffd427eab4c0 00000800:00000200:7.2:1616513077.994822:0:49227:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513077.994827:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.43@tcp1 of length 224/224 into md 0xf97c7639 [1] + 0 00000400:00000200:24.0:1616513077.994832:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04d48 00000400:00000200:24.0:1616513077.994834:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513077.994839:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513077.994844:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000800:00000200:20.0:1616513077.994846:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.994847:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513077.994850:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513077.994851:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513077.994875:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513077.994881:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513077.994884:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513077.994889:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513077.994893:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aec38 00000800:00000200:30.0:1616513077.994896:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513077.994899:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.994907:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661551396032 00010000:00000200:15.0:1616513077.994918:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688661551396032/t0(0) o400->87f73fb1-d6c1-4ce6-d8b5-1c5772ac090f@10.31.138.43@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.994934:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661551396032, offset 192 00000400:00000200:15.0:1616513077.994940:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.43@tcp1 00000400:00000200:15.0:1616513077.994949:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.43@tcp1 routed destination 00000400:00000200:15.0:1616513077.994965:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.43@tcp1(10.31.138.43@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.994971:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.994978:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.994981:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.994983:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.994988:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.994993:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.994995:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513077.995003:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.995008:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.995011:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.995018:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.995023:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c64c8 00000800:00000200:20.0:1616513077.995027:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.995029:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513077.999550:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513077.999604:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513077.999613:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513077.999622:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.193@tcp1 : PUT - for me 00000400:00000200:14.0:1616513077.999634:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.193@tcp1 of length 224 into portal 12 MB=0x5ffd813287f80 00000400:00000200:14.0:1616513077.999641:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.193@tcp1 of length 224/224 into md 0xf96450b9 [1] + 74720 00000400:00000200:14.0:1616513077.999646:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513077.999648:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513077.999655:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513077.999658:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513077.999659:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513077.999663:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513077.999701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513077.999715:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678382993280 00010000:00000200:26.0:1616513077.999727:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba5100 x1688678382993280/t0(0) o400->4e56515a-67d4-d803-32df-596b601f2808@10.31.128.193@tcp1:703/0 lens 224/224 e 0 to 0 dl 1616513083 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513077.999740:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688678382993280, offset 192 00000400:00000200:26.0:1616513077.999748:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.193@tcp1 00000800:00000200:20.0:1616513077.999754:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513077.999756:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.193@tcp1 routed destination 00000800:00000200:20.0:1616513077.999763:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513077.999770:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.193@tcp1(10.31.128.193@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513077.999772:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.193@tcp1 : PUT - for me 00000800:00000200:26.0:1616513077.999777:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513077.999780:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.193@tcp1 of length 224 into portal 26 MB=0x5ffd813287cc0 00000800:00000200:26.0:1616513077.999782:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513077.999785:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513077.999787:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513077.999790:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.193@tcp1 of length 224/224 into md 0xf97c7659 [1] + 0 00000800:00000200:26.0:1616513077.999791:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513077.999796:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb057e8 00000800:00000200:7.2:1616513077.999796:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513077.999799:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513077.999802:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513077.999806:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513077.999806:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb057e8 00000800:00000200:20.0:1616513077.999809:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.999812:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.999813:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513077.999846:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513077.999853:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513077.999855:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513077.999861:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513077.999864:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae110 00000800:00000200:30.0:1616513077.999868:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513077.999870:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513077.999879:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678382992576 00010000:00000200:15.0:1616513077.999889:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688678382992576/t0(0) o400->9d48ce93-79d3-6570-481d-3c4d57439e02@10.31.128.193@tcp1:34/0 lens 224/224 e 0 to 0 dl 1616513169 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513077.999903:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688678382992576, offset 192 00000400:00000200:15.0:1616513077.999909:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.193@tcp1 00000400:00000200:15.0:1616513077.999917:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.193@tcp1 routed destination 00000400:00000200:15.0:1616513077.999932:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.193@tcp1(10.31.128.193@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513077.999938:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513077.999944:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513077.999947:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513077.999949:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513077.999953:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513077.999958:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513077.999959:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513077.999968:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513077.999972:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513077.999974:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513077.999979:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513077.999985:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c64c8 00000800:00000200:20.0:1616513077.999988:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513077.999990:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513078.042685:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925120, portal 4 00000100:00000200:19.0:1616513078.042692:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925120, offset 0 00000400:00000200:19.0:1616513078.042698:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:19.0:1616513078.042705:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:19.0:1616513078.042718:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513078.042725:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:19.0:1616513078.042731:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513078.042734:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:19.0:1616513078.042736:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:19.0:1616513078.042742:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513078.042787:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513078.042845:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513078.042853:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:24.0:1616513078.042856:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.042863:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:24.0:1616513078.042870:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600d80 x1693067725925120/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513078.042883:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707c28 00000800:00000200:24.0:1616513078.042888:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513078.042891:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513078.042933:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513078.042946:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513078.042955:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513078.042963:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513078.042970:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626b00 00000400:00000200:20.0:1616513078.042976:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d4ba1 [1] + 192 00000400:00000200:20.0:1616513078.042981:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.042986:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600d80 x1693067725925120/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513078.042998:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513078.043000:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513078.043003:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513078.043004:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:19.0:1616513078.043008:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600d80 x1693067725925120/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513078.043021:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600d80 x1693067725925120/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513078.043031:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706cc0 00000800:00000200:11.2:1616513078.045633:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.045708:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.045716:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.045723:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.187@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.045731:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.187@tcp1 of length 224 into portal 26 MB=0x5ffd50004cc00 00000400:00000200:24.0:1616513078.045737:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.187@tcp1 of length 224/224 into md 0xf97c7679 [1] + 0 00000400:00000200:24.0:1616513078.045741:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04110 00000400:00000200:24.0:1616513078.045744:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.045746:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.045751:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04110 00000800:00000200:24.0:1616513078.045753:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.045756:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.045757:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.045760:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.045815:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665176984576 00010000:00000200:15.0:1616513078.045826:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688665176984576/t0(0) o400->183ea4bd-36e0-f406-e906-6fba699fc6ae@10.31.131.187@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.045841:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665176984576, offset 192 00000400:00000200:15.0:1616513078.045847:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.187@tcp1 00000400:00000200:15.0:1616513078.045855:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.187@tcp1 routed destination 00000400:00000200:15.0:1616513078.045872:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.187@tcp1(10.31.131.187@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.045878:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.045885:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.045888:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.045890:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.045895:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.045901:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.045901:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513078.045955:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513078.045956:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.045964:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.045967:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.045977:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.045984:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c64c8 00000800:00000200:18.0:1616513078.045988:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.045991:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.046021:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.046030:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513078.046033:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.046039:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.187@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.046048:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.187@tcp1 of length 224 into portal 12 MB=0x5ffd50004cd80 00000400:00000200:28.0:1616513078.046055:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.187@tcp1 of length 224/224 into md 0x865bbec1 [1] + 7168 00000400:00000200:28.0:1616513078.046060:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.046063:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513078.046071:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.046074:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.046076:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.046138:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665176984960 00010000:00000200:0.0:1616513078.046152:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6722400 x1688665176984960/t0(0) o400->f1688f5c-869f-afd6-bd1c-c94de21c9dbf@10.31.131.187@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.046167:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665176984960, offset 192 00000400:00000200:0.0:1616513078.046175:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.187@tcp1 00000400:00000200:0.0:1616513078.046183:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.187@tcp1 routed destination 00000400:00000200:0.0:1616513078.046198:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.187@tcp1(10.31.131.187@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.046206:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.046212:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.046215:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.046217:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.046223:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.046229:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.046239:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.046244:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.046247:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.046248:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.046254:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.046257:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:20.0:1616513078.046262:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.052841:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.052895:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.052906:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513078.052914:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.052916:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.205@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.052924:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.205@tcp1 of length 224 into portal 26 MB=0x5ffddb99193c0 00000400:00000200:20.0:1616513078.052931:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.205@tcp1 of length 224/224 into md 0xf97c7689 [1] + 0 00000400:00000200:20.0:1616513078.052936:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05ba0 00000400:00000200:20.0:1616513078.052939:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.052942:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513078.052947:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ba0 00000800:00000200:20.0:1616513078.052950:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.052953:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.052954:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.053011:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702649734080 00010000:00000200:15.0:1616513078.053021:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688702649734080/t0(0) o400->e1c75385-8220-5dc0-a7fb-58b3280ecc9d@10.31.141.205@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.053036:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702649734080, offset 192 00000400:00000200:15.0:1616513078.053042:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.205@tcp1 00000400:00000200:15.0:1616513078.053050:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.205@tcp1 routed destination 00000400:00000200:15.0:1616513078.053066:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.205@tcp1(10.31.141.205@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.053073:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.053079:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.053082:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.053084:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.053089:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.053095:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.053096:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513078.053107:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.053114:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.053117:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.053124:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.053129:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd827f8 00000800:00000200:30.0:1616513078.053133:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.053136:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.053141:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513078.053149:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.205@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.053157:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.205@tcp1 of length 224 into portal 12 MB=0x5ffddb9919440 00000800:00000200:22.0:1616513078.053161:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513078.053163:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.205@tcp1 of length 224/224 into md 0xf96450b9 [1] + 74944 00000400:00000200:30.0:1616513078.053168:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513078.053170:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513078.053177:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.053179:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.053181:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513078.053245:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702649734208 00010000:00000200:26.0:1616513078.053256:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7a880 x1688702649734208/t0(0) o400->e7a99691-4eb8-0d37-5b8f-27969355a6be@10.31.141.205@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513078.053270:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702649734208, offset 192 00000400:00000200:26.0:1616513078.053276:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.205@tcp1 00000400:00000200:26.0:1616513078.053284:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.205@tcp1 routed destination 00000400:00000200:26.0:1616513078.053298:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.205@tcp1(10.31.141.205@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513078.053304:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513078.053310:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513078.053312:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513078.053314:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513078.053319:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.053326:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.053377:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.053384:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.053386:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.053388:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.053392:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.053395:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af7e8 00000800:00000200:24.0:1616513078.053400:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.078399:0:49236:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513078.078456:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.078466:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513078.078475:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.131@tcp1 : PUT - for me 00000400:00000200:14.0:1616513078.078485:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.131@tcp1 of length 224 into portal 12 MB=0x5ffd4c2cb7d80 00000400:00000200:14.0:1616513078.078492:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.131@tcp1 of length 224/224 into md 0xf96450b9 [1] + 75168 00000400:00000200:14.0:1616513078.078497:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513078.078506:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513078.078510:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.078514:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.078518:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.078519:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.078578:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513078.078580:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664149818752 00010000:00000200:26.0:1616513078.078593:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba7980 x1688664149818752/t0(0) o400->f35aabd4-9980-7926-c280-e8499c4c8e45@10.31.136.131@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513078.078608:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664149818752, offset 192 00000400:00000200:26.0:1616513078.078614:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.131@tcp1 00000400:00000200:26.0:1616513078.078622:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.131@tcp1 routed destination 00000800:00000200:28.0:1616513078.078633:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513078.078638:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.131@tcp1(10.31.136.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513078.078642:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513078.078644:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513078.078651:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513078.078652:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.131@tcp1 : PUT - for me 00000800:00000200:26.0:1616513078.078655:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513078.078658:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513078.078661:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.131@tcp1 of length 224 into portal 26 MB=0x5ffd4c2cb7c80 00000800:00000200:26.0:1616513078.078663:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513078.078668:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.131@tcp1 of length 224/224 into md 0xf97c76b9 [1] + 0 00000800:00000200:7.2:1616513078.078669:0:49236:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513078.078673:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05320 00000400:00000200:28.0:1616513078.078676:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513078.078679:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:28.0:1616513078.078679:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:30.0:1616513078.078683:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.078685:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.078685:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05320 00000800:00000200:20.0:1616513078.078685:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.078688:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513078.078690:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513078.078691:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513078.078693:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa08 00000800:00000200:28.0:1616513078.078693:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513078.078696:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.078698:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.078750:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664149818496 00010000:00000200:15.0:1616513078.078761:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688664149818496/t0(0) o400->c2428504-8b58-d79d-f7c2-be0fd740fa27@10.31.136.131@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.078776:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664149818496, offset 192 00000400:00000200:15.0:1616513078.078782:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.131@tcp1 00000400:00000200:15.0:1616513078.078791:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.131@tcp1 routed destination 00000400:00000200:15.0:1616513078.078807:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.131@tcp1(10.31.136.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.078813:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.078819:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.078824:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.078826:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.078831:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.078836:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.078838:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513078.078846:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.078851:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.078854:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.078859:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.078865:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82440 00000800:00000200:20.0:1616513078.078869:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.078871:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:23.2F:1616513078.091823:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4c33dba00] (20)++ 00000800:00000200:23.0:1616513078.091841:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c33dba00] (21)++ 00000800:00000200:23.0:1616513078.091848:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.218@o2ib1 00000400:00000200:23.0:1616513078.091857:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.218@o2ib1 : PUT - for me 00000400:00000200:23.0:1616513078.091866:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.218@o2ib1 of length 224 into portal 26 MB=0x5fb73ae0b5590 00000400:00000200:23.0:1616513078.091872:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.218@o2ib1 of length 224/224 into md 0xf97c76c9 [1] + 0 00000400:00000200:23.0:1616513078.091877:0:6474:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05dc0 00000400:00000200:23.0:1616513078.091879:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:23.0:1616513078.091882:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:23.0:1616513078.091887:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05dc0 00000800:00000200:23.0:1616513078.091890:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4c33dba00] (22)++ 00000800:00000200:23.0:1616513078.091893:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4c33dba00] (23)-- 00000800:00000200:27.0:1616513078.091895:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c33dba00] (22)++ 00000800:00000200:23.0:1616513078.091903:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c33dba00] (23)-- 00000800:00000200:23.0:1616513078.091906:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c33dba00] (22)-- 00000800:00000200:27.0:1616513078.091907:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.218@o2ib1 00000400:00000200:27.0:1616513078.091915:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.218@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513078.091924:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.218@o2ib1 of length 224 into portal 12 MB=0x5fb73ae0b55b0 00000400:00000200:27.0:1616513078.091930:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.218@o2ib1 of length 224/224 into md 0x3657608d [1] + 91168 00000400:00000200:27.0:1616513078.091938:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513078.091940:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513078.091948:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1683849143342480 00000800:00000200:27.0:1616513078.091950:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4c33dba00] (21)++ 00000800:00000200:27.0:1616513078.091953:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4c33dba00] (22)-- 00000800:00000200:27.0:1616513078.091955:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c33dba00] (21)-- 00010000:00000200:15.0:1616513078.091959:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1683849143342480/t0(0) o400->81e43ee6-6281-cbcb-a690-58307b7beb54@10.242.46.218@o2ib1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.091973:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1683849143342480, offset 192 00000400:00000200:15.0:1616513078.091979:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.218@o2ib1 00000400:00000200:15.0:1616513078.091986:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.218@o2ib1 local destination 00000400:00000200:15.0:1616513078.092000:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.218@o2ib1(10.242.46.218@o2ib1:10.242.46.218@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.092006:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.218@o2ib1 00000800:00000200:15.0:1616513078.092012:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91ec000] -> 10.242.46.218@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.092015:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4c33dba00] (20)++ 00000100:00000200:19.0:1616513078.092016:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1683849143342512 00000800:00000200:15.0:1616513078.092017:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4c33dba00] (21)++ 00000800:00000200:15.0:1616513078.092022:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4c33dba00] (22)-- 00000800:00000200:23.2:1616513078.092027:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4c33dba00] (21)++ 00010000:00000200:19.0:1616513078.092029:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d841ded2400 x1683849143342512/t0(0) o400->2d95f4f3-2dbf-7275-e908-358bdabab8d0@10.242.46.218@o2ib1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513078.092029:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:23.0:1616513078.092035:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c33dba00] (22)++ 00000800:00000200:23.0:1616513078.092039:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4c33dba00] (23)-- 00000400:00000200:23.0:1616513078.092041:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:19.0:1616513078.092043:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1683849143342512, offset 192 00000800:00000200:27.0:1616513078.092044:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c33dba00] (22)-- 00000400:00000200:23.0:1616513078.092046:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.218@o2ib1: PUT: OK 00000400:00000200:19.0:1616513078.092049:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.218@o2ib1 00000400:00000200:23.0:1616513078.092053:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83b18 00000400:00000200:19.0:1616513078.092056:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.218@o2ib1 local destination 00000400:00000200:19.0:1616513078.092068:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.218@o2ib1(10.242.46.218@o2ib1:10.242.46.218@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513078.092077:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c33dba00] (21)-- 00000800:00000200:19.0:1616513078.092078:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.218@o2ib1 00000800:00000200:19.0:1616513078.092083:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91ec000] -> 10.242.46.218@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513078.092086:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4c33dba00] (20)++ 00000800:00000200:19.0:1616513078.092088:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4c33dba00] (21)++ 00000800:00000200:19.0:1616513078.092092:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4c33dba00] (22)-- 00000800:00000200:23.2:1616513078.092097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4c33dba00] (21)++ 00000800:00000200:23.0:1616513078.092104:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c33dba00] (22)++ 00000800:00000200:23.0:1616513078.092107:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4c33dba00] (23)-- 00000400:00000200:23.0:1616513078.092109:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:27.0:1616513078.092111:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c33dba00] (22)-- 00000400:00000200:23.0:1616513078.092113:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.218@o2ib1: PUT: OK 00000400:00000200:23.0:1616513078.092116:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706cc0 00000800:00000200:23.0:1616513078.092118:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c33dba00] (21)-- 00000800:00000200:7.2:1616513078.098852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.098908:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.098917:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513078.098923:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.098927:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.71@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.098938:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.71@tcp1 of length 224 into portal 12 MB=0x5ffd341e35480 00000400:00000200:22.0:1616513078.098945:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.71@tcp1 of length 224/224 into md 0xf96450b9 [1] + 75392 00000400:00000200:22.0:1616513078.098950:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.098953:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513078.098963:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.098966:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.098968:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513078.099030:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657692152960 00000800:00000200:11.2:1616513078.099040:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:26.0:1616513078.099043:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec206300 x1688657692152960/t0(0) o400->cd28ff2e-ae93-b9bb-9511-28db9d9bb7a8@10.31.133.71@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513078.099058:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657692152960, offset 192 00000400:00000200:26.0:1616513078.099064:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.71@tcp1 00000400:00000200:26.0:1616513078.099073:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.71@tcp1 routed destination 00000400:00000200:26.0:1616513078.099088:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.71@tcp1(10.31.133.71@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513078.099094:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513078.099095:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513078.099101:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513078.099104:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513078.099106:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513078.099110:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513078.099112:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513078.099116:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513078.099117:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513078.099119:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.71@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.099128:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.71@tcp1 of length 224 into portal 26 MB=0x5ffd341e35400 00000400:00000200:20.0:1616513078.099135:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.71@tcp1 of length 224/224 into md 0xf97c76e9 [1] + 0 00000400:00000200:20.0:1616513078.099140:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04ee0 00000400:00000200:20.0:1616513078.099143:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.099146:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513078.099151:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04ee0 00000800:00000200:20.0:1616513078.099154:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.099157:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.099158:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.099216:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657692152832 00010000:00000200:15.0:1616513078.099226:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688657692152832/t0(0) o400->aa907c2c-c998-4381-0604-bfc04be15bd1@10.31.133.71@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.099241:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657692152832, offset 192 00000400:00000200:15.0:1616513078.099247:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.71@tcp1 00000400:00000200:15.0:1616513078.099255:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.71@tcp1 routed destination 00000400:00000200:15.0:1616513078.099271:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.71@tcp1(10.31.133.71@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.099278:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.099284:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.099287:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.099289:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.099294:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513078.099297:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513078.099298:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.099301:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513078.099304:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.099307:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.099308:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:30.0:1616513078.099314:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513078.099314:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.099317:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.099318:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae550 00000800:00000200:30.0:1616513078.099322:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513078.099323:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:30.0:1616513078.099325:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513078.099329:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82ff0 00000800:00000200:20.0:1616513078.099331:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.099334:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513078.106595:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925136, portal 4 00000100:00000200:28.0:1616513078.106600:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925136, offset 0 00000400:00000200:28.0:1616513078.106607:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:28.0:1616513078.106613:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:28.0:1616513078.106625:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513078.106632:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513078.106638:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513078.106641:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513078.106643:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513078.106648:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513078.106695:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513078.106709:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513078.106714:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513078.106716:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.106719:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513078.106721:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513078.106726:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc7e06c00 x1693067725925136/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513078.106738:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd298 00000800:00000200:28.0:1616513078.106741:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513078.106820:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513078.106830:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513078.106836:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513078.106845:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513078.106853:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626b10 00000400:00000200:20.0:1616513078.106859:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d4bc1 [1] + 192 00000400:00000200:20.0:1616513078.106865:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.106870:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc7e06c00 x1693067725925136/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:24.0:1616513078.106881:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513078.106884:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513078.106887:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:28.0:1616513078.106890:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc7e06c00 x1693067725925136/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513078.106890:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:28.0:1616513078.106901:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc7e06c00 x1693067725925136/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513078.106910:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc000 00000800:00000200:7.2:1616513078.110843:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513078.110898:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.110908:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513078.110916:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.79@tcp1 : PUT - for me 00000400:00000200:14.0:1616513078.110926:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.79@tcp1 of length 224 into portal 26 MB=0x602bd4011f500 00000400:00000200:14.0:1616513078.110932:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.79@tcp1 of length 224/224 into md 0xf97c76f9 [1] + 0 00000400:00000200:14.0:1616513078.110937:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6b28 00000400:00000200:14.0:1616513078.110940:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513078.110943:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513078.110948:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6b28 00000800:00000200:14.0:1616513078.110951:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513078.110954:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513078.110956:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.110959:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.111013:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691861707257088 00010000:00000200:15.0:1616513078.111024:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1691861707257088/t0(0) o400->182a6905-4790-b56f-0ca1-90985d5e1492@10.31.129.79@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.111038:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691861707257088, offset 192 00000400:00000200:15.0:1616513078.111044:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.79@tcp1 00000400:00000200:15.0:1616513078.111053:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.79@tcp1 routed destination 00000400:00000200:15.0:1616513078.111068:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.79@tcp1(10.31.129.79@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.111075:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.111081:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.111084:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.111086:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.111091:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.111095:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.111098:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513078.111104:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.111108:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.111110:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.111116:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.111121:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd838f8 00000800:00000200:30.0:1616513078.111124:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.111126:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.111233:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.111290:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.111299:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.111308:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.79@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.111317:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.79@tcp1 of length 224 into portal 12 MB=0x602bd4011f740 00000400:00000200:20.0:1616513078.111323:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.79@tcp1 of length 224/224 into md 0x865bbec1 [1] + 7392 00000400:00000200:20.0:1616513078.111329:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.111331:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513078.111339:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.111342:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.111343:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.111347:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.111405:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691861707257664 00010000:00000200:0.0:1616513078.111418:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01200 x1691861707257664/t0(0) o400->d2f1d41e-7bc0-695f-905d-f5d386822bca@10.31.129.79@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.111432:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691861707257664, offset 192 00000400:00000200:0.0:1616513078.111453:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.79@tcp1 00000400:00000200:0.0:1616513078.111461:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.79@tcp1 routed destination 00000400:00000200:0.0:1616513078.111476:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.79@tcp1(10.31.129.79@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.111483:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.111491:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.111494:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.111496:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.111501:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.111506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.111558:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.111566:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.111568:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.111570:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.111575:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.111579:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04330 00000800:00000200:24.0:1616513078.111583:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.130353:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.130411:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.130420:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.130430:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.132@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.130453:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.132@tcp1 of length 224 into portal 26 MB=0x600030aa0c100 00000400:00000200:22.0:1616513078.130460:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.132@tcp1 of length 224/224 into md 0xf97c7709 [1] + 0 00000400:00000200:22.0:1616513078.130465:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6330 00000400:00000200:22.0:1616513078.130468:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513078.130470:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:22.0:1616513078.130471:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513078.130476:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6330 00000800:00000200:22.0:1616513078.130479:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513078.130482:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513078.130483:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513078.130484:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513078.130488:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513078.130492:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.132@tcp1 : PUT - for me 00000400:00000200:14.0:1616513078.130500:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.132@tcp1 of length 224 into portal 12 MB=0x600030aa0c5c0 00000400:00000200:14.0:1616513078.130507:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.132@tcp1 of length 224/224 into md 0xf96450b9 [1] + 75616 00000400:00000200:14.0:1616513078.130512:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513078.130515:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513078.130522:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.130525:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.130527:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.130540:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688862923473152 00010000:00000200:15.0:1616513078.130551:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688862923473152/t0(0) o400->b5dbf3d1-b357-f11d-3fe1-34d450bec2fd@10.31.129.132@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.130565:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688862923473152, offset 192 00000400:00000200:15.0:1616513078.130572:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.132@tcp1 00000400:00000200:15.0:1616513078.130579:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.132@tcp1 routed destination 00000100:00000200:26.0:1616513078.130597:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688862923474368 00000400:00000200:15.0:1616513078.130607:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.132@tcp1(10.31.129.132@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:26.0:1616513078.130610:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba5e80 x1688862923474368/t0(0) o400->e43d1e7d-5dc8-18e1-6a73-cd3909d175f8@10.31.129.132@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513078.130613:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.130620:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.130622:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.130624:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:26.0:1616513078.130625:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688862923474368, offset 192 00000800:00000200:15.0:1616513078.130629:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.130631:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.132@tcp1 00000800:00000200:7.2:1616513078.130633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.130636:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513078.130638:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.132@tcp1 routed destination 00000400:00000200:26.0:1616513078.130654:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.132@tcp1(10.31.129.132@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513078.130660:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513078.130666:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513078.130669:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513078.130671:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513078.130676:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.130681:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513078.130685:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.130693:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.130694:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.130696:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.130702:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.130708:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82c38 00000800:00000200:18.0:1616513078.130711:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.130745:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.130752:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.130755:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.130761:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.130767:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af4b8 00000800:00000200:28.0:1616513078.130770:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.130773:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.141808:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.141864:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.141873:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513078.141876:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513078.141882:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.115@tcp1 : PUT - for me 00000800:00000200:30.0:1616513078.141886:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513078.141886:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.141893:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.115@tcp1 of length 224 into portal 26 MB=0x60001f38f0d80 00000400:00000200:14.0:1616513078.141895:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.115@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.141900:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.115@tcp1 of length 224/224 into md 0xf97c7719 [1] + 0 00000400:00000200:14.0:1616513078.141904:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.115@tcp1 of length 224 into portal 12 MB=0x60001f38f0e40 00000400:00000200:22.0:1616513078.141905:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6908 00000400:00000200:22.0:1616513078.141907:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513078.141911:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.115@tcp1 of length 224/224 into md 0xf96450b9 [1] + 75840 00000100:00000200:22.0:1616513078.141916:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513078.141916:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513078.141921:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6908 00000100:00000200:14.0:1616513078.141923:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513078.141924:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.141927:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513078.141928:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.141930:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.141933:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.141935:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.141986:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688858241469824 00000100:00000200:26.0:1616513078.141993:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688858241470016 00010000:00000200:15.0:1616513078.141997:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688858241469824/t0(0) o400->691aea73-b2f9-a8ca-7a8a-446f3c9505b4@10.31.138.115@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:26.0:1616513078.142006:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba3f00 x1688858241470016/t0(0) o400->1f6d695b-912f-7666-d7d2-cfb59d9a17bc@10.31.138.115@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513078.142012:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688858241469824, offset 192 00000400:00000200:15.0:1616513078.142020:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.115@tcp1 00000400:00000200:15.0:1616513078.142029:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.115@tcp1 routed destination 00000100:00000200:26.0:1616513078.142032:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688858241470016, offset 192 00000400:00000200:26.0:1616513078.142039:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.115@tcp1 00000400:00000200:15.0:1616513078.142045:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.115@tcp1(10.31.138.115@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.142053:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513078.142055:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.115@tcp1 routed destination 00000800:00000200:15.0:1616513078.142059:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.142062:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.142064:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.142069:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.142071:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.115@tcp1(10.31.138.115@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513078.142074:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.142077:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513078.142078:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513078.142085:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513078.142087:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513078.142089:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513078.142094:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513078.142126:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.142133:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.142135:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.142136:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513078.142139:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513078.142142:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.142148:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd837e8 00000800:00000200:18.0:1616513078.142152:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513078.142196:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.142202:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.142205:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.142212:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.142218:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afdc0 00000800:00000200:20.0:1616513078.142222:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.142225:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.143633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513078.143688:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.143697:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.143707:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.95@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.143716:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.95@tcp1 of length 224 into portal 26 MB=0x5ffdd2d3bfd40 00000400:00000200:28.0:1616513078.143723:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.95@tcp1 of length 224/224 into md 0xf97c7729 [1] + 0 00000400:00000200:28.0:1616513078.143727:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7c28 00000400:00000200:28.0:1616513078.143730:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.143733:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.143737:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7c28 00000800:00000200:28.0:1616513078.143741:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.143742:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.143743:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.143746:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.143801:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700295314752 00010000:00000200:15.0:1616513078.143811:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688700295314752/t0(0) o400->d308ae40-d3cd-fe0d-ef1b-e918581244f0@10.31.128.95@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.143824:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700295314752, offset 192 00000400:00000200:15.0:1616513078.143830:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.95@tcp1 00000400:00000200:15.0:1616513078.143837:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.95@tcp1 routed destination 00000400:00000200:15.0:1616513078.143851:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.95@tcp1(10.31.128.95@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.143858:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.143864:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.143866:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.143868:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.143873:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.143878:0:49243:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.143879:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513078.143907:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.143918:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.143925:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513078.143933:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513078.143934:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.95@tcp1 : PUT - for me 00000800:00000200:22.0:1616513078.143940:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513078.143942:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.95@tcp1 of length 224 into portal 12 MB=0x5ffdd2d3c0140 00000400:00000200:22.0:1616513078.143943:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.143949:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.95@tcp1 of length 224/224 into md 0x865bbec1 [1] + 7616 00000400:00000200:22.0:1616513078.143950:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.143954:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513078.143956:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83540 00000100:00000200:20.0:1616513078.143957:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513078.143960:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513078.143964:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513078.143965:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513078.143967:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.143968:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.143972:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.144030:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700295315776 00010000:00000200:0.0:1616513078.144043:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01b00 x1688700295315776/t0(0) o400->7142c979-deaf-fea7-80eb-ba7c1fe4fc7f@10.31.128.95@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.144057:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700295315776, offset 192 00000400:00000200:0.0:1616513078.144063:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.95@tcp1 00000400:00000200:0.0:1616513078.144071:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.95@tcp1 routed destination 00000400:00000200:0.0:1616513078.144086:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.95@tcp1(10.31.128.95@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.144092:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.144099:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.144107:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.144109:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.144114:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.144119:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.144127:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.144131:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.144134:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513078.144136:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.144139:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.144143:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb053a8 00000800:00000200:20.0:1616513078.144146:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.147889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513078.147902:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.147909:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513078.147917:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.128@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.147926:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.128@tcp1 of length 224 into portal 26 MB=0x6018300227ac0 00000400:00000200:30.0:1616513078.147932:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.128@tcp1 of length 224/224 into md 0xf97c7749 [1] + 0 00000400:00000200:30.0:1616513078.147937:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6220 00000400:00000200:30.0:1616513078.147940:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513078.147942:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513078.147947:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6220 00000800:00000200:30.0:1616513078.147950:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.147952:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.147954:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.147958:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.148010:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512014867136 00010000:00000200:15.0:1616513078.148020:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1690512014867136/t0(0) o400->80d19db6-9dcd-dd36-7a64-cecdcdcdcd06@10.31.136.128@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.148033:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512014867136, offset 192 00000400:00000200:15.0:1616513078.148039:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.128@tcp1 00000400:00000200:15.0:1616513078.148046:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.128@tcp1 routed destination 00000400:00000200:15.0:1616513078.148061:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.128@tcp1(10.31.136.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.148071:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.148077:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.148080:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.148082:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.148086:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.148091:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.148093:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513078.148144:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.148151:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513078.148154:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513078.148161:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513078.148167:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd836d8 00000800:00000200:22.0:1616513078.148171:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.148174:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.149446:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.149502:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.149512:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.149521:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.128@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.149529:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.128@tcp1 of length 224 into portal 12 MB=0x6018300227c80 00000400:00000200:24.0:1616513078.149536:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.128@tcp1 of length 224/224 into md 0x865bbec1 [1] + 7840 00000400:00000200:24.0:1616513078.149541:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.149544:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513078.149550:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.149553:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.149555:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.149559:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.149625:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512014867584 00010000:00000200:0.0:1616513078.149638:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc0480 x1690512014867584/t0(0) o400->296273a4-1d72-8efa-9cf3-d3fbd262f4c0@10.31.136.128@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.149652:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512014867584, offset 192 00000400:00000200:0.0:1616513078.149658:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.128@tcp1 00000400:00000200:0.0:1616513078.149666:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.128@tcp1 routed destination 00000400:00000200:0.0:1616513078.149681:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.128@tcp1(10.31.136.128@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.149687:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.149693:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.149696:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.149698:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.149703:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.149708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.149762:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.149769:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.149772:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.149773:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.149778:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.149782:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:28.0:1616513078.149786:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.154960:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.155016:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.155025:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.155034:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.26@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.155043:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.26@tcp1 of length 224 into portal 26 MB=0x5ffd4fbffb900 00000400:00000200:20.0:1616513078.155050:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.26@tcp1 of length 224/224 into md 0xf97c7779 [1] + 0 00000400:00000200:20.0:1616513078.155055:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c78f8 00000400:00000200:20.0:1616513078.155058:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.155060:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513078.155065:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c78f8 00000800:00000200:20.0:1616513078.155068:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.155070:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.155071:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.155078:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.155129:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665109543168 00010000:00000200:15.0:1616513078.155140:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688665109543168/t0(0) o400->4b8765dd-5a13-a51c-86a4-cb8813a43884@10.31.136.26@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.155154:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665109543168, offset 192 00000400:00000200:15.0:1616513078.155160:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.26@tcp1 00000400:00000200:15.0:1616513078.155168:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.26@tcp1 routed destination 00000400:00000200:15.0:1616513078.155183:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.26@tcp1(10.31.136.26@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.155190:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.155197:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.155199:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.155202:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.155206:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.155207:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:15.0:1616513078.155213:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513078.155263:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.155275:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513078.155275:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513078.155279:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:30.0:1616513078.155281:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.155284:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.26@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.155287:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.155292:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd834b8 00000400:00000200:18.0:1616513078.155294:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.26@tcp1 of length 224 into portal 12 MB=0x5ffd4fbffb980 00000800:00000200:30.0:1616513078.155295:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.155298:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.155300:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.26@tcp1 of length 224/224 into md 0xf96450b9 [1] + 76064 00000400:00000200:18.0:1616513078.155306:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.155308:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.155315:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.155318:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.155320:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513078.155378:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665109543296 00010000:00000200:26.0:1616513078.155390:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb8d80 x1688665109543296/t0(0) o400->87ccbdc0-a00e-195f-e649-ae78b9437036@10.31.136.26@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513078.155406:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665109543296, offset 192 00000400:00000200:26.0:1616513078.155412:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.26@tcp1 00000400:00000200:26.0:1616513078.155420:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.26@tcp1 routed destination 00000400:00000200:26.0:1616513078.155451:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.26@tcp1(10.31.136.26@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513078.155458:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513078.155467:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513078.155470:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513078.155472:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513078.155477:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.155482:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.155491:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.155495:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.155497:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.155503:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.155505:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af870 00000800:00000200:20.0:1616513078.155508:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.155511:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.165723:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513078.165737:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.165745:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513078.165746:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513078.165753:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.73@tcp1 : PUT - for me 00000400:00000200:14.0:1616513078.165763:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.73@tcp1 of length 224 into portal 26 MB=0x5ffd43628f640 00000400:00000200:14.0:1616513078.165769:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.73@tcp1 of length 224/224 into md 0xf97c7799 [1] + 0 00000400:00000200:14.0:1616513078.165773:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6770 00000400:00000200:14.0:1616513078.165776:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513078.165779:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513078.165783:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6770 00000800:00000200:14.0:1616513078.165786:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.165789:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.165791:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.165851:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661790357056 00010000:00000200:15.0:1616513078.165862:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688661790357056/t0(0) o400->85665896-df09-1703-de57-8c20b266ad2a@10.31.133.73@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.165876:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661790357056, offset 192 00000400:00000200:15.0:1616513078.165882:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.73@tcp1 00000400:00000200:15.0:1616513078.165891:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.73@tcp1 routed destination 00000400:00000200:15.0:1616513078.165907:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.73@tcp1(10.31.133.73@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.165914:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.165920:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.165923:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.165927:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.165932:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.165937:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.165939:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513078.165990:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.165997:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.166001:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.166007:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.166013:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd823b8 00000800:00000200:18.0:1616513078.166017:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.166020:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513078.166205:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513078.166213:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.166220:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513078.166222:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513078.166227:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.73@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.166234:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.73@tcp1 of length 224 into portal 12 MB=0x5ffd43628f700 00000400:00000200:30.0:1616513078.166240:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.73@tcp1 of length 224/224 into md 0xf96450b9 [1] + 76288 00000400:00000200:30.0:1616513078.166244:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513078.166246:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513078.166252:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.166254:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.166255:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513078.166317:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661790357248 00010000:00000200:26.0:1616513078.166329:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a78000 x1688661790357248/t0(0) o400->eb922625-f6b8-9ca0-d3fd-a2a6ed31fbfd@10.31.133.73@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513078.166344:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661790357248, offset 192 00000400:00000200:26.0:1616513078.166350:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.73@tcp1 00000400:00000200:26.0:1616513078.166358:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.73@tcp1 routed destination 00000400:00000200:26.0:1616513078.166373:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.73@tcp1(10.31.133.73@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513078.166382:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513078.166388:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513078.166390:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513078.166393:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513078.166397:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.166444:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.166501:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.166507:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.166510:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.166516:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.166519:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae088 00000800:00000200:24.0:1616513078.166523:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.166526:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.168471:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.168527:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.168536:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.168545:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.121@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.168554:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.121@tcp1 of length 224 into portal 26 MB=0x5ffd68a1d9c00 00000400:00000200:20.0:1616513078.168561:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.121@tcp1 of length 224/224 into md 0xf97c77a9 [1] + 0 00000400:00000200:20.0:1616513078.168566:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c63b8 00000400:00000200:20.0:1616513078.168568:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.168571:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513078.168576:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c63b8 00000800:00000200:20.0:1616513078.168579:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.168582:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.168582:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.168586:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.168640:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688671788833792 00010000:00000200:15.0:1616513078.168651:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688671788833792/t0(0) o400->9893bdb4-55c7-67c2-bafa-140ba4bd7aa6@10.31.132.121@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.168665:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688671788833792, offset 192 00000400:00000200:15.0:1616513078.168671:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.121@tcp1 00000400:00000200:15.0:1616513078.168680:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.121@tcp1 routed destination 00000400:00000200:15.0:1616513078.168695:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.121@tcp1(10.31.132.121@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.168702:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.168708:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.168711:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.168714:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.168719:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.168723:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.168725:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513078.168775:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.168782:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.168786:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.168793:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.168799:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83980 00000800:00000200:18.0:1616513078.168803:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.168806:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.169201:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.169254:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.169264:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513078.169266:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.169272:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.121@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.169281:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.121@tcp1 of length 224 into portal 12 MB=0x5ffd68a1d9dc0 00000400:00000200:24.0:1616513078.169287:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.121@tcp1 of length 224/224 into md 0x865bbec1 [1] + 8064 00000400:00000200:24.0:1616513078.169292:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.169297:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513078.169303:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.169306:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.169308:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.169370:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688671788834240 00010000:00000200:0.0:1616513078.169382:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc1200 x1688671788834240/t0(0) o400->a998a73d-643d-87d8-47ee-91fc1234ebc8@10.31.132.121@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.169396:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688671788834240, offset 192 00000400:00000200:0.0:1616513078.169403:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.121@tcp1 00000400:00000200:0.0:1616513078.169410:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.121@tcp1 routed destination 00000400:00000200:0.0:1616513078.169425:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.121@tcp1(10.31.132.121@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.169432:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.169451:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.169454:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.169457:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.169461:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.169467:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.169478:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.169483:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.169486:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.169487:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.169492:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.169503:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000800:00000200:28.0:1616513078.169507:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.181570:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513078.181582:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.181589:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513078.181591:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513078.181597:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.194@tcp1 : PUT - for me 00000400:00000200:14.0:1616513078.181606:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.194@tcp1 of length 224 into portal 26 MB=0x6001beca3fd80 00000400:00000200:14.0:1616513078.181612:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.194@tcp1 of length 224/224 into md 0xf97c77b9 [1] + 0 00000400:00000200:14.0:1616513078.181616:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7078 00000400:00000200:14.0:1616513078.181618:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513078.181621:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513078.181625:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7078 00000800:00000200:14.0:1616513078.181628:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.181630:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.181632:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.181689:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969794551168 00010000:00000200:15.0:1616513078.181700:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688969794551168/t0(0) o400->e4216878-8d42-6b51-a191-94e6bd696ce8@10.31.138.194@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.181714:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688969794551168, offset 192 00000400:00000200:15.0:1616513078.181720:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.194@tcp1 00000400:00000200:15.0:1616513078.181728:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.194@tcp1 routed destination 00000400:00000200:15.0:1616513078.181744:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.194@tcp1(10.31.138.194@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.181750:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.181757:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.181760:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.181762:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.181767:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.181772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.181773:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513078.181793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513078.181835:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.181842:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.181846:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.181852:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513078.181853:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513078.181859:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83650 00000800:00000200:18.0:1616513078.181862:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513078.181863:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:18.0:1616513078.181865:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513078.181872:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.194@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.181880:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.194@tcp1 of length 224 into portal 12 MB=0x6001beca3fe00 00000400:00000200:20.0:1616513078.181886:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.194@tcp1 of length 224/224 into md 0x865bbec1 [1] + 8288 00000400:00000200:20.0:1616513078.181892:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.181894:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513078.181901:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.181904:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.181905:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.181909:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.181964:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969794551296 00010000:00000200:0.0:1616513078.181976:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01f80 x1688969794551296/t0(0) o400->24f38379-27d8-f75f-b5f6-269d24849482@10.31.138.194@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.181999:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688969794551296, offset 192 00000400:00000200:0.0:1616513078.182005:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.194@tcp1 00000400:00000200:0.0:1616513078.182012:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.194@tcp1 routed destination 00000400:00000200:0.0:1616513078.182026:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.194@tcp1(10.31.138.194@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.182033:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.182038:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.182045:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.182047:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.182051:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.182057:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.182067:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.182071:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.182073:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.182079:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.182082:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000800:00000200:20.0:1616513078.182085:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.182088:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.182449:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.182461:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.182471:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513078.182472:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.182480:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.195@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.182489:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.195@tcp1 of length 224 into portal 26 MB=0x5ffd444a8d900 00000400:00000200:24.0:1616513078.182497:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.195@tcp1 of length 224/224 into md 0xf97c77d9 [1] + 0 00000400:00000200:24.0:1616513078.182502:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c76d8 00000400:00000200:24.0:1616513078.182504:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.182507:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.182512:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c76d8 00000800:00000200:24.0:1616513078.182515:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.182518:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.182520:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.182527:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662033619200 00010000:00000200:15.0:1616513078.182533:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688662033619200/t0(0) o400->c9623477-2201-00a4-642d-e836e2acd008@10.31.136.195@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.182547:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662033619200, offset 192 00000400:00000200:15.0:1616513078.182553:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.195@tcp1 00000400:00000200:15.0:1616513078.182559:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.195@tcp1 routed destination 00000400:00000200:15.0:1616513078.182573:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.195@tcp1(10.31.136.195@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.182578:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.182584:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.182586:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.182588:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.182592:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.182596:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.182597:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.182606:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.182610:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.182613:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513078.182614:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513078.182618:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.182623:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82908 00000800:00000200:30.0:1616513078.182627:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.183243:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.183252:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.183256:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.183263:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.195@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.183269:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.195@tcp1 of length 224 into portal 12 MB=0x5ffd444a8dac0 00000400:00000200:20.0:1616513078.183274:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.195@tcp1 of length 224/224 into md 0x865bbec1 [1] + 8512 00000400:00000200:20.0:1616513078.183277:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.183279:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513078.183283:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.183285:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.183286:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.183288:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.183348:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662033619648 00010000:00000200:0.0:1616513078.183360:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e02400 x1688662033619648/t0(0) o400->aa2822ab-db07-4a6e-a56d-1187d6224123@10.31.136.195@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.183373:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662033619648, offset 192 00000400:00000200:0.0:1616513078.183379:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.195@tcp1 00000400:00000200:0.0:1616513078.183387:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.195@tcp1 routed destination 00000400:00000200:0.0:1616513078.183401:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.195@tcp1(10.31.136.195@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.183407:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.183413:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.183416:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.183418:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.183422:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.183427:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.183447:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.183450:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.183452:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.183455:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.183456:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.183459:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:20.0:1616513078.183461:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.185535:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.185593:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.185602:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.185610:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.201@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.185619:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.201@tcp1 of length 224 into portal 26 MB=0x60216978757c0 00000400:00000200:18.0:1616513078.185625:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.201@tcp1 of length 224/224 into md 0xf97c77f9 [1] + 0 00000400:00000200:18.0:1616513078.185630:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7a90 00000400:00000200:18.0:1616513078.185632:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.185635:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513078.185639:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7a90 00000800:00000200:18.0:1616513078.185642:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.185644:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513078.185646:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.185650:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.185714:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145915029440 00010000:00000200:15.0:1616513078.185724:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1691145915029440/t0(0) o400->383468a1-e1cb-e7d8-0171-68bb2e2072f7@10.31.140.201@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.185737:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691145915029440, offset 192 00000400:00000200:15.0:1616513078.185744:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.201@tcp1 00000400:00000200:15.0:1616513078.185752:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.201@tcp1 routed destination 00000400:00000200:15.0:1616513078.185766:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.201@tcp1(10.31.140.201@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.185772:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.185778:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.185781:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.185783:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.185788:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.185792:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.185794:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513078.185802:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513078.185808:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513078.185810:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513078.185811:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513078.185816:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513078.185821:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82d48 00000800:00000200:14.0:1616513078.185825:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.186585:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.186639:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.186647:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.186656:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.201@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.186664:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.201@tcp1 of length 224 into portal 12 MB=0x6021697875d00 00000400:00000200:20.0:1616513078.186670:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.201@tcp1 of length 224/224 into md 0x865bbec1 [1] + 8736 00000400:00000200:20.0:1616513078.186675:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.186677:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513078.186684:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.186688:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.186690:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.186693:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.186745:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145915030784 00010000:00000200:0.0:1616513078.186757:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e02d00 x1691145915030784/t0(0) o400->8e741bb3-1969-6e35-b0a3-02311d83795f@10.31.140.201@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.186770:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691145915030784, offset 192 00000400:00000200:0.0:1616513078.186776:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.201@tcp1 00000400:00000200:0.0:1616513078.186783:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.201@tcp1 routed destination 00000400:00000200:0.0:1616513078.186797:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.201@tcp1(10.31.140.201@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.186805:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.186811:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.186813:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.186815:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.186820:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.186825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.186878:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.186885:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.186888:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.186889:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.186895:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.186899:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb053a8 00000800:00000200:24.0:1616513078.186902:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.206387:0:49250:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.206455:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.206464:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.206473:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.106@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.206482:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.106@tcp1 of length 224 into portal 26 MB=0x5fffc6b609c80 00000400:00000200:22.0:1616513078.206489:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.106@tcp1 of length 224/224 into md 0xf97c7809 [1] + 0 00000400:00000200:22.0:1616513078.206494:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7b18 00000400:00000200:22.0:1616513078.206497:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.206500:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513078.206505:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7b18 00000800:00000200:18.0:1616513078.206507:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.206508:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.206511:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.206512:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.206569:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688834481888384 00010000:00000200:15.0:1616513078.206580:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688834481888384/t0(0) o400->b0d35f09-bf54-f26b-ea4f-0103bbb3932b@10.31.138.106@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.206594:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688834481888384, offset 192 00000400:00000200:15.0:1616513078.206600:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.106@tcp1 00000400:00000200:15.0:1616513078.206608:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.106@tcp1 routed destination 00000400:00000200:15.0:1616513078.206626:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.106@tcp1(10.31.138.106@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.206632:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.206639:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.206642:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.206644:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.206649:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.206656:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.206656:0:49250:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.206666:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.206672:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.206674:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.206680:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.206692:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83ba0 00000800:00000200:11.2:1616513078.206693:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513078.206712:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.206714:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.206771:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.206780:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.206789:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.106@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.206798:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.106@tcp1 of length 224 into portal 12 MB=0x5fffc6b609d00 00000400:00000200:28.0:1616513078.206804:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.106@tcp1 of length 224/224 into md 0x865bbec1 [1] + 8960 00000400:00000200:28.0:1616513078.206811:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.206814:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513078.206821:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.206823:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.206824:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.206827:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.206884:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688834481888512 00010000:00000200:0.0:1616513078.206896:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6723600 x1688834481888512/t0(0) o400->61a7f671-5c69-8346-40d5-0365c86fec17@10.31.138.106@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.206909:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688834481888512, offset 192 00000400:00000200:0.0:1616513078.206915:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.106@tcp1 00000400:00000200:0.0:1616513078.206922:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.106@tcp1 routed destination 00000400:00000200:0.0:1616513078.206936:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.106@tcp1(10.31.138.106@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.206943:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.206949:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.206951:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.206953:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.206958:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.206963:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.206973:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.206978:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.206980:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.206986:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.206990:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04330 00000800:00000200:20.0:1616513078.206994:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.206996:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.268968:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.269027:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.269036:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.269045:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.19@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.269054:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.19@tcp1 of length 224 into portal 26 MB=0x6018c0d67f080 00000400:00000200:20.0:1616513078.269062:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.19@tcp1 of length 224/224 into md 0xf97c7829 [1] + 0 00000400:00000200:20.0:1616513078.269066:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6440 00000400:00000200:20.0:1616513078.269069:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.269072:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513078.269077:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6440 00000800:00000200:20.0:1616513078.269080:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.269081:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.269083:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.269085:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.269124:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.269135:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.269140:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:15.0:1616513078.269143:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690550892228736 00000400:00000200:20.0:1616513078.269147:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.19@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.269153:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.19@tcp1 of length 224 into portal 12 MB=0x6018c0d67f340 00010000:00000200:15.0:1616513078.269154:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1690550892228736/t0(0) o400->0168afeb-e7e5-1ee8-8c28-c31578f38020@10.31.142.19@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513078.269158:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.19@tcp1 of length 224/224 into md 0x865bbec1 [1] + 9184 00000400:00000200:20.0:1616513078.269162:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.269165:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513078.269168:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690550892228736, offset 192 00000800:00000200:20.0:1616513078.269170:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.269172:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.269175:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:15.0:1616513078.269175:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.19@tcp1 00000800:00000200:20.0:1616513078.269178:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:15.0:1616513078.269183:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.19@tcp1 routed destination 00000400:00000200:15.0:1616513078.269198:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.19@tcp1(10.31.142.19@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.269204:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.269211:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.269214:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.269216:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.269220:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.269228:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.269228:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513078.269237:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690550892229440 00010000:00000200:0.0:1616513078.269249:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03a80 x1690550892229440/t0(0) o400->444b215e-f574-34a5-30ce-c4e7fdfb867e@10.31.142.19@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.269262:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690550892229440, offset 192 00000400:00000200:0.0:1616513078.269268:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.19@tcp1 00000400:00000200:0.0:1616513078.269274:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.19@tcp1 routed destination 00000800:00000200:18.0:1616513078.269283:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513078.269288:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.19@tcp1(10.31.142.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513078.269291:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.269294:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513078.269294:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.269299:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513078.269301:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513078.269302:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.269304:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513078.269307:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82e58 00000800:00000200:0.0:1616513078.269309:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513078.269311:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.269314:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.269314:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.269325:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.269330:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.269333:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.269334:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.269339:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.269343:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04ee0 00000800:00000200:28.0:1616513078.269347:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.287792:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.287849:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.287858:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.287867:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.192@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.287876:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.192@tcp1 of length 224 into portal 26 MB=0x5ffd3d47e13c0 00000400:00000200:20.0:1616513078.287882:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.192@tcp1 of length 224/224 into md 0xf97c7839 [1] + 0 00000400:00000200:20.0:1616513078.287887:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7980 00000400:00000200:20.0:1616513078.287890:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.287893:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513078.287898:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7980 00000800:00000200:20.0:1616513078.287901:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.287902:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.287903:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.287906:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.287961:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660151768000 00010000:00000200:15.0:1616513078.287972:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688660151768000/t0(0) o400->5e09710a-6155-b465-4abe-9ea5690ed1b7@10.31.128.192@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.287987:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660151768000, offset 192 00000400:00000200:15.0:1616513078.287994:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.192@tcp1 00000400:00000200:15.0:1616513078.288001:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.192@tcp1 routed destination 00000800:00000200:11.2:1616513078.288008:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:15.0:1616513078.288017:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.192@tcp1(10.31.128.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513078.288018:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.288023:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513078.288024:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.288030:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513078.288032:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.192@tcp1 : PUT - for me 00000800:00000200:15.0:1616513078.288035:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.288037:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513078.288038:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.192@tcp1 of length 224 into portal 12 MB=0x5ffd3d47e1800 00000800:00000200:15.0:1616513078.288042:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513078.288043:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.192@tcp1 of length 224/224 into md 0x865bbec1 [1] + 9408 00000400:00000200:20.0:1616513078.288047:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:15.0:1616513078.288049:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.288049:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513078.288050:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513078.288055:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.288057:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.288059:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.288061:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513078.288062:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.288069:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.288072:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.288079:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.288084:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82000 00000800:00000200:30.0:1616513078.288088:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.288090:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513078.288121:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660151769088 00010000:00000200:0.0:1616513078.288133:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03f00 x1688660151769088/t0(0) o400->34570b90-3568-7452-f390-eb9755966996@10.31.128.192@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.288148:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660151769088, offset 192 00000400:00000200:0.0:1616513078.288154:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.192@tcp1 00000400:00000200:0.0:1616513078.288162:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.192@tcp1 routed destination 00000400:00000200:0.0:1616513078.288177:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.192@tcp1(10.31.128.192@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.288184:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.288190:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.288192:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.288195:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.288200:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.288205:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.288216:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.288221:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.288224:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.288225:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.288231:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.288235:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05dc0 00000800:00000200:24.0:1616513078.288239:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.301892:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.301949:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.301959:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.301968:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.86@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.301977:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.86@tcp1 of length 224 into portal 26 MB=0x5ffdc61b86240 00000400:00000200:18.0:1616513078.301984:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.86@tcp1 of length 224/224 into md 0xf97c7849 [1] + 0 00000400:00000200:18.0:1616513078.301989:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c62a8 00000400:00000200:18.0:1616513078.301992:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.301995:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513078.302000:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c62a8 00000800:00000200:22.0:1616513078.302003:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.302003:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.302006:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.302010:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513078.302013:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513078.302013:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.302021:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.86@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.302029:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.86@tcp1 of length 224 into portal 12 MB=0x5ffdc61b862c0 00000400:00000200:22.0:1616513078.302035:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.86@tcp1 of length 224/224 into md 0xf96450b9 [1] + 76512 00000400:00000200:22.0:1616513078.302040:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.302043:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513078.302050:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.302053:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.302055:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.302064:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696880915008 00010000:00000200:15.0:1616513078.302075:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688696880915008/t0(0) o400->dbb565e3-cce1-443a-111a-72fefa348c55@10.31.141.86@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.302090:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696880915008, offset 192 00000400:00000200:15.0:1616513078.302096:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.86@tcp1 00000400:00000200:15.0:1616513078.302104:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.86@tcp1 routed destination 00000100:00000200:26.0:1616513078.302119:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696880915136 00000400:00000200:15.0:1616513078.302119:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.86@tcp1(10.31.141.86@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.302126:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:26.0:1616513078.302133:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec204c80 x1688696880915136/t0(0) o400->50ef5c05-e884-3148-048e-75e26d7fdc34@10.31.141.86@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513078.302133:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.302136:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.302138:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.302143:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.302147:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:26.0:1616513078.302148:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696880915136, offset 192 00000100:00000200:15.0:1616513078.302150:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513078.302154:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.86@tcp1 00000800:00000200:18.0:1616513078.302157:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513078.302162:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.86@tcp1 routed destination 00000800:00000200:18.0:1616513078.302162:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.302165:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.302165:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.302170:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.302176:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd824c8 00000400:00000200:26.0:1616513078.302177:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.86@tcp1(10.31.141.86@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513078.302179:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:26.0:1616513078.302184:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513078.302190:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513078.302193:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513078.302195:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513078.302200:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.302209:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.302265:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.302272:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.302275:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.302282:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.302286:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af3a8 00000800:00000200:20.0:1616513078.302290:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.302295:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.312342:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.312398:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.312407:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.312416:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.145@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.312426:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.145@tcp1 of length 224 into portal 12 MB=0x603fcdb916f40 00000400:00000200:20.0:1616513078.312445:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.145@tcp1 of length 224/224 into md 0x865bbec1 [1] + 9632 00000400:00000200:20.0:1616513078.312451:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.312452:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513078.312454:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513078.312461:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.312464:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.312466:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.312488:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513078.312524:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234410647360 00010000:00000200:0.0:1616513078.312537:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e04380 x1693234410647360/t0(0) o400->8e4e77a8-d9fd-85fd-9636-dc82f63e8ba1@10.31.138.145@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513078.312544:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513078.312550:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234410647360, offset 192 00000800:00000200:18.0:1616513078.312553:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513078.312556:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.145@tcp1 00000400:00000200:18.0:1616513078.312562:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.145@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.312563:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.145@tcp1 routed destination 00000400:00000200:0.0:1616513078.312578:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.145@tcp1(10.31.138.145@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.312584:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.312585:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.145@tcp1 of length 224 into portal 26 MB=0x603fcdb916ec0 00000800:00000200:0.0:1616513078.312590:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513078.312592:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.145@tcp1 of length 224/224 into md 0xf97c7869 [1] + 0 00000800:00000200:0.0:1616513078.312593:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513078.312595:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513078.312597:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:18.0:1616513078.312597:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7298 00000400:00000200:18.0:1616513078.312600:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513078.312600:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:18.0:1616513078.312602:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:7.2:1616513078.312605:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513078.312607:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7298 00000800:00000200:18.0:1616513078.312614:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513078.312616:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513078.312617:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513078.312623:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513078.312624:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513078.312626:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.312627:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.312633:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.312637:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05320 00000800:00000200:30.0:1616513078.312641:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.312672:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234410647232 00010000:00000200:15.0:1616513078.312683:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1693234410647232/t0(0) o400->bfd01171-76a7-a083-0ac7-255464d6045d@10.31.138.145@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.312697:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234410647232, offset 192 00000400:00000200:15.0:1616513078.312704:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.145@tcp1 00000400:00000200:15.0:1616513078.312712:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.145@tcp1 routed destination 00000400:00000200:15.0:1616513078.312729:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.145@tcp1(10.31.138.145@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.312735:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.312742:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.312745:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.312747:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.312752:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.312757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.312758:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513078.312768:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.312773:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.312775:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.312781:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.312786:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83ed0 00000800:00000200:20.0:1616513078.312790:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.312792:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.358251:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.358308:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.358321:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.358331:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.26@tcp1 : PUT - for me 00000800:00000200:18.0:1616513078.358331:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513078.358341:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.26@tcp1 of length 224 into portal 26 MB=0x6039499e15c80 00000800:00000200:18.0:1616513078.358345:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.358348:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.26@tcp1 of length 224/224 into md 0xf97c7889 [1] + 0 00000400:00000200:22.0:1616513078.358353:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7188 00000400:00000200:18.0:1616513078.358355:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.26@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.358356:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.358359:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513078.358364:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7188 00000400:00000200:18.0:1616513078.358364:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.26@tcp1 of length 224 into portal 12 MB=0x6039499e15d00 00000800:00000200:22.0:1616513078.358368:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513078.358371:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:18.0:1616513078.358371:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.26@tcp1 of length 224/224 into md 0xf96450b9 [1] + 76736 00000800:00000200:22.0:1616513078.358373:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513078.358375:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.358377:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.358380:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.358389:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.358392:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.358394:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.358427:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692786631990400 00010000:00000200:15.0:1616513078.358453:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1692786631990400/t0(0) o400->db62a2d3-1b8c-410b-584a-898f8bd669eb@10.31.140.26@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.358467:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692786631990400, offset 192 00000400:00000200:15.0:1616513078.358473:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.26@tcp1 00000100:00000200:2.0:1616513078.358478:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692786631990528 00000400:00000200:15.0:1616513078.358480:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.26@tcp1 routed destination 00010000:00000200:2.0:1616513078.358497:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebad00 x1692786631990528/t0(0) o400->660be278-d82a-12b1-8bfc-f38bb1ee74d6@10.31.140.26@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513078.358506:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.26@tcp1(10.31.140.26@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.358513:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:2.0:1616513078.358513:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692786631990528, offset 192 00000800:00000200:15.0:1616513078.358520:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513078.358520:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.26@tcp1 00000800:00000200:15.0:1616513078.358523:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.358525:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:2.0:1616513078.358528:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.26@tcp1 routed destination 00000800:00000200:15.0:1616513078.358530:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.358535:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.358537:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513078.358544:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.26@tcp1(10.31.140.26@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513078.358551:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513078.358556:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513078.358557:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513078.358562:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513078.358563:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.358565:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513078.358565:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513078.358567:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513078.358571:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:2.0:1616513078.358571:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513078.358577:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82330 00000800:00000200:11.2:1616513078.358578:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513078.358581:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513078.358633:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.358641:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.358644:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.358652:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.358656:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec088 00000800:00000200:24.0:1616513078.358660:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.358663:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513078.362574:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925152, portal 4 00000100:00000200:22.0:1616513078.362580:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925152, offset 0 00000400:00000200:22.0:1616513078.362587:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:22.0:1616513078.362592:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:22.0:1616513078.362605:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513078.362612:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:22.0:1616513078.362618:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513078.362621:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:22.0:1616513078.362623:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:22.0:1616513078.362628:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:15.0:1616513078.362676:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925168, portal 4 00000100:00000200:15.0:1616513078.362682:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925168, offset 0 00000800:00000200:5.2:1616513078.362685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:15.0:1616513078.362689:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:15.0:1616513078.362695:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:15.0:1616513078.362708:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.362715:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:15.0:1616513078.362721:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.362724:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:15.0:1616513078.362727:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:15.0:1616513078.362732:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (24)-- 00000800:00000200:20.0:1616513078.362742:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:20.0:1616513078.362749:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (24)-- 00000400:00000200:20.0:1616513078.362752:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.362759:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513078.362764:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc4ebd580 x1693067725925152/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513078.362775:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bdc0 00000800:00000200:20.0:1616513078.362780:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:20.0:1616513078.362782:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513078.362785:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513078.362787:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.362792:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:28.0:1616513078.362797:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:20.0:1616513078.362797:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc566780 x1693067725925168/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513078.362806:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82880 00000800:00000200:20.0:1616513078.362809:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513078.362869:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513078.362879:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513078.362884:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.230@o2ib1 00000800:00000200:28.0:1616513078.362887:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:20.0:1616513078.362892:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:28.0:1616513078.362897:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513078.362898:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626b20 00000800:00000200:24.0:1616513078.362900:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513078.362904:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d4c91 [1] + 192 00000400:00000200:28.0:1616513078.362906:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513078.362909:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.362913:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626b30 00000100:00000200:20.0:1616513078.362913:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc4ebd580 x1693067725925152/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513078.362925:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:28.0:1616513078.362927:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d4ca1 [1] + 192 00000800:00000200:20.0:1616513078.362927:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:20.0:1616513078.362929:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513078.362932:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.362935:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc4ebd580 x1693067725925152/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513078.362948:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc4ebd580 x1693067725925152/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513078.362958:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000100:00000200:28.0:1616513078.362963:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc566780 x1693067725925168/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513078.362979:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513078.362982:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513078.362984:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:15.0:1616513078.362989:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc566780 x1693067725925168/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513078.363002:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc566780 x1693067725925168/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513078.363013:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83f58 00000800:00000200:11.2:1616513078.435112:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.435173:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.435185:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.435194:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.15@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.435204:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.15@tcp1 of length 224 into portal 26 MB=0x5ffd3fd1fe880 00000400:00000200:24.0:1616513078.435212:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.15@tcp1 of length 224/224 into md 0xf97c78a9 [1] + 0 00000400:00000200:24.0:1616513078.435217:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7f58 00000400:00000200:24.0:1616513078.435221:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.435224:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513078.435230:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.435230:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7f58 00000800:00000200:24.0:1616513078.435233:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.435236:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.435238:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.435266:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513078.435294:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660833462400 00010000:00000200:15.0:1616513078.435305:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688660833462400/t0(0) o400->d4370d29-9641-c581-1490-e0ba1f8c6049@10.31.129.15@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513078.435318:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.435327:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513078.435328:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513078.435330:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660833462400, offset 192 00000400:00000200:20.0:1616513078.435336:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.15@tcp1 : PUT - for me 00000400:00000200:15.0:1616513078.435336:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.15@tcp1 00000400:00000200:15.0:1616513078.435345:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.15@tcp1 routed destination 00000400:00000200:20.0:1616513078.435346:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.15@tcp1 of length 224 into portal 12 MB=0x5ffd3fd1fec40 00000400:00000200:20.0:1616513078.435352:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.15@tcp1 of length 224/224 into md 0x865bbec1 [1] + 9856 00000400:00000200:20.0:1616513078.435358:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.435360:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:15.0:1616513078.435361:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.15@tcp1(10.31.129.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.435367:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513078.435368:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.435372:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.435373:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:15.0:1616513078.435374:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.435377:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.435379:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.435384:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.435391:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.435391:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513078.435400:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660833463360 00010000:00000200:0.0:1616513078.435415:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e04800 x1688660833463360/t0(0) o400->08807b65-65aa-e609-9290-5a532f324042@10.31.129.15@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.435431:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660833463360, offset 192 00000800:00000200:26.0:1616513078.435448:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513078.435451:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.15@tcp1 00000800:00000200:26.0:1616513078.435456:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.435459:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513078.435460:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.15@tcp1 routed destination 00000800:00000200:18.0:1616513078.435464:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.435465:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.435471:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83f58 00000800:00000200:26.0:1616513078.435475:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513078.435478:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.15@tcp1(10.31.129.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.435485:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.435491:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.435494:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.435496:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.435502:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.435507:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.435518:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.435523:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.435526:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.435528:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.435533:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.435537:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ba0 00000800:00000200:28.0:1616513078.435540:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.443903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.443958:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.443969:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513078.443971:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513078.443977:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.116@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.443987:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.116@tcp1 of length 224 into portal 26 MB=0x5ffd545945a00 00000800:00000200:30.0:1616513078.443990:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.443996:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.116@tcp1 of length 224/224 into md 0xf97c78b9 [1] + 0 00000400:00000200:30.0:1616513078.443998:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.116@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.444001:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c65d8 00000400:00000200:22.0:1616513078.444004:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.444007:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.116@tcp1 of length 224 into portal 12 MB=0x5ffd545945a80 00000100:00000200:22.0:1616513078.444007:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513078.444011:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c65d8 00000800:00000200:22.0:1616513078.444015:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000400:00000200:30.0:1616513078.444016:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.116@tcp1 of length 224/224 into md 0xf96450b9 [1] + 76960 00000800:00000200:22.0:1616513078.444018:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513078.444019:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.444021:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513078.444022:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:30.0:1616513078.444024:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513078.444032:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.444036:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.444038:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513078.444051:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666344020608 00010000:00000200:6.0:1616513078.444061:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7cc80 x1688666344020608/t0(0) o400->9764264e-2853-fc9f-a4bd-aed3b3271a14@10.31.136.116@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513078.444072:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666344020608, offset 192 00000100:00000200:15.0:1616513078.444076:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666344020480 00000400:00000200:6.0:1616513078.444077:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.116@tcp1 00000400:00000200:6.0:1616513078.444083:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.116@tcp1 routed destination 00010000:00000200:15.0:1616513078.444086:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688666344020480/t0(0) o400->74c21b66-86a1-fe02-faab-c1859bfa42dc@10.31.136.116@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:6.0:1616513078.444094:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.116@tcp1(10.31.136.116@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513078.444100:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:15.0:1616513078.444101:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666344020480, offset 192 00000800:00000200:6.0:1616513078.444104:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513078.444106:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:15.0:1616513078.444107:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.116@tcp1 00000800:00000200:6.0:1616513078.444108:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513078.444112:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:15.0:1616513078.444115:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.116@tcp1 routed destination 00000800:00000200:7.2:1616513078.444116:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:15.0:1616513078.444130:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.116@tcp1(10.31.136.116@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.444137:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.444143:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.444146:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.444148:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.444153:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.444159:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.444160:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513078.444171:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.444178:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.444181:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513078.444183:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.444188:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.444192:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01100 00000800:00000200:26.0:1616513078.444196:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.444213:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.444222:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.444225:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.444232:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.444238:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82550 00000800:00000200:28.0:1616513078.444241:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.444244:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.452296:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.452354:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.452364:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.452364:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513078.452371:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.452373:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.243@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.452378:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.243@tcp1 : PUT - for me 00000800:00000200:18.0:1616513078.452378:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513078.452383:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.243@tcp1 of length 224 into portal 26 MB=0x5ffdd166085c0 00000400:00000200:30.0:1616513078.452385:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.243@tcp1 of length 224 into portal 12 MB=0x5ffdd16608680 00000400:00000200:22.0:1616513078.452390:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.243@tcp1 of length 224/224 into md 0xf97c78c9 [1] + 0 00000400:00000200:22.0:1616513078.452395:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7540 00000400:00000200:22.0:1616513078.452398:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.452399:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.243@tcp1 of length 224/224 into md 0xf96450b9 [1] + 77184 00000400:00000200:30.0:1616513078.452403:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.452404:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513078.452408:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7540 00000100:00000200:30.0:1616513078.452411:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513078.452412:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.452414:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.452416:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.452416:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.452418:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.452420:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513078.452431:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699911833216 00010000:00000200:6.0:1616513078.452445:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a79b00 x1688699911833216/t0(0) o400->af48b774-975f-fe82-63b4-05e2f3b02533@10.31.140.243@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513078.452463:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699911833216, offset 192 00000400:00000200:6.0:1616513078.452467:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.243@tcp1 00000400:00000200:6.0:1616513078.452472:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.243@tcp1 routed destination 00000100:00000200:15.0:1616513078.452474:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699911833024 00000400:00000200:6.0:1616513078.452483:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.243@tcp1(10.31.140.243@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:15.0:1616513078.452485:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688699911833024/t0(0) o400->7bb95d09-1e0d-026e-52c2-fe5bd310fbee@10.31.140.243@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:6.0:1616513078.452488:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:6.0:1616513078.452492:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513078.452494:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513078.452496:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.452499:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699911833024, offset 192 00000800:00000200:6.0:1616513078.452499:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.452504:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:15.0:1616513078.452506:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.243@tcp1 00000400:00000200:15.0:1616513078.452514:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.243@tcp1 routed destination 00000400:00000200:15.0:1616513078.452532:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.243@tcp1(10.31.140.243@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.452539:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.452545:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.452548:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.452550:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513078.452552:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513078.452555:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513078.452559:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513078.452560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.452562:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513078.452563:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.452564:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.452571:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.452575:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00880 00000800:00000200:26.0:1616513078.452579:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513078.452616:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.452623:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.452627:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.452633:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.452639:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82088 00000800:00000200:24.0:1616513078.452643:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.452646:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.458460:0:49278:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.458515:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.458525:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.458525:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.458534:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.235@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.458545:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.235@tcp1 of length 224 into portal 26 MB=0x6050bbcba3900 00000400:00000200:22.0:1616513078.458552:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.235@tcp1 of length 224/224 into md 0xf97c78d9 [1] + 0 00000400:00000200:22.0:1616513078.458557:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6ff0 00000400:00000200:22.0:1616513078.458559:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.458562:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513078.458567:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6ff0 00000800:00000200:22.0:1616513078.458570:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.458573:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.458577:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.458628:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694397829363968 00010000:00000200:15.0:1616513078.458638:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1694397829363968/t0(0) o400->7c91e594-c7e4-2a72-2a81-0b803300062d@10.31.140.235@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.458652:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694397829363968, offset 192 00000400:00000200:15.0:1616513078.458657:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.235@tcp1 00000400:00000200:15.0:1616513078.458665:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.235@tcp1 routed destination 00000400:00000200:15.0:1616513078.458680:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.235@tcp1(10.31.140.235@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.458686:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.458691:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.458694:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.458696:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.458700:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.458705:0:49278:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.458707:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513078.458742:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513078.458759:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.458767:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.458770:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.458777:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.458783:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83100 00000800:00000200:18.0:1616513078.458787:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.458790:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.458797:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.458807:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.458816:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.235@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.458825:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.235@tcp1 of length 224 into portal 12 MB=0x6050bbcba39c0 00000400:00000200:28.0:1616513078.458832:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.235@tcp1 of length 224/224 into md 0x865bbec1 [1] + 10080 00000400:00000200:28.0:1616513078.458844:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.458846:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513078.458854:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.458857:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.458859:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.458862:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.458920:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694397829364160 00010000:00000200:0.0:1616513078.458932:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6721200 x1694397829364160/t0(0) o400->e58f4bbd-1564-a5c8-5744-14e27c789572@10.31.140.235@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.458947:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694397829364160, offset 192 00000400:00000200:0.0:1616513078.458954:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.235@tcp1 00000400:00000200:0.0:1616513078.458961:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.235@tcp1 routed destination 00000400:00000200:0.0:1616513078.458976:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.235@tcp1(10.31.140.235@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.458983:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.458989:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.458992:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.458994:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.458999:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.459004:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.459058:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.459065:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.459068:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513078.459069:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.459075:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.459078:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:24.0:1616513078.459083:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.472952:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513078.472965:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.472972:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513078.472980:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.202@tcp1 : PUT - for me 00000800:00000200:11.2:1616513078.472984:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:30.0:1616513078.472988:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.202@tcp1 of length 224 into portal 12 MB=0x605a0732186c0 00000400:00000200:30.0:1616513078.472994:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.202@tcp1 of length 224/224 into md 0xf96450b9 [1] + 77408 00000400:00000200:30.0:1616513078.472999:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513078.473001:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513078.473008:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.473011:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.473012:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.473015:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513078.473041:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.473051:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.473060:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.202@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.473069:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.202@tcp1 of length 224 into portal 26 MB=0x605a073218600 00000400:00000200:20.0:1616513078.473076:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.202@tcp1 of length 224/224 into md 0xf97c78e9 [1] + 0 00000100:00000200:10.0:1616513078.473076:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036544747200 00000400:00000200:20.0:1616513078.473081:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c66e8 00000400:00000200:20.0:1616513078.473084:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.473087:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00010000:00000200:10.0:1616513078.473088:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7e780 x1695036544747200/t0(0) o400->48825b8f-89e8-8520-c817-55b6b32b296b@10.31.129.202@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513078.473092:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c66e8 00000800:00000200:28.0:1616513078.473095:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.473097:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.473100:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.473102:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513078.473103:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036544747200, offset 192 00000400:00000200:10.0:1616513078.473109:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.202@tcp1 00000400:00000200:10.0:1616513078.473117:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.202@tcp1 routed destination 00000400:00000200:10.0:1616513078.473131:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.202@tcp1(10.31.129.202@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513078.473138:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513078.473144:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513078.473147:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513078.473149:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513078.473154:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.473156:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036544747008 00000800:00000200:7.2:1616513078.473159:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:15.0:1616513078.473167:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1695036544747008/t0(0) o400->95fb8adc-ef55-f755-8414-e79f81f098e2@10.31.129.202@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.473181:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036544747008, offset 192 00000400:00000200:15.0:1616513078.473187:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.202@tcp1 00000400:00000200:15.0:1616513078.473196:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.202@tcp1 routed destination 00000400:00000200:15.0:1616513078.473212:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.202@tcp1(10.31.129.202@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.473218:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513078.473222:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513078.473224:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513078.473228:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:15.0:1616513078.473229:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513078.473231:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:15.0:1616513078.473231:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.473236:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513078.473237:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.473241:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83870 00000800:00000200:11.2:1616513078.473241:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.473243:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513078.473245:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.473247:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.473252:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.473258:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.473261:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513078.473261:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513078.473268:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:30.0:1616513078.473270:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513078.473274:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83a90 00000800:00000200:30.0:1616513078.473275:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513078.473277:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513078.473278:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.473280:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513078.473282:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.213@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.473289:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.213@tcp1 of length 224 into portal 12 MB=0x5ffe04463c2c0 00000400:00000200:30.0:1616513078.473294:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.213@tcp1 of length 224/224 into md 0xf96450b9 [1] + 77632 00000400:00000200:30.0:1616513078.473297:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513078.473299:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513078.473303:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.473306:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.473307:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513078.473316:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688713568699072 00010000:00000200:10.0:1616513078.473325:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7f980 x1688713568699072/t0(0) o400->cef19c3a-c982-07ab-a293-3addd7fbee7e@10.31.142.213@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513078.473336:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688713568699072, offset 192 00000400:00000200:10.0:1616513078.473342:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.213@tcp1 00000400:00000200:10.0:1616513078.473348:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.213@tcp1 routed destination 00000400:00000200:10.0:1616513078.473361:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.213@tcp1(10.31.142.213@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513078.473367:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513078.473372:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513078.473374:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513078.473376:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513078.473379:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.473384:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.473394:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.473397:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.473399:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513078.473402:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.473404:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.473407:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e825d8 00000800:00000200:18.0:1616513078.473409:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.473487:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513078.473496:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.473502:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513078.473506:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.473510:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.213@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.473519:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.213@tcp1 of length 224 into portal 26 MB=0x5ffe04463c0c0 00000400:00000200:28.0:1616513078.473526:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.213@tcp1 of length 224/224 into md 0xf97c78f9 [1] + 0 00000400:00000200:28.0:1616513078.473530:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6a18 00000400:00000200:28.0:1616513078.473533:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.473538:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.473543:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6a18 00000800:00000200:28.0:1616513078.473546:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.473549:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.473550:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.473557:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688713568698560 00010000:00000200:15.0:1616513078.473563:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688713568698560/t0(0) o400->d915ccbf-d9e4-9136-c202-e12a8800c864@10.31.142.213@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.473574:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688713568698560, offset 192 00000400:00000200:15.0:1616513078.473578:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.213@tcp1 00000400:00000200:15.0:1616513078.473585:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.213@tcp1 routed destination 00000400:00000200:15.0:1616513078.473598:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.213@tcp1(10.31.142.213@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.473604:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.473608:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.473611:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.473612:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.473616:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.473619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.473620:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513078.473628:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.473633:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.473635:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.473638:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.473640:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.473644:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd835c8 00000800:00000200:28.0:1616513078.473647:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.474511:0:49280:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.474597:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.474607:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.474607:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513078.474612:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.474616:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.46@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.474618:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.46@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.474627:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.46@tcp1 of length 224 into portal 12 MB=0x5ffdb5ea9f040 00000400:00000200:22.0:1616513078.474627:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.46@tcp1 of length 224 into portal 26 MB=0x5ffdb5ea9efc0 00000400:00000200:30.0:1616513078.474631:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.46@tcp1 of length 224/224 into md 0xf96450b9 [1] + 77856 00000400:00000200:30.0:1616513078.474635:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513078.474638:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.46@tcp1 of length 224/224 into md 0xf97c7919 [1] + 0 00000400:00000200:22.0:1616513078.474643:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7320 00000100:00000200:30.0:1616513078.474644:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:22.0:1616513078.474645:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513078.474648:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000100:00000200:22.0:1616513078.474649:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:30.0:1616513078.474650:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513078.474651:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.474653:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.474654:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7320 00000800:00000200:22.0:1616513078.474657:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.474659:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.474661:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.474665:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692534669248 00010000:00000200:15.0:1616513078.474671:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688692534669248/t0(0) o400->fb0cbaf1-1772-5ab5-fb9c-55c9b50a5ded@10.31.141.46@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.474682:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692534669248, offset 192 00000400:00000200:15.0:1616513078.474699:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.46@tcp1 00000400:00000200:15.0:1616513078.474705:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.46@tcp1 routed destination 00000400:00000200:15.0:1616513078.474717:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.46@tcp1(10.31.141.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.474722:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.474726:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:10.0:1616513078.474726:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692534669376 00000800:00000200:15.0:1616513078.474729:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.474730:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.474733:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.474737:0:49280:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.474738:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:10.0:1616513078.474739:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a79f80 x1688692534669376/t0(0) o400->7a39da77-7778-6079-075c-55bf52125b75@10.31.141.46@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513078.474753:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692534669376, offset 192 00000400:00000200:10.0:1616513078.474759:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.46@tcp1 00000400:00000200:10.0:1616513078.474767:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.46@tcp1 routed destination 00000400:00000200:10.0:1616513078.474782:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.46@tcp1(10.31.141.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513078.474789:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513078.474790:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513078.474795:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513078.474798:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513078.474799:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513078.474800:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513078.474801:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.474803:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513078.474805:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513078.474809:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513078.474809:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513078.474815:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82220 00000800:00000200:18.0:1616513078.474819:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513078.474861:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.474868:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.474871:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.474877:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.474881:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e826e8 00000800:00000200:20.0:1616513078.474885:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.474888:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.476316:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.476370:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.476379:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.476390:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.72@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.476399:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.72@tcp1 of length 224 into portal 26 MB=0x601838aa1f3c0 00000400:00000200:24.0:1616513078.476406:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.72@tcp1 of length 224/224 into md 0xf97c7929 [1] + 0 00000400:00000200:24.0:1616513078.476411:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7dc0 00000400:00000200:24.0:1616513078.476414:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.476417:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.476421:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7dc0 00000800:00000200:28.0:1616513078.476424:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.476425:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.476427:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.476429:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.476483:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514338476992 00010000:00000200:15.0:1616513078.476493:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1690514338476992/t0(0) o400->50ed2a58-6127-5273-ee65-be025a769878@10.31.136.72@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.476507:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514338476992, offset 192 00000400:00000200:15.0:1616513078.476513:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.72@tcp1 00000400:00000200:15.0:1616513078.476521:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.72@tcp1 routed destination 00000400:00000200:15.0:1616513078.476536:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.72@tcp1(10.31.136.72@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.476542:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.476547:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.476550:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.476552:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.476556:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.476561:0:49280:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.476563:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513078.476620:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.476627:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.476630:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.476637:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.476643:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82220 00000800:00000200:26.0:1616513078.476647:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.476650:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.476982:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.477035:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.477044:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513078.477045:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.477052:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.72@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.477061:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.72@tcp1 of length 224 into portal 12 MB=0x601838aa1f5c0 00000400:00000200:20.0:1616513078.477068:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.72@tcp1 of length 224/224 into md 0x865bbec1 [1] + 10304 00000400:00000200:20.0:1616513078.477073:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.477076:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513078.477083:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.477086:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.477088:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.477155:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514338477504 00010000:00000200:0.0:1616513078.477168:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e04c80 x1690514338477504/t0(0) o400->bfd70e77-a3a2-ccbc-0d55-85bb86405fcf@10.31.136.72@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.477182:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514338477504, offset 192 00000400:00000200:0.0:1616513078.477189:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.72@tcp1 00000400:00000200:0.0:1616513078.477196:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.72@tcp1 routed destination 00000400:00000200:0.0:1616513078.477212:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.72@tcp1(10.31.136.72@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.477218:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.477224:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.477227:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.477231:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.477236:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.477241:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.477252:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.477257:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.477260:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.477262:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.477266:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.477270:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04110 00000800:00000200:28.0:1616513078.477274:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.490687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513078.490743:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.490753:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.490762:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.3@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.490772:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.3@tcp1 of length 224 into portal 26 MB=0x5ffd48ae18d00 00000400:00000200:28.0:1616513078.490779:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.3@tcp1 of length 224/224 into md 0xf97c7949 [1] + 0 00000400:00000200:28.0:1616513078.490784:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6110 00000400:00000200:28.0:1616513078.490787:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.490790:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.490794:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6110 00000800:00000200:28.0:1616513078.490797:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.490797:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.490801:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.490802:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.490859:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663211740416 00010000:00000200:15.0:1616513078.490870:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688663211740416/t0(0) o400->706f3c5f-9bcb-1a61-2f02-192f07274f54@10.31.138.3@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.490884:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663211740416, offset 192 00000400:00000200:15.0:1616513078.490890:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.3@tcp1 00000400:00000200:15.0:1616513078.490899:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.3@tcp1 routed destination 00000400:00000200:15.0:1616513078.490914:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.3@tcp1(10.31.138.3@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.490921:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.490927:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.490930:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.490932:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.490937:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.490944:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.490944:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.490957:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.490962:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.490965:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.490971:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.490977:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83a90 00000800:00000200:30.0:1616513078.490981:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.490983:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.491009:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.491070:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.491079:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513078.491081:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.491088:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.3@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.491097:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.3@tcp1 of length 224 into portal 12 MB=0x5ffd48ae18e00 00000400:00000200:24.0:1616513078.491103:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.3@tcp1 of length 224/224 into md 0x865bbec1 [1] + 10528 00000400:00000200:24.0:1616513078.491108:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.491111:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513078.491117:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.491121:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.491123:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513078.491137:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663211740672 00010000:00000200:0.0:1616513078.491149:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc4c80 x1688663211740672/t0(0) o400->a00e7138-c146-b892-de56-7043dd81952d@10.31.138.3@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.491163:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663211740672, offset 192 00000400:00000200:0.0:1616513078.491169:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.3@tcp1 00000400:00000200:0.0:1616513078.491176:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.3@tcp1 routed destination 00000400:00000200:0.0:1616513078.491190:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.3@tcp1(10.31.138.3@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.491197:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.491202:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.491205:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.491207:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.491212:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.491217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.491227:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.491232:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.491234:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513078.491235:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.491240:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.491244:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb057e8 00000800:00000200:20.0:1616513078.491248:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.497986:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513078.498039:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.498049:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.498057:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.61@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.498065:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.61@tcp1 of length 224 into portal 26 MB=0x5f90a1c42a340 00000400:00000200:28.0:1616513078.498072:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.61@tcp1 of length 224/224 into md 0xf97c7969 [1] + 0 00000400:00000200:28.0:1616513078.498076:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6550 00000400:00000200:28.0:1616513078.498079:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.498081:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.498086:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6550 00000800:00000200:28.0:1616513078.498088:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.498091:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.498093:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.498095:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.498122:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:15.0:1616513078.498147:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681196702671680 00010000:00000200:15.0:1616513078.498157:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1681196702671680/t0(0) o400->784ed8f8-5a39-6574-9201-203eb5ca0038@10.31.131.61@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.498170:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681196702671680, offset 192 00000800:00000200:26.0:1616513078.498178:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:15.0:1616513078.498178:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.61@tcp1 00000400:00000200:15.0:1616513078.498185:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.61@tcp1 routed destination 00000800:00000200:26.0:1616513078.498188:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:15.0:1616513078.498200:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.61@tcp1(10.31.131.61@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.498207:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513078.498209:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.61@tcp1 : PUT - for me 00000800:00000200:15.0:1616513078.498213:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.498216:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513078.498218:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:26.0:1616513078.498219:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.61@tcp1 of length 224 into portal 12 MB=0x5f90a1c42a3c0 00000800:00000200:15.0:1616513078.498223:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513078.498225:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.61@tcp1 of length 224/224 into md 0xf96450b9 [1] + 78080 00000100:00000200:15.0:1616513078.498229:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513078.498230:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513078.498233:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513078.498234:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513078.498240:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:22.0:1616513078.498242:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:26.0:1616513078.498243:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:30.0:1616513078.498245:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513078.498246:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513078.498247:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.498252:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513078.498259:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82088 00000800:00000200:22.0:1616513078.498262:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513078.498305:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681196702671808 00010000:00000200:10.0:1616513078.498318:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dc380 x1681196702671808/t0(0) o400->ceb68e41-d4af-2e8d-4f71-c7ef2f17a76d@10.31.131.61@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513078.498333:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681196702671808, offset 192 00000400:00000200:10.0:1616513078.498339:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.61@tcp1 00000400:00000200:10.0:1616513078.498347:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.61@tcp1 routed destination 00000400:00000200:10.0:1616513078.498362:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.61@tcp1(10.31.131.61@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513078.498370:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513078.498376:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513078.498379:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513078.498381:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513078.498386:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.498390:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.498399:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.498402:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.498405:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.498410:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.498413:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82dd0 00000800:00000200:28.0:1616513078.498415:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.498418:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.527721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513078.527788:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.527797:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.527807:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.40@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.527818:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.40@tcp1 of length 224 into portal 26 MB=0x5ffd51df72d40 00000400:00000200:28.0:1616513078.527825:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.40@tcp1 of length 224/224 into md 0xf97c7979 [1] + 0 00000400:00000200:28.0:1616513078.527830:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7ba0 00000400:00000200:28.0:1616513078.527833:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.527836:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.527841:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7ba0 00000800:00000200:24.0:1616513078.527843:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.527844:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.527847:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513078.527851:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.527854:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.527855:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.527864:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.40@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.527873:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.40@tcp1 of length 224 into portal 12 MB=0x5ffd51df72dc0 00000400:00000200:24.0:1616513078.527880:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.40@tcp1 of length 224/224 into md 0x865bbec1 [1] + 10752 00000400:00000200:24.0:1616513078.527886:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.527888:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513078.527896:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.527899:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.527901:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.527910:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665679408448 00010000:00000200:15.0:1616513078.527921:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688665679408448/t0(0) o400->dc34791c-8ab0-3ecc-024b-1705919d1360@10.31.136.40@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.527935:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665679408448, offset 192 00000400:00000200:15.0:1616513078.527942:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.40@tcp1 00000400:00000200:15.0:1616513078.527950:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.40@tcp1 routed destination 00000400:00000200:15.0:1616513078.527966:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.40@tcp1(10.31.136.40@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513078.527970:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665679408576 00000800:00000200:15.0:1616513078.527972:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.527979:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.527982:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.527984:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:4.0:1616513078.527984:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc0900 x1688665679408576/t0(0) o400->39abcc5f-d15e-be66-6190-367705608a7f@10.31.136.40@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:15.0:1616513078.527989:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.527994:0:49286:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.527996:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:4.0:1616513078.527999:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665679408576, offset 192 00000400:00000200:4.0:1616513078.528005:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.40@tcp1 00000400:00000200:4.0:1616513078.528014:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.40@tcp1 routed destination 00000400:00000200:4.0:1616513078.528029:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.40@tcp1(10.31.136.40@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.528037:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.528043:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.528048:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513078.528049:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513078.528050:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.528055:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513078.528057:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.528060:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.528060:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513078.528060:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513078.528067:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.528073:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83f58 00000800:00000200:18.0:1616513078.528077:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513078.528113:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.528120:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.528123:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513078.528124:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.528129:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.528133:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb188 00000800:00000200:20.0:1616513078.528137:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513078.554775:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925184, portal 4 00000100:00000200:14.0:1616513078.554781:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925184, offset 0 00000400:00000200:14.0:1616513078.554788:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:14.0:1616513078.554795:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:14.0:1616513078.554808:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.554814:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:14.0:1616513078.554822:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.554825:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:14.0:1616513078.554827:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:14.0:1616513078.554833:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513078.554880:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.554900:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513078.554909:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513078.554913:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513078.554920:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513078.554928:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d7464621f80 x1693067725925184/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513078.554940:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47566e8 00000800:00000200:1.0:1616513078.554945:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513078.554949:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513078.555049:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513078.555060:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.555067:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513078.555070:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513078.555075:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513078.555084:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626b40 00000400:00000200:1.0:1616513078.555092:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffb05 [1] + 192 00000400:00000200:1.0:1616513078.555097:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513078.555102:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d7464621f80 x1693067725925184/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513078.555116:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.555119:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513078.555121:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:14.0:1616513078.555126:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d7464621f80 x1693067725925184/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513078.555139:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d7464621f80 x1693067725925184/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513078.555152:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47564c8 00000800:00000200:11.2:1616513078.575021:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.575078:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.575088:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513078.575098:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.63@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.575108:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.63@tcp1 of length 224 into portal 26 MB=0x5ffda1c8f7b00 00000400:00000200:0.0:1616513078.575115:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.63@tcp1 of length 224/224 into md 0xf97c7999 [1] + 0 00000400:00000200:0.0:1616513078.575120:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7870 00000400:00000200:0.0:1616513078.575123:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513078.575126:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513078.575131:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7870 00000800:00000200:24.0:1616513078.575132:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.575135:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.575138:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.575140:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.575197:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688687130671872 00010000:00000200:15.0:1616513078.575208:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688687130671872/t0(0) o400->1136adcb-c07d-b618-7b7c-74b0a504d8f9@10.31.129.63@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.575234:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688687130671872, offset 192 00000400:00000200:15.0:1616513078.575240:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.63@tcp1 00000400:00000200:15.0:1616513078.575248:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.63@tcp1 routed destination 00000400:00000200:15.0:1616513078.575262:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.63@tcp1(10.31.129.63@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.575268:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.575277:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.575280:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.575282:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.575286:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.575292:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.575293:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513078.575348:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.575356:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.575359:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.575366:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.575373:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82330 00000800:00000200:26.0:1616513078.575377:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.575380:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.575386:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513078.575390:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.575394:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.63@tcp1 : PUT - for me 00000400:00000200:26.0:1616513078.575403:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.63@tcp1 of length 224 into portal 12 MB=0x5ffda1c8f7f40 00000400:00000200:26.0:1616513078.575410:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.63@tcp1 of length 224/224 into md 0xf96450b9 [1] + 78304 00000400:00000200:26.0:1616513078.575417:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513078.575420:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513078.575426:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.575429:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.575431:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513078.575503:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688687130672960 00010000:00000200:10.0:1616513078.575517:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8000 x1688687130672960/t0(0) o400->7ed81503-5421-8362-0895-ad7f37e9cad6@10.31.129.63@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513078.575531:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688687130672960, offset 192 00000400:00000200:10.0:1616513078.575537:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.63@tcp1 00000400:00000200:10.0:1616513078.575545:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.63@tcp1 routed destination 00000400:00000200:10.0:1616513078.575560:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.63@tcp1(10.31.129.63@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513078.575566:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513078.575572:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513078.575574:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513078.575576:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513078.575582:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.575587:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.575641:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.575650:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.575653:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.575659:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.575663:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e837e8 00000800:00000200:28.0:1616513078.575667:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.575670:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.576005:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.576016:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.576025:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.576027:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.576034:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.31@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.576049:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.31@tcp1 of length 224 into portal 26 MB=0x60186d959ffc0 00000400:00000200:24.0:1616513078.576056:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.31@tcp1 of length 224/224 into md 0xf97c79b9 [1] + 0 00000400:00000200:24.0:1616513078.576061:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6660 00000400:00000200:24.0:1616513078.576063:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.576066:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.576072:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6660 00000800:00000200:24.0:1616513078.576075:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.576078:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.576079:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.576086:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690528544063424 00010000:00000200:15.0:1616513078.576092:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1690528544063424/t0(0) o400->5100e0b1-726f-13af-1099-9f769f315dd4@10.31.141.31@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.576103:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690528544063424, offset 192 00000400:00000200:15.0:1616513078.576108:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.31@tcp1 00000400:00000200:15.0:1616513078.576114:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.31@tcp1 routed destination 00000400:00000200:15.0:1616513078.576126:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.31@tcp1(10.31.141.31@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.576131:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.576136:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.576139:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.576141:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.576144:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.576148:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.576149:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513078.576188:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513078.576198:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513078.576202:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513078.576204:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513078.576207:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513078.576210:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.576212:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513078.576212:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.31@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.576213:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.576220:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.31@tcp1 of length 224 into portal 12 MB=0x60186d95a0300 00000400:00000200:18.0:1616513078.576220:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.576226:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.31@tcp1 of length 224/224 into md 0x865bbec1 [1] + 10976 00000400:00000200:18.0:1616513078.576226:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd824c8 00000800:00000200:18.0:1616513078.576229:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513078.576231:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.576233:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513078.576239:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.576242:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.576244:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513078.576304:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690528544064256 00010000:00000200:4.0:1616513078.576317:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6727980 x1690528544064256/t0(0) o400->9322e70f-9cfa-8074-3073-0e40e2c94270@10.31.141.31@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513078.576332:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690528544064256, offset 192 00000400:00000200:4.0:1616513078.576339:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.31@tcp1 00000400:00000200:4.0:1616513078.576346:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.31@tcp1 routed destination 00000400:00000200:4.0:1616513078.576361:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.31@tcp1(10.31.141.31@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.576368:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.576374:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.576377:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.576379:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.576384:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.576388:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.576397:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.576400:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513078.576403:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.576405:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513078.576408:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513078.576412:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba7f8 00000800:00000200:0.0:1616513078.576416:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.588411:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.588481:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.588491:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.588500:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.171@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.588510:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.171@tcp1 of length 224 into portal 26 MB=0x5ffd3eff55140 00000400:00000200:22.0:1616513078.588517:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.171@tcp1 of length 224/224 into md 0xf97c79c9 [1] + 0 00000400:00000200:22.0:1616513078.588522:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6bb0 00000400:00000200:22.0:1616513078.588524:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.588527:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513078.588532:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6bb0 00000800:00000200:26.0:1616513078.588535:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.588536:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.588538:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513078.588540:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.588554:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660612567360 00010000:00000200:15.0:1616513078.588564:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688660612567360/t0(0) o400->105d82ba-a0e2-c4a0-cf43-001d06cdbc13@10.31.128.171@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.588577:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660612567360, offset 192 00000400:00000200:15.0:1616513078.588583:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.171@tcp1 00000400:00000200:15.0:1616513078.588592:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.171@tcp1 routed destination 00000400:00000200:15.0:1616513078.588606:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.171@tcp1(10.31.128.171@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.588613:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.588619:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.588622:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.588624:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.588628:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.588633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.588637:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513078.588643:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.588649:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.588651:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.588658:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.588663:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82e58 00000800:00000200:30.0:1616513078.588667:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.588669:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513078.588767:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513078.588778:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.588786:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513078.588788:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.588795:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.171@tcp1 : PUT - for me 00000400:00000200:26.0:1616513078.588805:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.171@tcp1 of length 224 into portal 12 MB=0x5ffd3eff55600 00000400:00000200:26.0:1616513078.588811:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.171@tcp1 of length 224/224 into md 0xf96450b9 [1] + 78528 00000400:00000200:26.0:1616513078.588817:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513078.588819:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513078.588828:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.588831:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.588833:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.588896:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660612568576 00010000:00000200:14.0:1616513078.588909:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647db180 x1688660612568576/t0(0) o400->cc8b766e-2149-21b3-433e-c0279b5aee2c@10.31.128.171@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.588924:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660612568576, offset 192 00000400:00000200:14.0:1616513078.588930:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.171@tcp1 00000400:00000200:14.0:1616513078.588938:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.171@tcp1 routed destination 00000400:00000200:14.0:1616513078.588954:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.171@tcp1(10.31.128.171@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.588961:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.588967:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.588970:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.588972:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.588977:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.589024:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.589080:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.589087:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.589090:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.589096:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.589100:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47564c8 00000800:00000200:20.0:1616513078.589104:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.589107:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.597891:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.597946:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.597956:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.597965:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.64@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.597973:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.64@tcp1 of length 224 into portal 26 MB=0x5f908c9faf000 00000400:00000200:24.0:1616513078.597980:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.64@tcp1 of length 224/224 into md 0xf97c79e9 [1] + 0 00000400:00000200:24.0:1616513078.597985:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7650 00000400:00000200:24.0:1616513078.597987:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.597990:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.597995:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7650 00000800:00000200:24.0:1616513078.597998:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.598000:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.598001:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.598007:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.598010:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.598010:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513078.598019:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.64@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.598028:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.64@tcp1 of length 224 into portal 12 MB=0x5f908c9faf080 00000400:00000200:0.0:1616513078.598034:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.64@tcp1 of length 224/224 into md 0x865bbec1 [1] + 11200 00000400:00000200:0.0:1616513078.598039:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513078.598042:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513078.598049:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.598053:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.598054:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.598057:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681191027273728 00010000:00000200:15.0:1616513078.598068:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1681191027273728/t0(0) o400->2c5da2d6-33e4-284d-1b7f-8d6bc2a58540@10.31.131.64@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513078.598071:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681191027273856 00000100:00000200:15.0:1616513078.598081:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681191027273728, offset 192 00010000:00000200:4.0:1616513078.598084:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b194c80 x1681191027273856/t0(0) o400->0ccca63c-abf1-ace5-80b3-6e2bccac9795@10.31.131.64@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513078.598087:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.64@tcp1 00000400:00000200:15.0:1616513078.598094:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.64@tcp1 routed destination 00000100:00000200:4.0:1616513078.598099:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681191027273856, offset 192 00000400:00000200:4.0:1616513078.598106:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.64@tcp1 00000400:00000200:15.0:1616513078.598109:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.64@tcp1(10.31.131.64@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:4.0:1616513078.598113:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.64@tcp1 routed destination 00000800:00000200:15.0:1616513078.598115:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.598122:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.598125:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.598127:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513078.598129:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.64@tcp1(10.31.131.64@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.598131:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513078.598136:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513078.598137:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.598138:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513078.598142:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.598145:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513078.598148:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513078.598148:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.598152:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513078.598153:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.598155:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513078.598157:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513078.598162:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.598166:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82d48 00000800:00000200:30.0:1616513078.598169:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.598171:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.598222:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.598231:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.598234:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.598235:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.598241:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.598245:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba770 00000800:00000200:28.0:1616513078.598249:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513078.602582:0:6484:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925200, portal 4 00000100:00000200:20.0:1616513078.602587:0:6484:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925200, offset 0 00000400:00000200:20.0:1616513078.602594:0:6484:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:20.0:1616513078.602600:0:6484:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:20.0:1616513078.602613:0:6484:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513078.602619:0:6484:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:20.0:1616513078.602626:0:6484:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513078.602629:0:6484:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:20.0:1616513078.602631:0:6484:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:20.0:1616513078.602637:0:6484:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513078.602684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513078.602757:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513078.602766:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513078.602768:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513078.602775:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513078.602781:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affc5a00 x1693067725925200/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513078.602793:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee227f8 00000800:00000200:22.0:1616513078.602797:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513078.602800:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513078.602864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513078.602873:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513078.602878:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:30.0:1616513078.602885:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513078.602892:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626b50 00000400:00000200:30.0:1616513078.602898:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c8049 [1] + 192 00000400:00000200:30.0:1616513078.602903:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513078.602907:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affc5a00 x1693067725925200/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513078.602921:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:30.0:1616513078.602923:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:18.0:1616513078.602924:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (23)-- 00000800:00000200:30.0:1616513078.602928:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:20.0:1616513078.602930:0:6484:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affc5a00 x1693067725925200/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513078.602943:0:6484:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affc5a00 x1693067725925200/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513078.602953:0:6484:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23210 00000800:00000200:11.2:1616513078.603072:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.603084:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.603091:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.603099:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.132@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.603113:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.132@tcp1 of length 224 into portal 26 MB=0x603fc0e844e40 00000400:00000200:20.0:1616513078.603119:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.132@tcp1 of length 224/224 into md 0xf97c7a09 [1] + 0 00000400:00000200:20.0:1616513078.603124:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6880 00000400:00000200:20.0:1616513078.603126:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.603128:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513078.603133:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6880 00000800:00000200:20.0:1616513078.603138:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.603140:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.603142:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.603144:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.603149:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513078.603154:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.603156:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.132@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.603161:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.132@tcp1 of length 224 into portal 12 MB=0x603fc0e844ec0 00000400:00000200:20.0:1616513078.603166:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.132@tcp1 of length 224/224 into md 0x865bbec1 [1] + 11424 00000400:00000200:20.0:1616513078.603170:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.603172:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513078.603177:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.603179:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.603180:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513078.603186:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230970457792 00010000:00000200:4.0:1616513078.603192:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05580 x1693230970457792/t0(0) o400->2375b177-48d7-a952-972f-03761d8fa43e@10.31.138.132@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513078.603197:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230970457664 00010000:00000200:15.0:1616513078.603208:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1693230970457664/t0(0) o400->7b18bd2d-243b-2691-f94a-6530bb445e63@10.31.138.132@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513078.603212:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230970457792, offset 192 00000400:00000200:4.0:1616513078.603217:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.132@tcp1 00000100:00000200:15.0:1616513078.603221:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230970457664, offset 192 00000400:00000200:4.0:1616513078.603223:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.132@tcp1 routed destination 00000400:00000200:15.0:1616513078.603227:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.132@tcp1 00000400:00000200:15.0:1616513078.603236:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.132@tcp1 routed destination 00000400:00000200:4.0:1616513078.603237:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.132@tcp1(10.31.138.132@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.603244:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:15.0:1616513078.603251:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.132@tcp1(10.31.138.132@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.603258:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.603258:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.603260:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513078.603262:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.603263:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.603265:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:15.0:1616513078.603266:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.603268:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513078.603272:0:49295:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.603273:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.603277:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.603279:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513078.603283:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.603287:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513078.603289:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513078.603289:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513078.603292:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.603294:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513078.603296:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513078.603297:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6babb0 00000400:00000200:28.0:1616513078.603300:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513078.603300:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513078.603301:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.603306:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.603313:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83650 00000800:00000200:28.0:1616513078.603316:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.606444:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.606509:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.606518:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513078.606519:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.606526:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.606527:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.243@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.606534:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.243@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.606536:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.243@tcp1 of length 224 into portal 26 MB=0x6000c45d49b80 00000400:00000200:30.0:1616513078.606541:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.243@tcp1 of length 224 into portal 12 MB=0x6000c45d49c80 00000400:00000200:18.0:1616513078.606543:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.243@tcp1 of length 224/224 into md 0xf97c7a19 [1] + 0 00000400:00000200:18.0:1616513078.606548:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6000 00000400:00000200:18.0:1616513078.606551:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.606553:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.243@tcp1 of length 224/224 into md 0xf96450b9 [1] + 78752 00000400:00000200:30.0:1616513078.606557:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.606557:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513078.606562:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6000 00000100:00000200:30.0:1616513078.606565:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.606565:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513078.606568:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513078.606571:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513078.606571:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.606573:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513078.606575:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.606577:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.606624:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688902571432832 00010000:00000200:15.0:1616513078.606634:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688902571432832/t0(0) o400->28c37073-8a15-27d7-7be8-375a88fd2ee9@10.31.141.243@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:14.0:1616513078.606642:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688902571433088 00000100:00000200:15.0:1616513078.606648:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688902571432832, offset 192 00010000:00000200:14.0:1616513078.606654:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7f500 x1688902571433088/t0(0) o400->9dfb3e89-57e4-1207-43db-da58c28c0b1d@10.31.141.243@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513078.606656:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.243@tcp1 00000400:00000200:15.0:1616513078.606663:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.243@tcp1 routed destination 00000100:00000200:14.0:1616513078.606667:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688902571433088, offset 192 00000400:00000200:14.0:1616513078.606673:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.243@tcp1 00000400:00000200:15.0:1616513078.606677:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.243@tcp1(10.31.141.243@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.606683:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513078.606686:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.243@tcp1 routed destination 00000800:00000200:15.0:1616513078.606689:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.606692:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.606694:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:14.0:1616513078.606700:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.243@tcp1(10.31.141.243@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.606703:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.606706:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513078.606707:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.606709:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513078.606712:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.606715:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.606717:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.606721:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.606726:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513078.606760:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.606768:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513078.606771:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.606771:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513078.606778:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513078.606783:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513078.606784:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd823b8 00000800:00000200:22.0:1616513078.606788:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513078.606790:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.606793:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.606799:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.606806:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47576d8 00000800:00000200:24.0:1616513078.606809:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.606812:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513078.618600:0:6484:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925216, portal 4 00000100:00000200:20.0:1616513078.618605:0:6484:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925216, offset 0 00000400:00000200:20.0:1616513078.618612:0:6484:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:20.0:1616513078.618618:0:6484:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:20.0:1616513078.618631:0:6484:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513078.618638:0:6484:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:20.0:1616513078.618644:0:6484:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513078.618647:0:6484:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:20.0:1616513078.618650:0:6484:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:20.0:1616513078.618655:0:6484:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513078.618705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513078.618726:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513078.618732:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513078.618735:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513078.618741:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513078.618748:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affd2880 x1693067725925216/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513078.618760:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee238f8 00000800:00000200:27.0:1616513078.618765:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513078.618768:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513078.618876:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513078.618886:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513078.618892:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:23.0:1616513078.618896:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513078.618900:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513078.618908:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626b60 00000400:00000200:27.0:1616513078.618914:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716688d [1] + 192 00000400:00000200:27.0:1616513078.618920:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513078.618924:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affd2880 x1693067725925216/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513078.618939:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513078.618942:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513078.618944:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:20.0:1616513078.618947:0:6484:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affd2880 x1693067725925216/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513078.618959:0:6484:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affd2880 x1693067725925216/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513078.618969:0:6484:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23210 00000100:00000200:13.0:1616513078.634678:0:6493:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925232, portal 4 00000100:00000200:13.0:1616513078.634689:0:6493:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925232, offset 0 00000400:00000200:13.0:1616513078.634695:0:6493:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:13.0:1616513078.634701:0:6493:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:13.0:1616513078.634714:0:6493:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:13.0:1616513078.634721:0:6493:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513078.634727:0:6493:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513078.634730:0:6493:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:13.0:1616513078.634732:0:6493:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:13.0:1616513078.634736:0:6493:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513078.634783:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.634803:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513078.634808:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513078.634811:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513078.634814:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513078.634818:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513078.634825:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843f7bc800 x1693067725925232/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513078.634836:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81783447f8 00000800:00000200:1.0:1616513078.634841:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513078.634960:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513078.634969:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.634976:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513078.634977:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513078.634984:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513078.634992:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626b70 00000400:00000200:1.0:1616513078.634998:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffb15 [1] + 192 00000400:00000200:1.0:1616513078.635006:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513078.635010:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843f7bc800 x1693067725925232/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513078.635024:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.635027:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513078.635029:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:13.0:1616513078.635031:0:6493:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843f7bc800 x1693067725925232/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:13.0:1616513078.635044:0:6493:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843f7bc800 x1693067725925232/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:13.0:1616513078.635054:0:6493:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345210 00000800:00000200:11.2:1616513078.636308:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.636365:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.636376:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513078.636385:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.180@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.636395:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.180@tcp1 of length 224 into portal 26 MB=0x5ffd490dec400 00000400:00000200:0.0:1616513078.636402:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.180@tcp1 of length 224/224 into md 0xf97c7a39 [1] + 0 00000400:00000200:0.0:1616513078.636407:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6e58 00000400:00000200:0.0:1616513078.636411:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513078.636414:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513078.636419:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6e58 00000800:00000200:28.0:1616513078.636420:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.636422:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.636425:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513078.636427:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513078.636430:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.636431:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.636467:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.180@tcp1 : PUT - for me 00000100:00000200:15.0:1616513078.636485:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663312221184 00000400:00000200:28.0:1616513078.636486:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.180@tcp1 of length 224 into portal 12 MB=0x5ffd490dec580 00000400:00000200:28.0:1616513078.636493:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.180@tcp1 of length 224/224 into md 0x865bbec1 [1] + 11648 00010000:00000200:15.0:1616513078.636496:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688663312221184/t0(0) o400->d030c25d-2e9c-e79b-c8bf-583d1642848f@10.31.131.180@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513078.636501:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.636503:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513078.636510:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663312221184, offset 192 00000800:00000200:28.0:1616513078.636511:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.636514:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.636516:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:15.0:1616513078.636517:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.180@tcp1 00000400:00000200:15.0:1616513078.636525:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.180@tcp1 routed destination 00000400:00000200:15.0:1616513078.636541:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.180@tcp1(10.31.131.180@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.636547:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.636554:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.636556:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.636559:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.636563:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.636570:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.636571:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513078.636575:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663312221568 00010000:00000200:4.0:1616513078.636588:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6725a00 x1688663312221568/t0(0) o400->f6b7d6b2-84d2-f8a9-9c06-0df413abd737@10.31.131.180@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513078.636602:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663312221568, offset 192 00000400:00000200:4.0:1616513078.636608:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.180@tcp1 00000400:00000200:4.0:1616513078.636616:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.180@tcp1 routed destination 00000400:00000200:4.0:1616513078.636630:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.180@tcp1(10.31.131.180@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.636636:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513078.636641:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513078.636642:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.636644:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.636646:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513078.636648:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.636651:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.636651:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513078.636651:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.636656:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513078.636657:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.636665:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd836d8 00000800:00000200:18.0:1616513078.636668:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513078.636719:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.636726:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.636728:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513078.636730:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.636734:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.636742:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba908 00000800:00000200:20.0:1616513078.636746:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:3.0:1616513078.650515:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925248, portal 4 00000100:00000200:3.0:1616513078.650521:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925248, offset 0 00000400:00000200:3.0:1616513078.650527:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:3.0:1616513078.650533:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:3.0:1616513078.650546:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513078.650552:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:3.0:1616513078.650558:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513078.650566:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:3.0:1616513078.650568:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:3.0:1616513078.650573:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513078.650618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.650635:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513078.650641:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513078.650643:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513078.650650:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513078.650657:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63bf00 x1693067725925248/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513078.650669:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2088 00000800:00000200:1.0:1616513078.650673:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513078.650676:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513078.650790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513078.650799:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.650804:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513078.650808:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513078.650823:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513078.650830:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626b80 00000400:00000200:1.0:1616513078.650835:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffb35 [1] + 192 00000400:00000200:1.0:1616513078.650840:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513078.650844:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63bf00 x1693067725925248/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513078.650856:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.650859:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513078.650860:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:3.0:1616513078.650865:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63bf00 x1693067725925248/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513078.650878:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63bf00 x1693067725925248/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513078.650888:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:11.2:1616513078.653258:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.653315:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.653325:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.653334:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.228@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.653343:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.228@tcp1 of length 224 into portal 26 MB=0x5fff34a254d40 00000400:00000200:24.0:1616513078.653350:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.228@tcp1 of length 224/224 into md 0xf97c7a49 [1] + 0 00000400:00000200:24.0:1616513078.653355:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2c38 00000400:00000200:24.0:1616513078.653358:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.653360:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.653365:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:28.0:1616513078.653369:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.653369:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.653371:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.653376:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.653378:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.653379:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.653388:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.228@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.653397:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.228@tcp1 of length 224 into portal 12 MB=0x5fff34a254dc0 00000400:00000200:28.0:1616513078.653403:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.228@tcp1 of length 224/224 into md 0x865bbec1 [1] + 11872 00000400:00000200:28.0:1616513078.653408:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.653411:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513078.653418:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.653421:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.653423:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.653431:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795269647680 00000100:00000200:4.0:1616513078.653445:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795269647808 00010000:00000200:15.0:1616513078.653453:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688795269647680/t0(0) o400->20c50f8b-0395-67e3-dfe2-65872cf15782@10.31.141.228@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:4.0:1616513078.653458:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6720900 x1688795269647808/t0(0) o400->730ae0a9-ba5d-ee8a-9e56-0aa29a85b8c2@10.31.141.228@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513078.653468:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688795269647680, offset 192 00000100:00000200:4.0:1616513078.653472:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688795269647808, offset 192 00000400:00000200:15.0:1616513078.653475:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.228@tcp1 00000400:00000200:4.0:1616513078.653478:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.228@tcp1 00000400:00000200:15.0:1616513078.653483:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.228@tcp1 routed destination 00000400:00000200:4.0:1616513078.653486:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.228@tcp1 routed destination 00000400:00000200:15.0:1616513078.653500:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.228@tcp1(10.31.141.228@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:4.0:1616513078.653502:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.228@tcp1(10.31.141.228@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.653511:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.653512:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513078.653517:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.653519:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.653520:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.653522:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513078.653522:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.653524:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513078.653527:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:15.0:1616513078.653529:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513078.653532:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.653536:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.653575:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513078.653585:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.653591:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.653594:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.653595:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.653600:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.653604:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb5c8 00000800:00000200:20.0:1616513078.653608:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513078.653633:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.653641:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.653644:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.653651:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.653657:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd837e8 00000800:00000200:26.0:1616513078.653661:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.653664:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.654488:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.654501:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513078.654507:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513078.654511:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513078.654512:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513078.654518:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513078.654520:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.109@tcp1 : PUT - for me 00000800:00000200:30.0:1616513078.654525:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513078.654529:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.109@tcp1 of length 224 into portal 12 MB=0x6019500f380c0 00000400:00000200:30.0:1616513078.654532:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.109@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.654536:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.109@tcp1 of length 224/224 into md 0x865bbec1 [1] + 12096 00000400:00000200:30.0:1616513078.654540:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.109@tcp1 of length 224 into portal 26 MB=0x6019500f38040 00000400:00000200:0.0:1616513078.654541:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.654546:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.109@tcp1 of length 224/224 into md 0xf97c7a59 [1] + 0 00000100:00000200:0.0:1616513078.654546:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:30.0:1616513078.654553:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6198 00000800:00000200:0.0:1616513078.654553:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513078.654555:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513078.654556:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:30.0:1616513078.654558:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1616513078.654558:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513078.654562:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6198 00000100:00000200:4.0:1616513078.654562:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589337977024 00000800:00000200:30.0:1616513078.654565:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.654568:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00010000:00000200:4.0:1616513078.654568:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b193600 x1690589337977024/t0(0) o400->db040549-914d-aa11-7596-90319d44f5d0@10.31.140.109@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:30.0:1616513078.654569:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.654571:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.654577:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589337976896 00000100:00000200:4.0:1616513078.654578:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690589337977024, offset 192 00000400:00000200:4.0:1616513078.654583:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.109@tcp1 00010000:00000200:15.0:1616513078.654584:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1690589337976896/t0(0) o400->dfa090c5-b2d0-3f73-43a1-6a352da12220@10.31.140.109@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:4.0:1616513078.654588:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.109@tcp1 routed destination 00000100:00000200:15.0:1616513078.654595:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690589337976896, offset 192 00000400:00000200:15.0:1616513078.654601:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.109@tcp1 00000400:00000200:4.0:1616513078.654603:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.109@tcp1(10.31.140.109@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:15.0:1616513078.654608:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.109@tcp1 routed destination 00000800:00000200:4.0:1616513078.654609:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513078.654614:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.654617:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513078.654618:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513078.654621:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:15.0:1616513078.654622:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.109@tcp1(10.31.140.109@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513078.654627:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.654628:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.654633:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513078.654636:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513078.654636:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.654638:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513078.654640:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:15.0:1616513078.654641:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513078.654642:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513078.654646:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.654646:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513078.654646:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513078.654648:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.654650:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb4b8 00000800:00000200:26.0:1616513078.654653:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513078.654657:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.654661:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.654664:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.654666:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.654670:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.654675:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd838f8 00000800:00000200:24.0:1616513078.654678:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.665443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513078.665456:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.665464:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513078.665472:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.60@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.665480:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.60@tcp1 of length 224 into portal 26 MB=0x60143c8567c40 00000400:00000200:30.0:1616513078.665486:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.60@tcp1 of length 224/224 into md 0xf97c7a69 [1] + 0 00000400:00000200:30.0:1616513078.665490:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6cc0 00000400:00000200:30.0:1616513078.665492:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513078.665495:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513078.665500:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6cc0 00000800:00000200:30.0:1616513078.665502:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.665505:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513078.665506:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.665509:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.665565:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690240495811648 00010000:00000200:15.0:1616513078.665576:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1690240495811648/t0(0) o400->51ec7790-edc3-e4e1-c898-836fd6983c5a@10.31.132.60@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.665590:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690240495811648, offset 192 00000400:00000200:15.0:1616513078.665596:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.60@tcp1 00000400:00000200:15.0:1616513078.665604:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.60@tcp1 routed destination 00000400:00000200:15.0:1616513078.665619:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.60@tcp1(10.31.132.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.665626:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.665632:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.665635:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.665639:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.665644:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.665651:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.665651:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.665705:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.665712:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513078.665715:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513078.665721:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513078.665728:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83b18 00000800:00000200:22.0:1616513078.665732:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.665735:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.665870:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513078.665925:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.665934:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.665943:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.60@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.665951:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.60@tcp1 of length 224 into portal 12 MB=0x60143c8567dc0 00000400:00000200:28.0:1616513078.665957:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.60@tcp1 of length 224/224 into md 0x865bbec1 [1] + 12320 00000400:00000200:28.0:1616513078.665962:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.665964:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513078.665970:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.665972:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.665974:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.665977:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513078.666030:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690240495812032 00010000:00000200:4.0:1616513078.666042:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6720d80 x1690240495812032/t0(0) o400->9f26f9a8-36dd-7f73-57f4-7ebab8b12d58@10.31.132.60@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513078.666055:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690240495812032, offset 192 00000400:00000200:4.0:1616513078.666061:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.60@tcp1 00000400:00000200:4.0:1616513078.666069:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.60@tcp1 routed destination 00000400:00000200:4.0:1616513078.666082:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.60@tcp1(10.31.132.60@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.666089:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.666094:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.666097:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.666099:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.666104:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.666109:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.666162:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.666168:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.666171:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.666178:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.666181:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bae58 00000800:00000200:20.0:1616513078.666185:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.666188:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.671176:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.671231:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.671240:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.671240:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.671249:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.242@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.671258:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.242@tcp1 of length 224 into portal 26 MB=0x5ffd3d5072800 00000400:00000200:18.0:1616513078.671266:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.242@tcp1 of length 224/224 into md 0xf97c7a79 [1] + 0 00000400:00000200:18.0:1616513078.671271:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7cb0 00000400:00000200:18.0:1616513078.671274:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.671276:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513078.671281:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7cb0 00000800:00000200:18.0:1616513078.671284:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.671287:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.671289:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.671346:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660160751616 00010000:00000200:15.0:1616513078.671357:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688660160751616/t0(0) o400->4c8b6dbd-4150-e9e6-40c7-1a5dc6361da0@10.31.128.242@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.671381:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660160751616, offset 192 00000400:00000200:15.0:1616513078.671387:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.242@tcp1 00000400:00000200:15.0:1616513078.671395:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.242@tcp1 routed destination 00000400:00000200:15.0:1616513078.671410:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.242@tcp1(10.31.128.242@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.671416:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.671422:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.671425:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.671427:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.671431:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.671436:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.671449:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513078.671490:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.671497:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513078.671501:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513078.671507:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513078.671514:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd827f8 00000800:00000200:22.0:1616513078.671517:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.671520:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513078.671525:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513078.671535:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.671540:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513078.671544:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513078.671546:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.242@tcp1 : PUT - for me 00000400:00000200:30.0:1616513078.671552:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.242@tcp1 of length 224 into portal 12 MB=0x5ffd3d5072c40 00000400:00000200:30.0:1616513078.671557:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.242@tcp1 of length 224/224 into md 0xf96450b9 [1] + 78976 00000400:00000200:30.0:1616513078.671561:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513078.671563:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513078.671568:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513078.671570:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.671572:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.671635:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660160752704 00010000:00000200:14.0:1616513078.671647:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7de80 x1688660160752704/t0(0) o400->16cea02b-3dca-6cd5-b031-ccdb66612f0f@10.31.128.242@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.671662:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660160752704, offset 192 00000400:00000200:14.0:1616513078.671671:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.242@tcp1 00000400:00000200:14.0:1616513078.671679:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.242@tcp1 routed destination 00000400:00000200:14.0:1616513078.671694:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.242@tcp1(10.31.128.242@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.671700:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.671706:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.671709:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.671711:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.671716:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.671761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.671827:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.671834:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513078.671837:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513078.671844:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513078.671848:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757cb0 00000800:00000200:0.0:1616513078.671852:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.671855:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513078.682546:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925264, portal 4 00000100:00000200:4.0:1616513078.682552:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925264, offset 0 00000400:00000200:4.0:1616513078.682559:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:4.0:1616513078.682565:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:4.0:1616513078.682578:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.682584:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:4.0:1616513078.682591:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000100:00000200:20.0:1616513078.682594:0:6487:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925280, portal 4 00000800:00000200:4.0:1616513078.682594:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:4.0:1616513078.682596:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:20.0:1616513078.682600:0:6487:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925280, offset 0 00000800:00000200:4.0:1616513078.682601:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:20.0:1616513078.682606:0:6487:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:20.0:1616513078.682612:0:6487:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:20.0:1616513078.682625:0:6487:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513078.682632:0:6487:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:20.0:1616513078.682638:0:6487:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513078.682641:0:6487:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:20.0:1616513078.682644:0:6487:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.2:1616513078.682648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:20.0:1616513078.682649:0:6487:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513078.682666:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:27.0:1616513078.682672:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:27.0:1616513078.682675:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513078.682681:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513078.682688:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cd6725100 x1693067725925264/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513078.682709:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba3b8 00000800:00000200:27.0:1616513078.682713:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513078.682716:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513078.682721:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513078.682722:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513078.682727:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513078.682732:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affd4380 x1693067725925280/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513078.682741:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23a08 00000800:00000200:27.0:1616513078.682743:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513078.682745:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513078.682832:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513078.682841:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513078.682846:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000800:00000200:23.0:1616513078.682850:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513078.682853:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513078.682860:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626b90 00000800:00000200:23.0:1616513078.682862:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513078.682864:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513078.682867:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371668fd [1] + 192 00000400:00000200:23.0:1616513078.682871:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513078.682872:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513078.682876:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cd6725100 x1693067725925264/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513078.682879:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626ba0 00000800:00000200:27.0:1616513078.682888:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:23.0:1616513078.682890:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716690d [1] + 192 00000800:00000200:27.0:1616513078.682891:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513078.682892:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:23.0:1616513078.682896:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513078.682899:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cd6725100 x1693067725925264/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513078.682923:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cd6725100 x1693067725925264/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513078.682932:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000100:00000200:23.0:1616513078.682937:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affd4380 x1693067725925280/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:23.0:1616513078.682953:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:23.0:1616513078.682958:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:23.0:1616513078.682960:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:20.0:1616513078.682962:0:6487:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affd4380 x1693067725925280/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513078.682976:0:6487:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affd4380 x1693067725925280/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513078.682986:0:6487:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23210 00000800:00000200:7.2:1616513078.683362:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.683417:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.683427:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.683454:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.102@tcp1 : PUT - for me 00000800:00000200:26.0:1616513078.683471:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.683473:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.102@tcp1 of length 224 into portal 26 MB=0x6005e1e3a1140 00000400:00000200:22.0:1616513078.683480:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.102@tcp1 of length 224/224 into md 0xf97c7a89 [1] + 0 00000400:00000200:22.0:1616513078.683485:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7a08 00000400:00000200:22.0:1616513078.683488:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.683491:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513078.683496:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7a08 00000800:00000200:22.0:1616513078.683499:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.683502:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.683504:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.683558:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689254094311744 00010000:00000200:15.0:1616513078.683569:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1689254094311744/t0(0) o400->435199b1-3c45-47b1-bd72-e9c2d65741a1@10.31.138.102@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.683582:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689254094311744, offset 192 00000400:00000200:15.0:1616513078.683588:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.102@tcp1 00000400:00000200:15.0:1616513078.683596:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.102@tcp1 routed destination 00000400:00000200:15.0:1616513078.683610:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.102@tcp1(10.31.138.102@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.683617:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.683623:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.683625:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.683627:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.683632:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.683638:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.683638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.683690:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.683697:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.683700:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513078.683702:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.683707:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.683713:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd838f8 00000800:00000200:18.0:1616513078.683716:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513078.683772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513078.683782:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.683792:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513078.683794:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.683801:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.102@tcp1 : PUT - for me 00000400:00000200:26.0:1616513078.683810:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.102@tcp1 of length 224 into portal 12 MB=0x6005e1e3a12c0 00000400:00000200:26.0:1616513078.683817:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.102@tcp1 of length 224/224 into md 0xf96450b9 [1] + 79200 00000400:00000200:26.0:1616513078.683822:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513078.683824:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513078.683831:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.683834:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.683836:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.683857:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689254094312128 00010000:00000200:14.0:1616513078.683869:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dba80 x1689254094312128/t0(0) o400->4fae495f-d635-ada5-8d2d-754426c2a84f@10.31.138.102@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.683884:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689254094312128, offset 192 00000400:00000200:14.0:1616513078.683891:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.102@tcp1 00000400:00000200:14.0:1616513078.683898:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.102@tcp1 routed destination 00000400:00000200:14.0:1616513078.683914:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.102@tcp1(10.31.138.102@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.683921:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.683927:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.683929:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.683931:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.683936:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.683943:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.683963:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.683968:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.683972:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.683978:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.683982:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756d48 00000800:00000200:20.0:1616513078.683984:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.683987:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513078.698916:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925296, portal 4 00000100:00000200:22.0:1616513078.698922:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925296, offset 0 00000400:00000200:22.0:1616513078.698929:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513078.698935:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:22.0:1616513078.698948:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513078.698954:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:22.0:1616513078.698960:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513078.698963:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513078.698966:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513078.698971:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513078.699024:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513078.699038:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:30.0:1616513078.699044:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:30.0:1616513078.699047:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.699053:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:30.0:1616513078.699059:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d7464621680 x1693067725925296/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513078.699070:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4af68 00000800:00000200:30.0:1616513078.699073:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:30.0:1616513078.699076:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513078.699103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513078.699118:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513078.699124:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513078.699131:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513078.699137:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626bb0 00000400:00000200:22.0:1616513078.699143:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c8109 [1] + 192 00000400:00000200:22.0:1616513078.699147:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.699152:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d7464621680 x1693067725925296/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513078.699170:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513078.699177:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513078.699179:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513078.699181:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:30.0:1616513078.699183:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d7464621680 x1693067725925296/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513078.699194:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d7464621680 x1693067725925296/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513078.699202:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:11.2:1616513078.705082:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.705136:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.705145:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513078.705152:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513078.705154:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.160@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.705162:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.160@tcp1 of length 224 into portal 26 MB=0x5ffd44e56c900 00000800:00000200:28.0:1616513078.705163:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513078.705169:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.160@tcp1 of length 224/224 into md 0xf97c7ac9 [1] + 0 00000400:00000200:28.0:1616513078.705171:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.160@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.705174:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c77e8 00000400:00000200:0.0:1616513078.705177:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513078.705179:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.705180:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.160@tcp1 of length 224 into portal 12 MB=0x5ffd44e56c980 00000400:00000200:0.0:1616513078.705184:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c77e8 00000400:00000200:28.0:1616513078.705187:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.160@tcp1 of length 224/224 into md 0x865bbec1 [1] + 12544 00000800:00000200:0.0:1616513078.705187:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513078.705190:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513078.705191:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.705193:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513078.705194:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513078.705196:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513078.705203:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.705206:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.705208:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.705250:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662196013312 00010000:00000200:15.0:1616513078.705262:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688662196013312/t0(0) o400->b273cf26-3bc6-c7ea-13bc-92c350ad3f2f@10.31.138.160@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513078.705264:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662196013440 00010000:00000200:4.0:1616513078.705277:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6723f00 x1688662196013440/t0(0) o400->06e5470d-0a07-54c2-c0cf-505bf90fb9f1@10.31.138.160@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513078.705278:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662196013312, offset 192 00000400:00000200:15.0:1616513078.705285:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.160@tcp1 00000100:00000200:4.0:1616513078.705290:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662196013440, offset 192 00000400:00000200:15.0:1616513078.705292:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.160@tcp1 routed destination 00000400:00000200:4.0:1616513078.705296:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.160@tcp1 00000400:00000200:4.0:1616513078.705303:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.160@tcp1 routed destination 00000400:00000200:15.0:1616513078.705308:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.160@tcp1(10.31.138.160@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.705316:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:4.0:1616513078.705318:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.160@tcp1(10.31.138.160@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.705322:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.705324:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.705325:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.705327:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513078.705330:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.705332:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513078.705332:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.705334:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.705338:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513078.705339:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.705340:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513078.705344:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513078.705352:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.705357:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.705359:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.705364:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.705369:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd836d8 00000800:00000200:30.0:1616513078.705372:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.705374:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513078.705400:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.705407:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.705410:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513078.705411:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.705417:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.705421:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000800:00000200:24.0:1616513078.705425:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.730976:0:49308:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513078.731032:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.731042:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513078.731051:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.108@tcp1 : PUT - for me 00000400:00000200:26.0:1616513078.731062:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.108@tcp1 of length 224 into portal 26 MB=0x6001ed4c745c0 00000400:00000200:26.0:1616513078.731069:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.108@tcp1 of length 224/224 into md 0xf97c7ad9 [1] + 0 00000400:00000200:26.0:1616513078.731074:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6f68 00000400:00000200:26.0:1616513078.731077:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513078.731080:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513078.731085:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6f68 00000800:00000200:22.0:1616513078.731087:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.731089:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.731091:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.731093:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.731146:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688982279112128 00010000:00000200:15.0:1616513078.731156:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688982279112128/t0(0) o400->d782fa64-cfc7-8ea4-80bb-ade481bf6d0f@10.31.142.108@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.731170:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688982279112128, offset 192 00000400:00000200:15.0:1616513078.731175:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.108@tcp1 00000400:00000200:15.0:1616513078.731184:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.108@tcp1 routed destination 00000400:00000200:15.0:1616513078.731198:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.108@tcp1(10.31.142.108@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.731204:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.731210:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.731213:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.731215:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.731219:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.731224:0:49308:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.731226:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513078.731236:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513078.731242:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.731246:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513078.731253:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.731259:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83650 00000800:00000200:30.0:1616513078.731263:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513078.731266:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.731494:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.731554:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.731563:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.731573:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.108@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.731582:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.108@tcp1 of length 224 into portal 12 MB=0x6001ed4c75340 00000400:00000200:20.0:1616513078.731588:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.108@tcp1 of length 224/224 into md 0x865bbec1 [1] + 12768 00000400:00000200:20.0:1616513078.731594:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.731597:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513078.731603:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.731606:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.731607:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.731611:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513078.731666:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688982279115584 00010000:00000200:4.0:1616513078.731678:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e00d80 x1688982279115584/t0(0) o400->27899fe5-d1b1-cf51-1d42-9abd117888b4@10.31.142.108@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513078.731700:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688982279115584, offset 192 00000400:00000200:4.0:1616513078.731706:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.108@tcp1 00000400:00000200:4.0:1616513078.731713:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.108@tcp1 routed destination 00000400:00000200:4.0:1616513078.731726:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.108@tcp1(10.31.142.108@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.731732:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.731737:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.731740:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.731742:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.731746:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.731752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.731805:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.731812:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513078.731815:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513078.731822:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513078.731826:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6badd0 00000800:00000200:0.0:1616513078.731829:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.731833:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:3.2F:1616513078.734671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4cadf1600] (20)++ 00000800:00000200:3.0:1616513078.734699:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4cadf1600] (21)++ 00000800:00000200:3.0:1616513078.734708:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.160@o2ib1 00000400:00000200:3.0:1616513078.734717:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.160@o2ib1 : PUT - for me 00000400:00000200:3.0:1616513078.734725:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.160@o2ib1 of length 224 into portal 26 MB=0x60144f886bad0 00000400:00000200:3.0:1616513078.734731:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.160@o2ib1 of length 224/224 into md 0xf97c7b09 [1] + 0 00000400:00000200:3.0:1616513078.734736:0:6473:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c74b8 00000400:00000200:3.0:1616513078.734739:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:3.0:1616513078.734742:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:3.0:1616513078.734747:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c74b8 00000800:00000200:3.0:1616513078.734750:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4cadf1600] (22)++ 00000800:00000200:31.0:1616513078.734752:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4cadf1600] (22)++ 00000800:00000200:3.0:1616513078.734753:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4cadf1600] (23)-- 00000800:00000200:3.0:1616513078.734759:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4cadf1600] (23)-- 00000800:00000200:3.0:1616513078.734762:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4cadf1600] (22)-- 00000800:00000200:31.0:1616513078.734763:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.160@o2ib1 00000400:00000200:31.0:1616513078.734771:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.160@o2ib1 : PUT - for me 00000400:00000200:31.0:1616513078.734779:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.160@o2ib1 of length 224 into portal 12 MB=0x60144f886bae0 00000400:00000200:31.0:1616513078.734785:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.160@o2ib1 of length 224/224 into md 0x3657608d [1] + 91392 00000400:00000200:31.0:1616513078.734791:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:31.0:1616513078.734793:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:31.0:1616513078.734801:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4cadf1600] (21)++ 00000800:00000200:31.0:1616513078.734804:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4cadf1600] (22)-- 00000800:00000200:31.0:1616513078.734806:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4cadf1600] (21)-- 00000100:00000200:15.0:1616513078.734806:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690245599247056 00010000:00000200:15.0:1616513078.734816:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1690245599247056/t0(0) o400->6381dae2-09d3-16b7-deb9-cfcf65f7a865@10.242.46.160@o2ib1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.734829:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690245599247056, offset 192 00000400:00000200:15.0:1616513078.734835:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.160@o2ib1 00000400:00000200:15.0:1616513078.734842:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.160@o2ib1 local destination 00000400:00000200:15.0:1616513078.734856:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.160@o2ib1(10.242.46.160@o2ib1:10.242.46.160@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.734862:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.160@o2ib1 00000100:00000200:19.0:1616513078.734868:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690245599247072 00000800:00000200:15.0:1616513078.734868:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf00] -> 10.242.46.160@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.734870:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4cadf1600] (20)++ 00000800:00000200:15.0:1616513078.734872:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4cadf1600] (21)++ 00000800:00000200:15.0:1616513078.734877:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4cadf1600] (22)-- 00010000:00000200:19.0:1616513078.734881:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cdc562880 x1690245599247072/t0(0) o400->062f64fb-1a41-6c2a-3edb-871ec6c64b4f@10.242.46.160@o2ib1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:3.2:1616513078.734882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4cadf1600] (21)++ 00000100:00000200:15.0:1616513078.734884:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:3.0:1616513078.734891:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4cadf1600] (22)++ 00000100:00000200:19.0:1616513078.734896:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690245599247072, offset 192 00000800:00000200:3.0:1616513078.734896:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4cadf1600] (23)-- 00000400:00000200:3.0:1616513078.734898:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.0:1616513078.734900:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4cadf1600] (22)-- 00000400:00000200:19.0:1616513078.734902:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.160@o2ib1 00000400:00000200:3.0:1616513078.734903:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.160@o2ib1: PUT: OK 00000400:00000200:3.0:1616513078.734908:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82e58 00000400:00000200:19.0:1616513078.734909:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.160@o2ib1 local destination 00000400:00000200:19.0:1616513078.734922:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.160@o2ib1(10.242.46.160@o2ib1:10.242.46.160@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513078.734927:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4cadf1600] (21)-- 00000800:00000200:19.0:1616513078.734929:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.160@o2ib1 00000800:00000200:19.0:1616513078.734934:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf00] -> 10.242.46.160@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513078.734937:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4cadf1600] (20)++ 00000800:00000200:19.0:1616513078.734939:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4cadf1600] (21)++ 00000800:00000200:19.0:1616513078.734944:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4cadf1600] (22)-- 00000800:00000200:3.2:1616513078.734949:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4cadf1600] (21)++ 00000800:00000200:3.0:1616513078.734957:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4cadf1600] (22)++ 00000800:00000200:3.0:1616513078.734960:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4cadf1600] (23)-- 00000400:00000200:3.0:1616513078.734962:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:27.0:1616513078.734966:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4cadf1600] (22)-- 00000400:00000200:3.0:1616513078.734967:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.160@o2ib1: PUT: OK 00000400:00000200:3.0:1616513078.734970:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707a90 00000800:00000200:3.0:1616513078.734972:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4cadf1600] (21)-- 00000800:00000200:7.2:1616513078.744627:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.744682:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.744693:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.744702:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.186@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.744712:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.186@tcp1 of length 224 into portal 26 MB=0x6058f170db700 00000400:00000200:22.0:1616513078.744719:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.186@tcp1 of length 224/224 into md 0xf97c7b19 [1] + 0 00000400:00000200:22.0:1616513078.744724:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7100 00000400:00000200:22.0:1616513078.744727:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.744730:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513078.744735:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7100 00000800:00000200:26.0:1616513078.744738:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.744739:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.744741:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513078.744745:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513078.744748:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.744749:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513078.744757:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.186@tcp1 : PUT - for me 00000400:00000200:26.0:1616513078.744766:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.186@tcp1 of length 224 into portal 12 MB=0x6058f170db780 00000100:00000200:15.0:1616513078.744767:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694961985500928 00000400:00000200:26.0:1616513078.744773:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.186@tcp1 of length 224/224 into md 0xf96450b9 [1] + 79424 00000400:00000200:26.0:1616513078.744778:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:15.0:1616513078.744778:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1694961985500928/t0(0) o400->86ef905d-ccb6-d76a-0f16-01838fcca9e5@10.31.129.186@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513078.744781:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513078.744789:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.744792:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.744792:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694961985500928, offset 192 00000800:00000200:26.0:1616513078.744794:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:15.0:1616513078.744798:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.186@tcp1 00000400:00000200:15.0:1616513078.744806:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.186@tcp1 routed destination 00000400:00000200:15.0:1616513078.744821:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.186@tcp1(10.31.129.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.744828:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.744834:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.744837:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.744839:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.744844:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.744849:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.744851:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:14.0:1616513078.744855:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694961985501056 00010000:00000200:14.0:1616513078.744868:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9680 x1694961985501056/t0(0) o400->c1b9b63a-8c95-2d01-63f8-b7e3b550ac9d@10.31.129.186@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.744882:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694961985501056, offset 192 00000400:00000200:14.0:1616513078.744888:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.186@tcp1 00000400:00000200:14.0:1616513078.744896:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.186@tcp1 routed destination 00000800:00000200:18.0:1616513078.744908:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513078.744910:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.186@tcp1(10.31.129.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513078.744916:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513078.744916:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513078.744919:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.744919:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513078.744922:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513078.744926:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:14.0:1616513078.744926:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.744928:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513078.744932:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82330 00000800:00000200:14.0:1616513078.744933:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513078.744936:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.744942:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.744998:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.745005:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.745008:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.745014:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.745018:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756c38 00000800:00000200:28.0:1616513078.745022:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.745025:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.758129:0:49311:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.758184:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.758193:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513078.758199:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.758202:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.45@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.758212:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.45@tcp1 of length 224 into portal 26 MB=0x5f92e0431e1c0 00000400:00000200:22.0:1616513078.758219:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.45@tcp1 of length 224/224 into md 0xf97c7b39 [1] + 0 00000400:00000200:22.0:1616513078.758224:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6dd0 00000400:00000200:22.0:1616513078.758227:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.758230:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513078.758235:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6dd0 00000800:00000200:22.0:1616513078.758238:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.758241:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.758243:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.758297:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681350917743040 00010000:00000200:15.0:1616513078.758307:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1681350917743040/t0(0) o400->78089470-cfe8-d7d9-6ff2-da75d4385df8@10.31.132.45@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513078.758316:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513078.758320:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681350917743040, offset 192 00000400:00000200:15.0:1616513078.758326:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.45@tcp1 00000800:00000200:0.0:1616513078.758332:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513078.758334:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.45@tcp1 routed destination 00000800:00000200:0.0:1616513078.758342:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:15.0:1616513078.758349:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.45@tcp1(10.31.132.45@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513078.758354:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.45@tcp1 : PUT - for me 00000800:00000200:15.0:1616513078.758356:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.758361:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:0.0:1616513078.758363:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.45@tcp1 of length 224 into portal 12 MB=0x5f92e0431e240 00000800:00000200:15.0:1616513078.758366:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.758368:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513078.758370:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.45@tcp1 of length 224/224 into md 0x865bbec1 [1] + 12992 00000800:00000200:15.0:1616513078.758373:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513078.758375:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:7.2:1616513078.758377:0:49311:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513078.758378:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513078.758379:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513078.758384:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.758387:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.758387:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:30.0:1616513078.758388:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513078.758391:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513078.758394:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513078.758397:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513078.758400:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513078.758403:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513078.758409:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82088 00000800:00000200:30.0:1616513078.758413:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513078.758461:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681350917743168 00010000:00000200:4.0:1616513078.758483:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b197980 x1681350917743168/t0(0) o400->5d34a9b8-a9aa-e3c4-22c2-40b721d1a89d@10.31.132.45@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513078.758498:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681350917743168, offset 192 00000400:00000200:4.0:1616513078.758504:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.45@tcp1 00000400:00000200:4.0:1616513078.758513:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.45@tcp1 routed destination 00000400:00000200:4.0:1616513078.758528:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.45@tcp1(10.31.132.45@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.758534:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.758540:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.758543:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.758545:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.758550:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.758555:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.758608:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.758615:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.758617:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.758624:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.758628:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba000 00000800:00000200:28.0:1616513078.758632:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.758635:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.765526:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.765581:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.765591:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.765600:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.208@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.765609:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.208@tcp1 of length 224 into portal 26 MB=0x5ffdfb537ea00 00000400:00000200:24.0:1616513078.765618:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.208@tcp1 of length 224/224 into md 0xf97c7b49 [1] + 0 00000400:00000200:24.0:1616513078.765623:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7d38 00000400:00000200:24.0:1616513078.765626:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.765629:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.765633:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7d38 00000800:00000200:0.0:1616513078.765635:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.765637:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.765639:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513078.765642:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.765645:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.765646:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:15.0:1616513078.765654:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711166683648 00000400:00000200:0.0:1616513078.765654:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.208@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.765663:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.208@tcp1 of length 224 into portal 12 MB=0x5ffdfb537eb00 00010000:00000200:15.0:1616513078.765665:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688711166683648/t0(0) o400->4771c03a-b2f1-f8fe-eb3d-7d31f0acbbf6@10.31.142.208@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513078.765669:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.208@tcp1 of length 224/224 into md 0x865bbec1 [1] + 13216 00000400:00000200:0.0:1616513078.765674:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513078.765677:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:15.0:1616513078.765678:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688711166683648, offset 192 00000800:00000200:0.0:1616513078.765683:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513078.765684:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.208@tcp1 00000800:00000200:0.0:1616513078.765687:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.765688:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:15.0:1616513078.765691:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.208@tcp1 routed destination 00000400:00000200:15.0:1616513078.765705:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.208@tcp1(10.31.142.208@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.765712:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.765717:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.765720:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.765722:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.765727:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.765733:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.765733:0:49312:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513078.765772:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711166683904 00010000:00000200:4.0:1616513078.765785:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b196300 x1688711166683904/t0(0) o400->d50887d3-d4d6-59cf-26cc-71020a259815@10.31.142.208@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:26.0:1616513078.765792:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.765799:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:4.0:1616513078.765800:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688711166683904, offset 192 00000400:00000200:26.0:1616513078.765803:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513078.765806:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.208@tcp1 00000400:00000200:26.0:1616513078.765809:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:4.0:1616513078.765814:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.208@tcp1 routed destination 00000400:00000200:26.0:1616513078.765816:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82220 00000800:00000200:26.0:1616513078.765819:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.765822:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513078.765830:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.208@tcp1(10.31.142.208@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.765838:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.765845:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.765847:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.765850:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.765855:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.765859:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.765915:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.765922:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.765924:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.765926:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.765931:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.765935:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbba0 00000800:00000200:20.0:1616513078.765939:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.769315:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.769340:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.769350:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.769358:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.24@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.769366:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.24@tcp1 of length 224 into portal 12 MB=0x60022873004c0 00000400:00000200:18.0:1616513078.769372:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.24@tcp1 of length 224/224 into md 0xf96450b9 [1] + 79648 00000400:00000200:18.0:1616513078.769379:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.769382:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.769389:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.769391:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.769393:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513078.769396:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.769468:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688998157223104 00010000:00000200:14.0:1616513078.769481:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eba400 x1688998157223104/t0(0) o400->62ed83e7-e399-18ec-bf91-50b44e4be118@10.31.132.24@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.769496:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688998157223104, offset 192 00000400:00000200:14.0:1616513078.769502:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.24@tcp1 00000400:00000200:14.0:1616513078.769510:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.24@tcp1 routed destination 00000400:00000200:14.0:1616513078.769524:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.24@tcp1(10.31.132.24@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513078.769526:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.769531:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513078.769537:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.769540:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513078.769541:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.769542:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.769547:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513078.769551:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513078.769552:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513078.769556:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.769560:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.24@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.769569:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.24@tcp1 of length 224 into portal 26 MB=0x6002287300200 00000400:00000200:28.0:1616513078.769576:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.24@tcp1 of length 224/224 into md 0xf97c7b69 [1] + 0 00000400:00000200:28.0:1616513078.769580:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7e48 00000400:00000200:28.0:1616513078.769583:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.769586:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.769591:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7e48 00000800:00000200:28.0:1616513078.769594:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.769597:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.769599:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513078.769604:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.769611:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:15.0:1616513078.769612:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688998157222400 00000400:00000200:26.0:1616513078.769614:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.769620:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:15.0:1616513078.769623:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688998157222400/t0(0) o400->1c2fdf40-8a8e-ee56-3352-a1110239def2@10.31.132.24@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513078.769624:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756330 00000800:00000200:26.0:1616513078.769628:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.769631:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.769635:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688998157222400, offset 192 00000400:00000200:15.0:1616513078.769641:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.24@tcp1 00000400:00000200:15.0:1616513078.769649:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.24@tcp1 routed destination 00000400:00000200:15.0:1616513078.769664:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.24@tcp1(10.31.132.24@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.769670:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.769675:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.769678:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.769680:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.769685:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.769689:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.769691:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513078.769748:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.769761:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.769765:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.769772:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.769778:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82330 00000800:00000200:24.0:1616513078.769781:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.769784:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.772374:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.772425:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.772445:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.772454:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.41@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.772462:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.41@tcp1 of length 224 into portal 12 MB=0x5ffd3f0e8b140 00000400:00000200:18.0:1616513078.772468:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.41@tcp1 of length 224/224 into md 0xf96450b9 [1] + 79872 00000400:00000200:18.0:1616513078.772473:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.772475:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513078.772479:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.772481:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.772484:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.772486:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.772544:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660628517184 00010000:00000200:14.0:1616513078.772557:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebc800 x1688660628517184/t0(0) o400->c106f27a-643a-a51d-1c19-b324505f71b0@10.31.138.41@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.772571:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660628517184, offset 192 00000400:00000200:14.0:1616513078.772578:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.41@tcp1 00000400:00000200:14.0:1616513078.772585:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.41@tcp1 routed destination 00000400:00000200:14.0:1616513078.772600:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.41@tcp1(10.31.138.41@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.772607:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513078.772613:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.772616:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513078.772618:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.772623:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.772628:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.772681:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.772688:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.772690:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.772697:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.772701:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47567f8 00000800:00000200:26.0:1616513078.772705:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.772708:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.774498:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.774551:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.774560:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513078.774568:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.41@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.774576:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.41@tcp1 of length 224 into portal 26 MB=0x5ffd3f0e8b000 00000400:00000200:0.0:1616513078.774582:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.41@tcp1 of length 224/224 into md 0xf97c7b79 [1] + 0 00000400:00000200:0.0:1616513078.774587:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7210 00000400:00000200:0.0:1616513078.774590:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513078.774592:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513078.774597:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7210 00000800:00000200:0.0:1616513078.774600:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.774602:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.774603:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513078.774607:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.774658:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660628516864 00010000:00000200:15.0:1616513078.774668:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688660628516864/t0(0) o400->00f96db6-d767-f6de-0b0f-0e2b8794264e@10.31.138.41@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.774682:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660628516864, offset 192 00000400:00000200:15.0:1616513078.774687:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.41@tcp1 00000400:00000200:15.0:1616513078.774695:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.41@tcp1 routed destination 00000400:00000200:15.0:1616513078.774710:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.41@tcp1(10.31.138.41@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.774716:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.774721:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.774724:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.774726:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.774731:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513078.774737:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513078.774738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.774790:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.774797:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.774800:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.774806:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.774812:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83650 00000800:00000200:24.0:1616513078.774816:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.774818:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513078.778539:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925312, portal 4 00000100:00000200:28.0:1616513078.778546:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925312, offset 0 00000400:00000200:28.0:1616513078.778552:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:28.0:1616513078.778558:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:28.0:1616513078.778570:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513078.778577:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:28.0:1616513078.778582:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513078.778585:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:28.0:1616513078.778587:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:28.0:1616513078.778592:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513078.778639:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513078.778696:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513078.778702:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513078.778705:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513078.778712:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513078.778718:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc7e00900 x1693067725925312/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513078.778729:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd298 00000800:00000200:2.0:1616513078.778733:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513078.778736:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513078.778742:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513078.778748:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513078.778750:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513078.778757:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626bc0 00000400:00000200:2.0:1616513078.778763:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c81d9 [1] + 192 00000400:00000200:2.0:1616513078.778768:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513078.778772:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc7e00900 x1693067725925312/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513078.778786:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513078.778788:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513078.778790:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:28.0:1616513078.778794:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc7e00900 x1693067725925312/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513078.778806:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc7e00900 x1693067725925312/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513078.778815:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc000 00000800:00000200:7.2:1616513078.786671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.786730:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.786739:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513078.786739:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513078.786748:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.108@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.786766:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.108@tcp1 of length 224 into portal 12 MB=0x5f9149b680e40 00000400:00000200:22.0:1616513078.786773:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.108@tcp1 of length 224/224 into md 0xf96450b9 [1] + 80096 00000400:00000200:22.0:1616513078.786778:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.786781:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.786783:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.786788:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.786791:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.786793:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.786805:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:14.0:1616513078.786814:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681241785503296 00000800:00000200:28.0:1616513078.786815:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00010000:00000200:14.0:1616513078.786831:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec205100 x1681241785503296/t0(0) o400->1d4132a2-e9c1-242e-b9b8-22b6139a1de4@10.31.143.108@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513078.786835:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.108@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.786843:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.108@tcp1 of length 224 into portal 26 MB=0x5f9149b680dc0 00000400:00000200:28.0:1616513078.786850:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.108@tcp1 of length 224/224 into md 0xf97c7b99 [1] + 0 00000400:00000200:28.0:1616513078.786854:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7430 00000400:00000200:28.0:1616513078.786857:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513078.786857:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681241785503296, offset 192 00000100:00000200:28.0:1616513078.786860:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.786865:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7430 00000800:00000200:28.0:1616513078.786867:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513078.786869:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.108@tcp1 00000800:00000200:0.0:1616513078.786869:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.786870:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.786875:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513078.786877:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.108@tcp1 routed destination 00000400:00000200:14.0:1616513078.786892:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.108@tcp1(10.31.143.108@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.786898:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513078.786904:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.786907:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513078.786909:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.786914:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.786920:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.786941:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681241785503168 00010000:00000200:15.0:1616513078.786951:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1681241785503168/t0(0) o400->37a75478-c27e-6822-cd6b-d22d9c5940c4@10.31.143.108@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.786966:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681241785503168, offset 192 00000400:00000200:15.0:1616513078.786972:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.108@tcp1 00000800:00000200:26.0:1616513078.786973:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.786980:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:15.0:1616513078.786980:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.108@tcp1 routed destination 00000400:00000200:26.0:1616513078.786982:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.786990:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.786994:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756a18 00000400:00000200:15.0:1616513078.786996:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.108@tcp1(10.31.143.108@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513078.786998:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.787001:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:15.0:1616513078.787003:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.787009:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.787012:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.787014:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.787019:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.787024:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.787025:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513078.787077:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.787084:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.787087:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.787093:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.787099:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd838f8 00000800:00000200:24.0:1616513078.787102:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.787105:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.791184:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.791239:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.791249:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513078.791258:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.30.11@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.791268:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.30.11@tcp1 of length 224 into portal 26 MB=0x60012fad0ea40 00000400:00000200:0.0:1616513078.791275:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.30.11@tcp1 of length 224/224 into md 0xf97c7bb9 [1] + 0 00000400:00000200:0.0:1616513078.791279:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c75c8 00000400:00000200:0.0:1616513078.791282:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513078.791285:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513078.791290:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c75c8 00000800:00000200:28.0:1616513078.791293:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.791293:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.791296:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513078.791298:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.791353:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688931377670720 00010000:00000200:15.0:1616513078.791363:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688931377670720/t0(0) o400->139616c1-da95-39c3-8c93-658dd8b30732@10.31.30.11@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.791377:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688931377670720, offset 192 00000400:00000200:15.0:1616513078.791384:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.30.11@tcp1 00000400:00000200:15.0:1616513078.791392:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.30.11@tcp1 routed destination 00000400:00000200:15.0:1616513078.791406:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.30.11@tcp1(10.31.30.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.791412:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.791418:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.791421:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.791423:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.791427:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.791432:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.791444:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513078.791447:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.791454:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.791457:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.791464:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.791470:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd838f8 00000800:00000200:18.0:1616513078.791474:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.791477:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.791482:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.791490:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.30.11@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.791499:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.30.11@tcp1 of length 224 into portal 12 MB=0x60012fad0ebc0 00000800:00000200:22.0:1616513078.791500:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.791505:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.30.11@tcp1 of length 224/224 into md 0xf96450b9 [1] + 80320 00000400:00000200:18.0:1616513078.791510:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.791513:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.791518:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.791521:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.791524:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.791594:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688931377671104 00010000:00000200:14.0:1616513078.791605:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebec00 x1688931377671104/t0(0) o400->23153974-f429-9142-d25f-2525816fad68@10.31.30.11@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.791618:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688931377671104, offset 192 00000400:00000200:14.0:1616513078.791624:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.30.11@tcp1 00000400:00000200:14.0:1616513078.791632:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.30.11@tcp1 routed destination 00000400:00000200:14.0:1616513078.791646:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.30.11@tcp1(10.31.30.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.791652:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.791658:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.791660:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.791662:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.791667:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.791671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.791724:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.791731:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.791734:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.791741:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.791746:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757c28 00000800:00000200:24.0:1616513078.791750:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.791753:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.794527:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925328, portal 4 00000100:00000200:15.0:1616513078.794532:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925328, offset 0 00000400:00000200:15.0:1616513078.794539:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:15.0:1616513078.794544:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:15.0:1616513078.794556:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.794562:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:15.0:1616513078.794567:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.794570:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:15.0:1616513078.794572:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:15.0:1616513078.794576:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513078.794621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513078.794641:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513078.794646:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513078.794648:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513078.794655:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513078.794661:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63ec00 x1693067725925328/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513078.794673:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82220 00000800:00000200:27.0:1616513078.794677:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513078.794681:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513078.794793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513078.794802:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513078.794807:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:3.0:1616513078.794812:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513078.794815:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513078.794823:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626bd0 00000400:00000200:27.0:1616513078.794829:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371669dd [1] + 192 00000400:00000200:27.0:1616513078.794834:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513078.794838:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63ec00 x1693067725925328/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513078.794851:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513078.794854:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513078.794856:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:15.0:1616513078.794858:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63ec00 x1693067725925328/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513078.794870:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63ec00 x1693067725925328/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513078.794879:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82330 00000800:00000200:11.2:1616513078.798451:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513078.798504:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.798513:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.798519:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513078.798521:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.225@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.798530:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.225@tcp1 of length 224 into portal 26 MB=0x5ffdc5f546700 00000800:00000200:0.0:1616513078.798530:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513078.798538:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.225@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.798539:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.225@tcp1 of length 224/224 into md 0xf97c7bc9 [1] + 0 00000400:00000200:28.0:1616513078.798543:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c67f8 00000400:00000200:28.0:1616513078.798546:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.798548:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513078.798548:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.225@tcp1 of length 224 into portal 12 MB=0x5ffdc5f546780 00000400:00000200:28.0:1616513078.798553:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c67f8 00000400:00000200:0.0:1616513078.798554:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.225@tcp1 of length 224/224 into md 0x865bbec1 [1] + 13440 00000800:00000200:28.0:1616513078.798556:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513078.798558:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:0.0:1616513078.798559:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:28.0:1616513078.798560:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.798562:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513078.798562:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513078.798569:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.798572:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.798574:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513078.798593:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696840808320 00010000:00000200:4.0:1616513078.798606:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b190900 x1688696840808320/t0(0) o400->2b46b1b6-2e17-ffa6-a2bb-9a2d4f876571@10.31.141.225@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:15.0:1616513078.798617:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696840808192 00000100:00000200:4.0:1616513078.798620:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696840808320, offset 192 00000400:00000200:4.0:1616513078.798627:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.225@tcp1 00010000:00000200:15.0:1616513078.798628:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688696840808192/t0(0) o400->8b702c5c-09ca-de75-9008-ffe6fe089572@10.31.141.225@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:4.0:1616513078.798634:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.225@tcp1 routed destination 00000100:00000200:15.0:1616513078.798642:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696840808192, offset 192 00000400:00000200:15.0:1616513078.798649:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.225@tcp1 00000400:00000200:4.0:1616513078.798650:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.225@tcp1(10.31.141.225@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:15.0:1616513078.798657:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.225@tcp1 routed destination 00000800:00000200:4.0:1616513078.798657:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513078.798663:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.798666:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513078.798668:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513078.798673:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:15.0:1616513078.798675:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.225@tcp1(10.31.141.225@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.798682:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:15.0:1616513078.798689:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.798691:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:15.0:1616513078.798694:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:15.0:1616513078.798699:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.798704:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:15.0:1616513078.798705:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.798719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513078.798757:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.798765:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.798768:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.798768:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.798775:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513078.798778:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513078.798781:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82330 00000800:00000200:2.0:1616513078.798784:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513078.798785:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513078.798787:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513078.798793:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513078.798797:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baf68 00000800:00000200:2.0:1616513078.798800:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513078.798803:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.801155:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513078.801185:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513078.801212:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.801221:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.801230:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.12@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.801240:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.12@tcp1 of length 224 into portal 26 MB=0x5ffd40bbf7280 00000800:00000200:18.0:1616513078.801242:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513078.801247:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.12@tcp1 of length 224/224 into md 0xf97c7bd9 [1] + 0 00000800:00000200:18.0:1616513078.801251:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513078.801252:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6aa0 00000400:00000200:24.0:1616513078.801254:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.801257:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513078.801259:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.12@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.801262:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6aa0 00000800:00000200:24.0:1616513078.801265:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.801268:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513078.801268:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.12@tcp1 of length 224 into portal 12 MB=0x5ffd40bbf7640 00000800:00000200:0.0:1616513078.801269:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.801273:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513078.801274:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.12@tcp1 of length 224/224 into md 0xf96450b9 [1] + 80544 00000400:00000200:18.0:1616513078.801278:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.801281:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.801287:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.801289:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.801291:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.801294:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513078.801327:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661078798976 00010000:00000200:15.0:1616513078.801338:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688661078798976/t0(0) o400->0c04e16a-4a0f-e2b0-6c38-122798ac9e18@10.31.133.12@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:14.0:1616513078.801348:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661078799936 00000100:00000200:15.0:1616513078.801352:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661078798976, offset 192 00000400:00000200:15.0:1616513078.801358:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.12@tcp1 00010000:00000200:14.0:1616513078.801360:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eba880 x1688661078799936/t0(0) o400->56a971cc-b304-614f-686f-4906fadfb969@10.31.133.12@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:15.0:1616513078.801366:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.12@tcp1 routed destination 00000100:00000200:14.0:1616513078.801373:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661078799936, offset 192 00000400:00000200:14.0:1616513078.801380:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.12@tcp1 00000400:00000200:15.0:1616513078.801392:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.12@tcp1(10.31.133.12@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.801398:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513078.801401:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.12@tcp1 routed destination 00000800:00000200:15.0:1616513078.801404:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.801407:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.801409:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.801414:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513078.801415:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.12@tcp1(10.31.133.12@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513078.801418:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:15.0:1616513078.801420:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513078.801424:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.801430:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.801432:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.801447:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.801451:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.801456:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513078.801472:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513078.801479:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513078.801484:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513078.801491:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513078.801497:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd838f8 00000800:00000200:2.0:1616513078.801501:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513078.801504:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.801510:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.801516:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.801519:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.801525:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.801529:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756000 00000800:00000200:28.0:1616513078.801533:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.801535:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.817911:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.817929:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.817938:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513078.817946:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.227@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.817955:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.227@tcp1 of length 224 into portal 26 MB=0x5ffe352731680 00000400:00000200:0.0:1616513078.817961:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.227@tcp1 of length 224/224 into md 0xf97c7bf9 [1] + 0 00000400:00000200:0.0:1616513078.817966:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6d48 00000400:00000200:0.0:1616513078.817968:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513078.817971:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513078.817975:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6d48 00000800:00000200:0.0:1616513078.817980:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.817983:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513078.817983:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513078.817987:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.818041:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726689486464 00010000:00000200:15.0:1616513078.818052:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688726689486464/t0(0) o400->d5275b05-1dc1-75cc-90a7-1274e2e288b7@10.31.140.227@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:15.0:1616513078.818067:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726689486464, offset 192 00000400:00000200:15.0:1616513078.818073:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.227@tcp1 00000400:00000200:15.0:1616513078.818081:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.227@tcp1 routed destination 00000400:00000200:15.0:1616513078.818096:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.227@tcp1(10.31.140.227@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513078.818101:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.818103:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.818110:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.818114:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.818117:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:15.0:1616513078.818122:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:15.0:1616513078.818128:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513078.818154:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.818164:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.818172:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.227@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.818181:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.227@tcp1 of length 224 into portal 12 MB=0x5ffe352731800 00000400:00000200:18.0:1616513078.818186:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.227@tcp1 of length 224/224 into md 0xf96450b9 [1] + 80768 00000400:00000200:18.0:1616513078.818191:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.818193:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.818200:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513078.818203:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513078.818204:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.818207:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.818210:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.818213:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.818218:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.818223:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd838f8 00000100:00000200:14.0:1616513078.818224:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726689486848 00000800:00000200:18.0:1616513078.818226:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.818228:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:14.0:1616513078.818236:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebcc80 x1688726689486848/t0(0) o400->cec79e90-1f81-9b27-c3c3-f3ebd9d152ae@10.31.140.227@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.818251:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726689486848, offset 192 00000400:00000200:14.0:1616513078.818257:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.227@tcp1 00000400:00000200:14.0:1616513078.818264:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.227@tcp1 routed destination 00000400:00000200:14.0:1616513078.818280:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.227@tcp1(10.31.140.227@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.818286:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.818292:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.818295:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.818297:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.818302:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.818307:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.818358:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.818365:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.818372:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.818378:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.818382:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47562a8 00000800:00000200:28.0:1616513078.818386:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.818389:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.820507:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.820560:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.820569:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.820578:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.125@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.820587:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.125@tcp1 of length 224 into portal 12 MB=0x602db10292100 00000400:00000200:22.0:1616513078.820593:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.125@tcp1 of length 224/224 into md 0xf96450b9 [1] + 80992 00000400:00000200:22.0:1616513078.820598:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.820601:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513078.820607:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.820610:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513078.820612:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.820614:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.820617:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:14.0:1616513078.820670:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691989752488192 00000800:00000200:24.0:1616513078.820682:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:14.0:1616513078.820682:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec201680 x1691989752488192/t0(0) o400->1ab23b43-cf92-fe3c-37c0-92e158f55987@10.31.129.125@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513078.820691:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:14.0:1616513078.820697:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691989752488192, offset 192 00000400:00000200:24.0:1616513078.820700:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.125@tcp1 : PUT - for me 00000400:00000200:14.0:1616513078.820703:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.125@tcp1 00000400:00000200:24.0:1616513078.820709:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.125@tcp1 of length 224 into portal 26 MB=0x602db10291bc0 00000400:00000200:14.0:1616513078.820710:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.125@tcp1 routed destination 00000400:00000200:24.0:1616513078.820716:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.125@tcp1 of length 224/224 into md 0xf97c7c19 [1] + 0 00000400:00000200:24.0:1616513078.820720:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6ee0 00000400:00000200:24.0:1616513078.820723:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513078.820725:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.125@tcp1(10.31.129.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513078.820726:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.820730:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6ee0 00000800:00000200:14.0:1616513078.820731:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513078.820734:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.820736:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513078.820736:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513078.820737:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.820740:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513078.820742:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.820746:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513078.820751:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.820752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513078.820789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:15.0:1616513078.820802:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691989752486848 00000800:00000200:18.0:1616513078.820805:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.820812:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:15.0:1616513078.820813:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1691989752486848/t0(0) o400->58cc8f16-b9aa-90b3-8688-b02731291b4d@10.31.129.125@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513078.820815:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.820821:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:15.0:1616513078.820827:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691989752486848, offset 192 00000400:00000200:18.0:1616513078.820828:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756088 00000800:00000200:18.0:1616513078.820832:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:15.0:1616513078.820834:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.125@tcp1 00000800:00000200:18.0:1616513078.820835:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.820842:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:15.0:1616513078.820842:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.125@tcp1 routed destination 00000800:00000200:28.0:1616513078.820852:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:15.0:1616513078.820858:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.125@tcp1(10.31.129.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.820865:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.820868:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.75@tcp1 : PUT - for me 00000800:00000200:15.0:1616513078.820871:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.820874:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:15.0:1616513078.820876:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:28.0:1616513078.820878:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.75@tcp1 of length 224 into portal 26 MB=0x5ffd497c49f80 00000800:00000200:15.0:1616513078.820881:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513078.820884:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.75@tcp1 of length 224/224 into md 0xf97c7c39 [1] + 0 00000100:00000200:15.0:1616513078.820887:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513078.820889:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6c38 00000400:00000200:28.0:1616513078.820892:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.820895:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513078.820899:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:28.0:1616513078.820900:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6c38 00000800:00000200:28.0:1616513078.820905:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:20.0:1616513078.820908:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:28.0:1616513078.820909:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:28.0:1616513078.820910:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.820912:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513078.820913:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:15.0:1616513078.820914:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663427948416 00000400:00000200:20.0:1616513078.820918:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00010000:00000200:15.0:1616513078.820920:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688663427948416/t0(0) o400->66eadc11-aff6-2e24-a0ff-e2671493ed3f@10.31.138.75@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513078.820925:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0a770 00000800:00000200:20.0:1616513078.820928:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513078.820932:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663427948416, offset 192 00000400:00000200:15.0:1616513078.820937:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.75@tcp1 00000400:00000200:15.0:1616513078.820943:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.75@tcp1 routed destination 00000400:00000200:15.0:1616513078.820957:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.75@tcp1(10.31.138.75@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513078.820962:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:15.0:1616513078.820968:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513078.820970:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513078.820972:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513078.820976:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:15.0:1616513078.820980:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.820980:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.820989:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.820994:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513078.820997:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513078.820998:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.821002:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513078.821008:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0aff0 00000800:00000200:22.0:1616513078.821011:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.821013:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.821022:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.821028:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513078.821031:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.821035:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.75@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.821042:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.75@tcp1 of length 224 into portal 12 MB=0x5ffd497c4a040 00000400:00000200:24.0:1616513078.821047:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.75@tcp1 of length 224/224 into md 0x865bbec1 [1] + 13664 00000400:00000200:24.0:1616513078.821053:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.821055:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513078.821061:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.821063:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.821065:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513078.821127:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663427948608 00010000:00000200:4.0:1616513078.821139:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc4380 x1688663427948608/t0(0) o400->d3ef9ac3-93eb-947a-77ef-eaabd34f070d@10.31.138.75@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513078.821154:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663427948608, offset 192 00000400:00000200:4.0:1616513078.821161:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.75@tcp1 00000400:00000200:4.0:1616513078.821168:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.75@tcp1 routed destination 00000400:00000200:4.0:1616513078.821184:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.75@tcp1(10.31.138.75@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.821190:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.821197:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.821199:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.821202:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.821207:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.821211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.821221:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.821225:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513078.821227:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513078.821230:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513078.821233:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513078.821237:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba088 00000800:00000200:0.0:1616513078.821240:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.826393:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.826449:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.826459:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.826468:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.60@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.826478:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.60@tcp1 of length 224 into portal 12 MB=0x5ffdd92cc2e80 00000400:00000200:18.0:1616513078.826485:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.60@tcp1 of length 224/224 into md 0xf96450b9 [1] + 81216 00000400:00000200:18.0:1616513078.826490:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.826492:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.826499:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.826502:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.826503:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.826506:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.826519:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701999263360 00010000:00000200:14.0:1616513078.826532:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebf500 x1688701999263360/t0(0) o400->06e2be3c-11ab-8184-3795-506cbc0d67a9@10.31.129.60@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:31.0:1616513078.826539:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925344, portal 4 00000100:00000200:31.0:1616513078.826546:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925344, offset 0 00000100:00000200:14.0:1616513078.826546:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701999263360, offset 192 00000400:00000200:31.0:1616513078.826553:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:14.0:1616513078.826553:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.60@tcp1 00000400:00000200:31.0:1616513078.826559:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:14.0:1616513078.826562:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.60@tcp1 routed destination 00000400:00000200:31.0:1616513078.826574:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513078.826579:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.60@tcp1(10.31.129.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513078.826581:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:31.0:1616513078.826587:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.826588:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:31.0:1616513078.826591:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000100:00000200:0.0:1616513078.826592:0:6480:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925360, portal 4 00000800:00000200:31.0:1616513078.826593:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:14.0:1616513078.826594:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.826597:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513078.826597:0:6480:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925360, offset 0 00000800:00000200:31.0:1616513078.826598:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:14.0:1616513078.826600:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513078.826602:0:6480:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513078.826605:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513078.826608:0:6480:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:7.2:1616513078.826610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513078.826619:0:6480:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.826625:0:6480:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:0.0:1616513078.826630:0:6480:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.826633:0:6480:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:0.0:1616513078.826635:0:6480:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:0.0:1616513078.826639:0:6480:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:5.2:1616513078.826645:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513078.826660:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:26.0:1616513078.826663:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513078.826667:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:26.0:1616513078.826670:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513078.826671:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.826673:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.826673:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:22.0:1616513078.826676:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513078.826678:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.826679:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:28.0:1616513078.826684:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be47080 x1693067725925344/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513078.826685:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756110 00000800:00000200:25.2:1616513078.826686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513078.826686:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513078.826694:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513078.826694:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.229@tcp1 : PUT - for me 00000800:00000200:18.0:1616513078.826696:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513078.826697:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c27f8 00000800:00000200:26.0:1616513078.826699:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:28.0:1616513078.826701:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:26.0:1616513078.826704:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513078.826704:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.229@tcp1 of length 224 into portal 26 MB=0x604a2a2eda780 00000400:00000200:26.0:1616513078.826705:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513078.826706:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:26.0:1616513078.826711:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:22.0:1616513078.826711:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.229@tcp1 of length 224/224 into md 0xf97c7c49 [1] + 0 00000400:00000200:22.0:1616513078.826715:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7ed0 00000100:00000200:26.0:1616513078.826716:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affc7980 x1693067725925360/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513078.826718:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.826723:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513078.826727:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05e48 00000800:00000200:11.2:1616513078.826727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513078.826729:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7ed0 00000800:00000200:26.0:1616513078.826730:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:22.0:1616513078.826732:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.826735:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.826737:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513078.826754:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946424960896 00010000:00000200:19.0:1616513078.826765:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1693946424960896/t0(0) o400->11d5f9ec-d3d6-7b5d-5da4-3d28eb32ed91@10.31.140.229@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513078.826779:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946424960896, offset 192 00000800:00000200:20.0:1616513078.826783:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:19.0:1616513078.826786:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.229@tcp1 00000800:00000200:20.0:1616513078.826793:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:19.0:1616513078.826794:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.229@tcp1 routed destination 00000800:00000200:0.0:1616513078.826795:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.826804:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.60@tcp1 : PUT - for me 00000400:00000200:19.0:1616513078.826811:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.229@tcp1(10.31.140.229@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513078.826817:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.826821:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.60@tcp1 of length 224 into portal 26 MB=0x5ffdd92cc2b00 00000800:00000200:19.0:1616513078.826824:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:5.2:1616513078.826826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:19.0:1616513078.826827:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513078.826828:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.60@tcp1 of length 224/224 into md 0xf97c7c69 [1] + 0 00000800:00000200:19.0:1616513078.826829:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513078.826833:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c7760 00000800:00000200:19.0:1616513078.826834:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.826836:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.826837:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:20.0:1616513078.826839:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:11.2:1616513078.826841:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513078.826844:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c7760 00000800:00000200:24.0:1616513078.826845:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000100:00000200:19.0:1616513078.826845:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513078.826847:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513078.826847:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513078.826850:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:20.0:1616513078.826850:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513078.826852:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:25.2:1616513078.826852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:20.0:1616513078.826852:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:24.0:1616513078.826853:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513078.826854:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513078.826854:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.826860:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.826862:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626be0 00000800:00000200:2.0:1616513078.826863:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000400:00000200:28.0:1616513078.826865:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707210 00000800:00000200:28.0:1616513078.826868:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513078.826868:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701999262464 00000400:00000200:24.0:1616513078.826869:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d4e21 [1] + 192 00000800:00000200:2.0:1616513078.826870:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513078.826872:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:24.0:1616513078.826874:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.826879:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be47080 x1693067725925344/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00010000:00000200:12.0:1616513078.826879:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688701999262464/t0(0) o400->2f012bef-8839-3f88-1ce6-2cb0cd305c49@10.31.129.60@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513078.826879:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513078.826887:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626bf0 00000800:00000200:24.0:1616513078.826894:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:12.0:1616513078.826894:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701999262464, offset 192 00000400:00000200:2.0:1616513078.826894:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c82b9 [1] + 192 00000800:00000200:24.0:1616513078.826897:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513078.826899:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000400:00000200:2.0:1616513078.826899:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:12.0:1616513078.826900:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.60@tcp1 00000800:00000200:11.2:1616513078.826902:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:2.0:1616513078.826904:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affc7980 x1693067725925360/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513078.826906:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be47080 x1693067725925344/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513078.826908:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.60@tcp1 routed destination 00000800:00000200:20.0:1616513078.826911:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.826917:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:31.0:1616513078.826920:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be47080 x1693067725925344/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:0.0:1616513078.826921:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.826925:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.229@tcp1 : PUT - for me 00000400:00000200:12.0:1616513078.826925:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.60@tcp1(10.31.129.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513078.826929:0:6480:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affc7980 x1693067725925360/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513078.826930:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000800:00000200:2.0:1616513078.826931:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000400:00000200:20.0:1616513078.826932:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.229@tcp1 of length 224 into portal 12 MB=0x604a2a2eda880 00000800:00000200:12.0:1616513078.826932:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513078.826934:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513078.826936:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000400:00000200:20.0:1616513078.826937:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.229@tcp1 of length 224/224 into md 0x865bbec1 [1] + 13888 00000800:00000200:12.0:1616513078.826938:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513078.826940:0:6480:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affc7980 x1693067725925360/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513078.826941:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513078.826942:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513078.826943:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513078.826944:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513078.826948:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513078.826949:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513078.826949:0:6480:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000800:00000200:20.0:1616513078.826951:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.826953:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.826953:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513078.826956:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513078.826962:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.826966:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.826968:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513078.826971:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.826973:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.826977:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782406e8 00000800:00000200:26.0:1616513078.826979:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513078.827017:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946424961152 00010000:00000200:4.0:1616513078.827029:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e00480 x1693946424961152/t0(0) o400->12ba146b-d6fe-3a51-dcb7-064c5a048026@10.31.140.229@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513078.827044:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946424961152, offset 192 00000400:00000200:4.0:1616513078.827050:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.229@tcp1 00000400:00000200:4.0:1616513078.827057:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.229@tcp1 routed destination 00000400:00000200:4.0:1616513078.827072:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.229@tcp1(10.31.140.229@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.827078:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.827084:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.827087:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.827089:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.827094:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.827098:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.827107:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.827111:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.827112:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.827116:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.827117:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.827120:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bab28 00000800:00000200:28.0:1616513078.827122:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.838094:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.838149:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.838158:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.838167:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.174@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.838176:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.174@tcp1 of length 224 into portal 26 MB=0x5ffd46a406d80 00000400:00000200:18.0:1616513078.838183:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.174@tcp1 of length 224/224 into md 0xf97c7c99 [1] + 0 00000400:00000200:18.0:1616513078.838188:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c6088 00000400:00000200:18.0:1616513078.838191:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.838193:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513078.838197:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c6088 00000800:00000200:18.0:1616513078.838200:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513078.838202:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.838203:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.838206:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513078.838260:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662664310144 00010000:00000200:12.0:1616513078.838271:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688662664310144/t0(0) o400->4a3e0de9-6641-e810-cb94-71d4264471ff@10.31.138.174@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.838296:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662664310144, offset 192 00000400:00000200:12.0:1616513078.838302:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.174@tcp1 00000400:00000200:12.0:1616513078.838310:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.174@tcp1 routed destination 00000800:00000200:11.2:1616513078.838320:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:12.0:1616513078.838325:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.174@tcp1(10.31.138.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.838331:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513078.838337:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.838339:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513078.838340:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513078.838342:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.838347:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513078.838347:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513078.838352:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513078.838353:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513078.838355:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.174@tcp1 : PUT - for me 00000400:00000200:0.0:1616513078.838362:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.174@tcp1 of length 224 into portal 12 MB=0x5ffd46a406e00 00000400:00000200:0.0:1616513078.838370:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.174@tcp1 of length 224/224 into md 0x865bbec1 [1] + 14112 00000400:00000200:0.0:1616513078.838374:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513078.838377:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513078.838383:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.838386:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513078.838388:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.838390:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513078.838450:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.838457:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513078.838460:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:4.0:1616513078.838464:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662664310272 00000400:00000200:22.0:1616513078.838466:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513078.838472:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241320 00000800:00000200:22.0:1616513078.838475:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:4.0:1616513078.838477:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86d580 x1688662664310272/t0(0) o400->a58adc60-b703-f826-be94-e04df37cd365@10.31.138.174@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:22.0:1616513078.838478:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513078.838491:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662664310272, offset 192 00000400:00000200:4.0:1616513078.838497:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.174@tcp1 00000400:00000200:4.0:1616513078.838505:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.174@tcp1 routed destination 00000400:00000200:4.0:1616513078.838520:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.174@tcp1(10.31.138.174@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.838526:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.838532:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.838535:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.838537:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.838541:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.838547:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.838602:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.838609:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.838612:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.838618:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.838622:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba220 00000800:00000200:24.0:1616513078.838626:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.838629:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.846532:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513078.846586:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513078.846596:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513078.846604:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.85@tcp1 : PUT - for me 00000400:00000200:2.0:1616513078.846613:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.85@tcp1 of length 224 into portal 26 MB=0x5ffe286585300 00000400:00000200:2.0:1616513078.846620:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.85@tcp1 of length 224/224 into md 0xf97c7cb9 [1] + 0 00000400:00000200:2.0:1616513078.846625:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4bd3c73a8 00000400:00000200:2.0:1616513078.846628:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513078.846631:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513078.846635:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4bd3c73a8 00000800:00000200:18.0:1616513078.846638:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513078.846638:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513078.846641:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513078.846645:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.846648:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513078.846648:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.846656:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.85@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.846664:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.85@tcp1 of length 224 into portal 12 MB=0x5ffe286585380 00000400:00000200:18.0:1616513078.846670:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.85@tcp1 of length 224/224 into md 0xf96450b9 [1] + 81440 00000400:00000200:18.0:1616513078.846677:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.846679:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.846685:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.846688:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.846690:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513078.846696:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688723265180416 00010000:00000200:12.0:1616513078.846705:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688723265180416/t0(0) o400->64437f88-811c-0544-2652-ddc54ba39cfc@10.31.128.85@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.846719:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688723265180416, offset 192 00000400:00000200:12.0:1616513078.846725:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.85@tcp1 00000400:00000200:12.0:1616513078.846732:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.85@tcp1 routed destination 00000400:00000200:12.0:1616513078.846746:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.85@tcp1(10.31.128.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:14.0:1616513078.846747:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688723265180544 00000800:00000200:12.0:1616513078.846752:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:14.0:1616513078.846758:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebe300 x1688723265180544/t0(0) o400->1d141320-a4ea-5eb2-1aed-85ba5145da33@10.31.128.85@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513078.846758:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.846760:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513078.846762:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.846767:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513078.846772:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688723265180544, offset 192 00000800:00000200:7.2:1616513078.846772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513078.846773:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513078.846777:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.85@tcp1 00000400:00000200:14.0:1616513078.846784:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.85@tcp1 routed destination 00000400:00000200:14.0:1616513078.846798:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.85@tcp1(10.31.128.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.846804:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.846809:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.846812:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.846814:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.846818:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513078.846825:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.846832:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.846835:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513078.846836:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.846841:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.846847:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a90 00000800:00000200:26.0:1616513078.846851:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513078.846864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.846919:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.846925:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.846928:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.846934:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.846938:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47577e8 00000800:00000200:20.0:1616513078.846941:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.846944:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.850415:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.850469:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.850478:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.850487:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.219@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.850496:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.219@tcp1 of length 224 into portal 26 MB=0x5ffe6cfc64300 00000400:00000200:24.0:1616513078.850502:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.219@tcp1 of length 224/224 into md 0xf97c7cc9 [1] + 0 00000400:00000200:24.0:1616513078.850507:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd825d8 00000400:00000200:24.0:1616513078.850509:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.850514:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.850518:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd825d8 00000800:00000200:24.0:1616513078.850521:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.850522:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.850523:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.850528:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.850531:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513078.850531:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.850539:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.219@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.850548:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.219@tcp1 of length 224 into portal 12 MB=0x5ffe6cfc64380 00000400:00000200:28.0:1616513078.850554:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.219@tcp1 of length 224/224 into md 0x865bbec1 [1] + 14336 00000400:00000200:28.0:1616513078.850559:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.850561:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513078.850567:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.850570:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.850573:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513078.850577:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688741676991232 00010000:00000200:12.0:1616513078.850587:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688741676991232/t0(0) o400->7ee65fd6-cec9-11fd-eb82-b2987ec2d48c@10.31.136.219@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.850600:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688741676991232, offset 192 00000400:00000200:12.0:1616513078.850605:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.219@tcp1 00000400:00000200:12.0:1616513078.850612:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.219@tcp1 routed destination 00000400:00000200:12.0:1616513078.850628:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.219@tcp1(10.31.136.219@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513078.850629:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688741676991360 00000800:00000200:12.0:1616513078.850634:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513078.850639:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.850641:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:4.0:1616513078.850641:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6720000 x1688741676991360/t0(0) o400->bd6e16df-64b7-d35c-945c-99a439a26491@10.31.136.219@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513078.850643:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.850648:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.850653:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513078.850654:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:4.0:1616513078.850655:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688741676991360, offset 192 00000400:00000200:4.0:1616513078.850660:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.219@tcp1 00000400:00000200:4.0:1616513078.850667:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.219@tcp1 routed destination 00000400:00000200:4.0:1616513078.850683:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.219@tcp1(10.31.136.219@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.850689:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.850694:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.850696:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.850699:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.850703:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513078.850704:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513078.850708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513078.850710:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.850713:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.850719:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.850724:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240b28 00000800:00000200:18.0:1616513078.850727:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.850730:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513078.850774:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.850781:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513078.850783:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.850785:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513078.850789:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513078.850792:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb320 00000800:00000200:0.0:1616513078.850796:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513078.858571:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925376, portal 4 00000100:00000200:19.0:1616513078.858577:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925376, offset 0 00000400:00000200:19.0:1616513078.858584:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:19.0:1616513078.858589:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:19.0:1616513078.858602:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513078.858609:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:19.0:1616513078.858615:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513078.858618:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:19.0:1616513078.858620:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:19.0:1616513078.858625:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513078.858672:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.858692:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513078.858697:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513078.858700:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513078.858707:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513078.858713:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be40480 x1693067725925376/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513078.858737:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706880 00000800:00000200:1.0:1616513078.858742:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:5.0:1616513078.858745:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513078.858856:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513078.858866:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.858872:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513078.858881:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513078.858881:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513078.858889:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626c00 00000400:00000200:1.0:1616513078.858895:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffb45 [1] + 192 00000400:00000200:1.0:1616513078.858900:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513078.858904:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be40480 x1693067725925376/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513078.858917:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.858920:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513078.858921:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:19.0:1616513078.858924:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be40480 x1693067725925376/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513078.858937:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be40480 x1693067725925376/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513078.858947:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000800:00000200:7.2:1616513078.860603:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513078.860659:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513078.860670:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513078.860679:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.27@tcp1 : PUT - for me 00000400:00000200:2.0:1616513078.860690:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.27@tcp1 of length 224 into portal 26 MB=0x5ffd3e012d140 00000400:00000200:2.0:1616513078.860697:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.27@tcp1 of length 224/224 into md 0xf97c7cd9 [1] + 0 00000400:00000200:2.0:1616513078.860701:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82f68 00000400:00000200:2.0:1616513078.860704:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513078.860707:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513078.860711:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82f68 00000800:00000200:26.0:1616513078.860714:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513078.860714:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513078.860717:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513078.860719:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513078.860771:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660346065216 00010000:00000200:12.0:1616513078.860780:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688660346065216/t0(0) o400->564e6f62-379a-647f-0b86-e96cd6b5311b@10.31.138.27@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.860792:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660346065216, offset 192 00000400:00000200:12.0:1616513078.860798:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.27@tcp1 00000400:00000200:12.0:1616513078.860805:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.27@tcp1 routed destination 00000400:00000200:12.0:1616513078.860818:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.27@tcp1(10.31.138.27@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.860824:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513078.860829:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.860831:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513078.860833:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.860837:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513078.860843:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.860843:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513078.860863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513078.860894:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.860901:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.860903:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.860909:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.860915:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240110 00000800:00000200:18.0:1616513078.860918:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513078.860921:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513078.860921:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513078.860930:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.860939:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.27@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.860948:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.27@tcp1 of length 224 into portal 12 MB=0x5ffd3e012d2c0 00000400:00000200:28.0:1616513078.860955:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.27@tcp1 of length 224/224 into md 0x865bbec1 [1] + 14560 00000400:00000200:28.0:1616513078.860960:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.860962:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513078.860969:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.860972:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.860974:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.860977:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513078.861030:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660346065600 00010000:00000200:4.0:1616513078.861042:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6725100 x1688660346065600/t0(0) o400->e6acbcd2-ae9a-c45e-4262-8d55621b1241@10.31.138.27@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513078.861055:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660346065600, offset 192 00000400:00000200:4.0:1616513078.861061:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.27@tcp1 00000400:00000200:4.0:1616513078.861068:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.27@tcp1 routed destination 00000400:00000200:4.0:1616513078.861082:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.27@tcp1(10.31.138.27@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513078.861088:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513078.861094:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.861096:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.861098:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513078.861103:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.861108:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.861162:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.861169:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.861171:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.861178:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.861182:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb760 00000800:00000200:24.0:1616513078.861186:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.861189:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:23.0:1616513078.870541:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:23.0:1616513078.870548:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513078.874789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513078.874843:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.874852:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.874860:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.95@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.874869:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.95@tcp1 of length 224 into portal 26 MB=0x5ffd4f0c33c40 00000400:00000200:28.0:1616513078.874875:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.95@tcp1 of length 224/224 into md 0xf97c7d09 [1] + 0 00000400:00000200:28.0:1616513078.874880:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd822a8 00000400:00000200:28.0:1616513078.874883:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.874886:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.874890:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd822a8 00000800:00000200:28.0:1616513078.874893:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.874895:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.874896:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513078.874900:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513078.874947:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664921029696 00010000:00000200:12.0:1616513078.874957:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688664921029696/t0(0) o400->59005034-7f69-0ea2-469e-6fa743b9b70c@10.31.138.95@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.874968:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664921029696, offset 192 00000400:00000200:12.0:1616513078.874974:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.95@tcp1 00000400:00000200:12.0:1616513078.874981:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.95@tcp1 routed destination 00000400:00000200:12.0:1616513078.874994:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.95@tcp1(10.31.138.95@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.874999:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513078.875004:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.875008:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513078.875010:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.875014:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513078.875019:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.875020:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.875076:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.875083:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.875086:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.875092:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.875098:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240770 00000800:00000200:26.0:1616513078.875101:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.875104:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513078.875498:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.875551:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.875560:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.875568:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.95@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.875577:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.95@tcp1 of length 224 into portal 12 MB=0x5ffd4f0c33d40 00000400:00000200:18.0:1616513078.875583:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.95@tcp1 of length 224/224 into md 0xf96450b9 [1] + 81664 00000400:00000200:18.0:1616513078.875588:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.875590:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.875596:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.875598:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.875600:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.875603:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.875657:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664921029952 00010000:00000200:14.0:1616513078.875668:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9f80 x1688664921029952/t0(0) o400->bdd77829-e716-9f34-5ab5-b887459a0cc5@10.31.138.95@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.875681:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664921029952, offset 192 00000400:00000200:14.0:1616513078.875687:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.95@tcp1 00000400:00000200:14.0:1616513078.875694:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.95@tcp1 routed destination 00000400:00000200:14.0:1616513078.875710:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.95@tcp1(10.31.138.95@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.875716:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.875721:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.875724:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.875726:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.875730:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.875736:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.875790:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.875797:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.875799:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513078.875805:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513078.875806:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.875810:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757100 00000800:00000200:24.0:1616513078.875814:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.893348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513078.893404:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.893413:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.893422:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.192@tcp1 : PUT - for me 00000400:00000200:20.0:1616513078.893446:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.192@tcp1 of length 224 into portal 26 MB=0x5ffdcff3a0800 00000400:00000200:20.0:1616513078.893453:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.192@tcp1 of length 224/224 into md 0xf97c7d19 [1] + 0 00000800:00000200:28.0:1616513078.893458:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513078.893458:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83188 00000400:00000200:20.0:1616513078.893461:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.893463:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513078.893468:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513078.893468:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83188 00000800:00000200:20.0:1616513078.893471:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513078.893474:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513078.893476:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.893477:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.192@tcp1 : PUT - for me 00000800:00000200:20.0:1616513078.893478:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.893484:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.893486:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.192@tcp1 of length 224 into portal 12 MB=0x5ffdcff3a0a40 00000400:00000200:20.0:1616513078.893491:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.230@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.893492:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.192@tcp1 of length 224/224 into md 0x865bbec1 [1] + 14784 00000400:00000200:28.0:1616513078.893498:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.893498:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.230@tcp1 of length 224 into portal 26 MB=0x5ffd44f0b0fc0 00000100:00000200:28.0:1616513078.893500:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513078.893503:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.230@tcp1 of length 224/224 into md 0xf97c7d39 [1] + 0 00000400:00000200:20.0:1616513078.893506:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83430 00000800:00000200:28.0:1616513078.893507:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:20.0:1616513078.893508:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513078.893510:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513078.893512:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513078.893514:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.893514:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83430 00000800:00000200:28.0:1616513078.893516:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.893516:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.893518:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513078.893520:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513078.893520:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.893522:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.893527:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.230@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.893534:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.230@tcp1 of length 224 into portal 12 MB=0x5ffd44f0b1080 00000400:00000200:28.0:1616513078.893539:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.230@tcp1 of length 224/224 into md 0x865bbec1 [1] + 15008 00000400:00000200:28.0:1616513078.893543:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.893544:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513078.893549:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.893552:0:6461:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.893555:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513078.893556:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699523434496 00000800:00000200:28.0:1616513078.893557:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513078.893557:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662207827904 00000800:00000200:11.2:1616513078.893561:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:8.0:1616513078.893566:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688699523434496/t0(0) o400->3cbcef72-e585-b8fb-cec9-d328ff3ed248@10.31.141.192@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:12.0:1616513078.893568:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688662207827904/t0(0) o400->ed2bad13-1b50-5d30-3dcd-2e69a1df0da2@10.31.132.230@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513078.893570:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.893575:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513078.893577:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513078.893579:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513078.893579:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699523434496, offset 192 00000100:00000200:4.0:1616513078.893579:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699523435072 00000100:00000200:24.0:1616513078.893582:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662207828096 00000100:00000200:12.0:1616513078.893582:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662207827904, offset 192 00000400:00000200:8.0:1616513078.893585:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.192@tcp1 00000400:00000200:12.0:1616513078.893590:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.230@tcp1 00010000:00000200:4.0:1616513078.893590:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6723180 x1688699523435072/t0(0) o400->22d28a3a-a44a-faac-36a6-71004361f228@10.31.141.192@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:8.0:1616513078.893592:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.192@tcp1 routed destination 00010000:00000200:24.0:1616513078.893595:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6721b00 x1688662207828096/t0(0) o400->592c02f9-a77b-370d-4000-af6bfc233f2c@10.31.132.230@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513078.893602:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.230@tcp1 routed destination 00000100:00000200:4.0:1616513078.893602:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699523435072, offset 192 00000400:00000200:8.0:1616513078.893606:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.192@tcp1(10.31.141.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:4.0:1616513078.893608:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.192@tcp1 00000100:00000200:24.0:1616513078.893611:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662207828096, offset 192 00000800:00000200:8.0:1616513078.893613:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513078.893617:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.230@tcp1 00000400:00000200:12.0:1616513078.893617:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.230@tcp1(10.31.132.230@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.893623:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:4.0:1616513078.893625:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.192@tcp1 routed destination 00000800:00000200:8.0:1616513078.893632:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513078.893634:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.230@tcp1 routed destination 00000800:00000200:8.0:1616513078.893634:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513078.893636:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513078.893638:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.192@tcp1(10.31.141.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513078.893641:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513078.893644:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:8.0:1616513078.893648:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.893648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513078.893649:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.230@tcp1(10.31.132.230@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513078.893656:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513078.893657:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.893657:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.893659:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513078.893659:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513078.893661:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.893662:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513078.893663:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:24.0:1616513078.893664:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513078.893666:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.893667:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513078.893667:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:24.0:1616513078.893671:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:11.2:1616513078.893671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:12.0:1616513078.893673:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513078.893690:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513078.893697:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:24.0:1616513078.893699:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513078.893700:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:26.0:1616513078.893702:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:28.0:1616513078.893704:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:24.0:1616513078.893704:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.893706:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.893708:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240b28 00000800:00000200:26.0:1616513078.893709:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:0.0:1616513078.893710:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.893711:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.893712:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.893714:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.893715:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:28.0:1616513078.893718:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513078.893719:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.893724:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71ed0 00000800:00000200:26.0:1616513078.893730:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513078.893732:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.893736:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.893737:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.893742:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.893745:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb3a8 00000800:00000200:26.0:1616513078.893747:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.893750:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513078.896094:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.896147:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.896155:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.896164:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.125@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.896173:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.125@tcp1 of length 224 into portal 12 MB=0x603fc0c85b600 00000400:00000200:18.0:1616513078.896181:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.125@tcp1 of length 224/224 into md 0xf96450b9 [1] + 81888 00000400:00000200:18.0:1616513078.896186:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.896188:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.896193:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.896196:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.896198:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.896200:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.896211:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513078.896219:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.125@tcp1 : PUT - for me 00000400:00000200:26.0:1616513078.896228:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.125@tcp1 of length 224 into portal 26 MB=0x603fc0c85b540 00000400:00000200:26.0:1616513078.896235:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.125@tcp1 of length 224/224 into md 0xf97c7d49 [1] + 0 00000400:00000200:26.0:1616513078.896240:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83e48 00000400:00000200:26.0:1616513078.896243:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513078.896245:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513078.896249:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83e48 00000800:00000200:26.0:1616513078.896252:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:14.0:1616513078.896253:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230936995328 00000800:00000200:26.0:1616513078.896255:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513078.896257:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.896260:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:14.0:1616513078.896263:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9680 x1693230936995328/t0(0) o400->a79b57f5-0f99-b227-e46c-eaac5850f334@10.31.138.125@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.896276:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230936995328, offset 192 00000400:00000200:14.0:1616513078.896281:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.125@tcp1 00000400:00000200:14.0:1616513078.896288:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.125@tcp1 routed destination 00000400:00000200:14.0:1616513078.896301:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.125@tcp1(10.31.138.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.896306:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513078.896311:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:12.0:1616513078.896313:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230936995136 00000800:00000200:14.0:1616513078.896314:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513078.896316:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.896320:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:12.0:1616513078.896324:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1693230936995136/t0(0) o400->31d0af92-2e6a-ffeb-c238-923a92c4ae45@10.31.138.125@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513078.896325:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.896337:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:12.0:1616513078.896338:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230936995136, offset 192 00000400:00000200:12.0:1616513078.896344:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.125@tcp1 00000800:00000200:22.0:1616513078.896345:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513078.896347:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513078.896349:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513078.896352:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.125@tcp1 routed destination 00000400:00000200:22.0:1616513078.896354:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513078.896358:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47575c8 00000800:00000200:22.0:1616513078.896362:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513078.896366:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.125@tcp1(10.31.138.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.896373:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513078.896378:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.896381:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513078.896383:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513078.896388:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513078.896394:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513078.896435:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.896490:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.896497:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.896500:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513078.896506:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513078.896506:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.896512:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241320 00000800:00000200:20.0:1616513078.896515:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.898924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513078.898980:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.898989:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.898997:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.195@tcp1 : PUT - for me 00000400:00000200:24.0:1616513078.899006:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.195@tcp1 of length 224 into portal 26 MB=0x605a0731118c0 00000400:00000200:24.0:1616513078.899013:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.195@tcp1 of length 224/224 into md 0xf97c7d69 [1] + 0 00000400:00000200:24.0:1616513078.899018:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83298 00000400:00000200:24.0:1616513078.899021:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.899023:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.899027:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83298 00000800:00000200:24.0:1616513078.899031:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.899032:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.899033:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.899037:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513078.899090:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036543670464 00010000:00000200:12.0:1616513078.899100:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1695036543670464/t0(0) o400->80bf5179-5d95-2311-2bf9-267d0670ac30@10.31.129.195@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513078.899105:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:12.0:1616513078.899114:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036543670464, offset 192 00000400:00000200:12.0:1616513078.899120:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.195@tcp1 00000400:00000200:12.0:1616513078.899128:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.195@tcp1 routed destination 00000400:00000200:12.0:1616513078.899143:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.195@tcp1(10.31.129.195@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.899150:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513078.899155:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.899158:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.899161:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513078.899161:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513078.899165:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513078.899170:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:12.0:1616513078.899171:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513078.899179:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.195@tcp1 : PUT - for me 00000400:00000200:2.0:1616513078.899187:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.195@tcp1 of length 224 into portal 12 MB=0x605a073111940 00000400:00000200:2.0:1616513078.899194:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.195@tcp1 of length 224/224 into md 0xf96450b9 [1] + 82112 00000400:00000200:2.0:1616513078.899199:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513078.899201:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513078.899208:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513078.899210:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513078.899212:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513078.899215:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.899223:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.899225:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.899232:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513078.899237:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782417e8 00000800:00000200:26.0:1616513078.899241:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.899244:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.899266:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036543670592 00010000:00000200:14.0:1616513078.899277:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621b00 x1695036543670592/t0(0) o400->fd1adc7a-681b-cd02-94e3-3cc48b5002e0@10.31.129.195@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.899289:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036543670592, offset 192 00000400:00000200:14.0:1616513078.899294:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.195@tcp1 00000400:00000200:14.0:1616513078.899301:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.195@tcp1 routed destination 00000400:00000200:14.0:1616513078.899313:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.195@tcp1(10.31.129.195@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.899319:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.899324:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.899326:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.899330:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.899334:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.899339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.899393:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.899399:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513078.899402:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513078.899408:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513078.899412:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757ed0 00000800:00000200:0.0:1616513078.899416:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.899419:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.902634:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.902702:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.902712:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.902721:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.135@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.902731:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.135@tcp1 of length 224 into portal 12 MB=0x603fc0c64ba80 00000400:00000200:22.0:1616513078.902737:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.135@tcp1 of length 224/224 into md 0xf96450b9 [1] + 82336 00000400:00000200:22.0:1616513078.902742:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.902745:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513078.902752:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.902754:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513078.902756:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513078.902760:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.902810:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230934833792 00010000:00000200:14.0:1616513078.902820:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec201f80 x1693230934833792/t0(0) o400->0b66f511-5915-fd0f-3ed8-0a4086e86ea7@10.31.138.135@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.902833:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230934833792, offset 192 00000400:00000200:14.0:1616513078.902839:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.135@tcp1 00000400:00000200:14.0:1616513078.902846:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.135@tcp1 routed destination 00000400:00000200:14.0:1616513078.902859:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.135@tcp1(10.31.138.135@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.902864:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513078.902869:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.902872:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513078.902874:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.902878:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.902883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513078.902898:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513078.902936:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.902943:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513078.902946:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513078.902952:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513078.902956:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513078.902957:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756dd0 00000800:00000200:26.0:1616513078.902960:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513078.902965:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513078.902965:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513078.902974:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.135@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.902983:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.135@tcp1 of length 224 into portal 26 MB=0x603fc0c64ba00 00000400:00000200:28.0:1616513078.902990:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.135@tcp1 of length 224/224 into md 0xf97c7d89 [1] + 0 00000400:00000200:28.0:1616513078.902994:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82770 00000400:00000200:28.0:1616513078.902997:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.903000:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.903004:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82770 00000800:00000200:28.0:1616513078.903007:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.903010:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.903012:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.903016:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513078.903069:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230934833664 00010000:00000200:12.0:1616513078.903080:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1693230934833664/t0(0) o400->e76d766f-37b5-8f15-a9b1-b9cacd6e09f9@10.31.138.135@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.903094:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230934833664, offset 192 00000400:00000200:12.0:1616513078.903100:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.135@tcp1 00000400:00000200:12.0:1616513078.903108:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.135@tcp1 routed destination 00000400:00000200:12.0:1616513078.903123:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.135@tcp1(10.31.138.135@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.903129:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513078.903135:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.903138:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513078.903140:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513078.903144:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.903150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513078.903151:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513078.903205:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.903212:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513078.903214:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513078.903221:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513078.903227:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ee0 00000800:00000200:0.0:1616513078.903230:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.903233:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513078.908661:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0c00] (20)++ 00000800:00000200:24.0:1616513078.908717:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (21)++ 00000800:00000200:24.0:1616513078.908726:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.225@o2ib1 00000400:00000200:24.0:1616513078.908735:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.225@o2ib1 : PUT - for me 00000400:00000200:24.0:1616513078.908742:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.225@o2ib1 of length 224 into portal 26 MB=0x5ff4f8848c640 00000400:00000200:24.0:1616513078.908749:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.225@o2ib1 of length 224/224 into md 0xf97c7d99 [1] + 0 00000400:00000200:24.0:1616513078.908754:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82ee0 00000400:00000200:24.0:1616513078.908756:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.908759:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513078.908763:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82ee0 00000800:00000200:24.0:1616513078.908766:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0c00] (22)++ 00000800:00000200:24.0:1616513078.908769:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0c00] (23)-- 00000800:00000200:28.0:1616513078.908770:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (23)-- 00000800:00000200:24.0:1616513078.908776:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (21)-- 00000100:00000200:12.0:1616513078.908830:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688091937523264 00010000:00000200:12.0:1616513078.908840:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688091937523264/t0(0) o400->fdb8ae67-11cf-c699-6cf2-70170cd4b5ec@10.242.46.225@o2ib1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.908854:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688091937523264, offset 192 00000400:00000200:12.0:1616513078.908862:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.225@o2ib1 00000400:00000200:12.0:1616513078.908869:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.225@o2ib1 local destination 00000400:00000200:12.0:1616513078.908883:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.225@o2ib1(10.242.46.225@o2ib1:10.242.46.225@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.908889:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.225@o2ib1 00000800:00000200:12.0:1616513078.908895:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f00] -> 10.242.46.225@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.908897:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0c00] (20)++ 00000800:00000200:12.0:1616513078.908900:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0c00] (21)++ 00000800:00000200:12.0:1616513078.908905:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0c00] (22)-- 00000100:00000200:12.0:1616513078.908911:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513078.908917:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0c00] (21)++ 00000800:00000200:0.0:1616513078.908981:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (22)++ 00000800:00000200:0.0:1616513078.908988:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0c00] (23)-- 00000400:00000200:0.0:1616513078.908991:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513078.908998:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.225@o2ib1: PUT: OK 00000400:00000200:0.0:1616513078.909003:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240dd0 00000800:00000200:0.0:1616513078.909008:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (22)-- 00000800:00000200:0.0:1616513078.909011:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0c00] (21)-- 00000800:00000200:11.2:1616513078.912769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513078.912825:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.912834:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513078.912843:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.93@tcp1 : PUT - for me 00000400:00000200:28.0:1616513078.912852:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.93@tcp1 of length 224 into portal 26 MB=0x5ffd5dfeb2840 00000400:00000200:28.0:1616513078.912858:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.93@tcp1 of length 224/224 into md 0xf97c7da9 [1] + 0 00000400:00000200:28.0:1616513078.912864:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83dc0 00000400:00000200:28.0:1616513078.912866:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513078.912871:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513078.912876:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83dc0 00000800:00000200:28.0:1616513078.912879:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.912881:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513078.912881:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.912885:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513078.912939:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668933400640 00010000:00000200:12.0:1616513078.912950:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688668933400640/t0(0) o400->20a24bd7-f156-5ce7-e588-885bf1c43107@10.31.136.93@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.912964:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668933400640, offset 192 00000400:00000200:12.0:1616513078.912970:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.93@tcp1 00000400:00000200:12.0:1616513078.912977:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.93@tcp1 routed destination 00000400:00000200:12.0:1616513078.912992:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.93@tcp1(10.31.136.93@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.912999:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513078.913005:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.913007:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513078.913009:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.913014:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513078.913020:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.913020:0:49327:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513078.913076:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513078.913083:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513078.913086:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513078.913092:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513078.913098:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240aa0 00000800:00000200:2.0:1616513078.913101:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513078.913104:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513078.913293:0:49327:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513078.913351:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.913360:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513078.913369:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.93@tcp1 : PUT - for me 00000400:00000200:26.0:1616513078.913377:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.93@tcp1 of length 224 into portal 12 MB=0x5ffd5dfeb2980 00000400:00000200:26.0:1616513078.913384:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.93@tcp1 of length 224/224 into md 0xf96450b9 [1] + 82560 00000400:00000200:26.0:1616513078.913389:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513078.913392:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513078.913398:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.913401:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513078.913403:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.913406:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.913459:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668933400960 00010000:00000200:14.0:1616513078.913469:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9f80 x1688668933400960/t0(0) o400->5df8e835-6da4-e805-4cc0-2ade00c3dd85@10.31.136.93@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.913482:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668933400960, offset 192 00000400:00000200:14.0:1616513078.913487:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.93@tcp1 00000400:00000200:14.0:1616513078.913494:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.93@tcp1 routed destination 00000400:00000200:14.0:1616513078.913507:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.93@tcp1(10.31.136.93@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.913512:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.913517:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.913519:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.913521:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.913525:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.913530:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.913584:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.913591:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513078.913593:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513078.913600:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513078.913604:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756440 00000800:00000200:0.0:1616513078.913607:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.913610:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.920006:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513078.920062:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513078.920071:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513078.920079:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.244@tcp1 : PUT - for me 00000400:00000200:2.0:1616513078.920088:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.244@tcp1 of length 224 into portal 26 MB=0x604a612bfbb80 00000400:00000200:2.0:1616513078.920097:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.244@tcp1 of length 224/224 into md 0xf97c7db9 [1] + 0 00000400:00000200:2.0:1616513078.920102:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83210 00000400:00000200:2.0:1616513078.920104:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513078.920107:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513078.920111:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83210 00000800:00000200:2.0:1616513078.920114:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.920117:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513078.920117:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513078.920121:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513078.920176:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693961185901440 00010000:00000200:12.0:1616513078.920187:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1693961185901440/t0(0) o400->a68ec468-8ee9-e472-aed9-611c84d44f86@10.31.130.244@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.920201:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693961185901440, offset 192 00000400:00000200:12.0:1616513078.920207:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.244@tcp1 00000400:00000200:12.0:1616513078.920214:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.244@tcp1 routed destination 00000400:00000200:12.0:1616513078.920229:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.244@tcp1(10.31.130.244@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513078.920232:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513078.920240:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513078.920246:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.920249:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.920251:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513078.920256:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:12.0:1616513078.920263:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513078.920286:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513078.920295:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513078.920304:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.244@tcp1 : PUT - for me 00000400:00000200:26.0:1616513078.920312:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.244@tcp1 of length 224 into portal 12 MB=0x604a612bfbfc0 00000400:00000200:26.0:1616513078.920319:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.244@tcp1 of length 224/224 into md 0xf96450b9 [1] + 82784 00000400:00000200:26.0:1616513078.920324:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513078.920327:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513078.920333:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513078.920336:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513078.920338:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.920341:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.920349:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.920352:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513078.920353:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513078.920359:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.920364:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241ba0 00000800:00000200:18.0:1616513078.920368:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.920405:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693961185902528 00010000:00000200:14.0:1616513078.920415:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647df080 x1693961185902528/t0(0) o400->c3f51f2d-3ac1-fd01-b4d2-afa45741bf7f@10.31.130.244@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.920428:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693961185902528, offset 192 00000400:00000200:14.0:1616513078.920446:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.244@tcp1 00000400:00000200:14.0:1616513078.920452:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.244@tcp1 routed destination 00000400:00000200:14.0:1616513078.920465:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.244@tcp1(10.31.130.244@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.920470:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.920475:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.920477:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.920479:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.920483:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.920491:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513078.920546:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513078.920552:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513078.920556:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513078.920562:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513078.920566:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756440 00000800:00000200:24.0:1616513078.920570:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513078.920572:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:31.0:1616513078.922518:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925408, portal 4 00000100:00000200:31.0:1616513078.922521:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925408, offset 0 00000400:00000200:31.0:1616513078.922527:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:31.0:1616513078.922532:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000100:00000200:23.0:1616513078.922540:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925392, portal 4 00000400:00000200:31.0:1616513078.922546:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000100:00000200:23.0:1616513078.922546:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925392, offset 0 00000800:00000200:31.0:1616513078.922552:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:23.0:1616513078.922553:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:31.0:1616513078.922558:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:23.0:1616513078.922559:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:31.0:1616513078.922561:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:31.0:1616513078.922563:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:31.0:1616513078.922567:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000400:00000200:23.0:1616513078.922572:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513078.922579:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:23.0:1616513078.922584:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513078.922587:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:23.0:1616513078.922589:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:23.0:1616513078.922594:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:25.2:1616513078.922613:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:1.2:1616513078.922644:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.922662:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513078.922667:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:2.0:1616513078.922670:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000400:00000200:1.0:1616513078.922670:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513078.922677:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:1.0:1616513078.922677:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:2.0:1616513078.922681:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:1.0:1616513078.922684:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63ad00 x1693067725925392/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513078.922687:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:1.0:1616513078.922696:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f3a8 00000800:00000200:1.0:1616513078.922700:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:2.0:1616513078.922702:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bff080 x1693067725925408/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513078.922703:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000400:00000200:2.0:1616513078.922714:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c34b8 00000800:00000200:2.0:1616513078.922719:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513078.922722:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513078.922728:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:2.0:1616513078.922735:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513078.922742:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626c20 00000400:00000200:2.0:1616513078.922749:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c8429 [1] + 192 00000400:00000200:2.0:1616513078.922754:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513078.922758:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bff080 x1693067725925408/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513078.922772:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513078.922772:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513078.922775:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:2.0:1616513078.922776:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:31.0:1616513078.922780:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bff080 x1693067725925408/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513078.922792:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bff080 x1693067725925408/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513078.922801:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000800:00000200:1.2:1616513078.922811:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513078.922820:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.922827:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513078.922830:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513078.922835:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513078.922843:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626c10 00000400:00000200:1.0:1616513078.922849:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffb55 [1] + 192 00000400:00000200:1.0:1616513078.922854:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513078.922859:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63ad00 x1693067725925392/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513078.922872:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513078.922877:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513078.922879:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:23.0:1616513078.922881:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63ad00 x1693067725925392/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513078.922894:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63ad00 x1693067725925392/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513078.922904:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eee0 00000800:00000200:7.2:1616513078.951520:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513078.951577:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.951586:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.951595:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.10@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.951603:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.10@tcp1 of length 224 into portal 26 MB=0x5fffca9a61380 00000400:00000200:18.0:1616513078.951610:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.10@tcp1 of length 224/224 into md 0xf97c7dd9 [1] + 0 00000400:00000200:18.0:1616513078.951614:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82110 00000400:00000200:18.0:1616513078.951617:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.951620:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513078.951624:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82110 00000800:00000200:18.0:1616513078.951627:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.951630:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513078.951632:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.951640:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513078.951643:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513078.951643:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.951652:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.10@tcp1 : PUT - for me 00000400:00000200:26.0:1616513078.951661:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.10@tcp1 of length 224 into portal 12 MB=0x5fffca9a615c0 00000400:00000200:26.0:1616513078.951668:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.10@tcp1 of length 224/224 into md 0xf96450b9 [1] + 83008 00000400:00000200:26.0:1616513078.951673:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513078.951676:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513078.951683:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.951686:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.951688:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513078.951691:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688835526628224 00010000:00000200:12.0:1616513078.951703:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688835526628224/t0(0) o400->7a2b6c6a-3766-090d-7b79-d477a0de8b38@10.31.142.10@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.951718:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688835526628224, offset 192 00000400:00000200:12.0:1616513078.951724:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.10@tcp1 00000400:00000200:12.0:1616513078.951731:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.10@tcp1 routed destination 00000100:00000200:14.0:1616513078.951745:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688835526628800 00000400:00000200:12.0:1616513078.951747:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.10@tcp1(10.31.142.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.951754:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:14.0:1616513078.951757:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647df980 x1688835526628800/t0(0) o400->6dd82631-b0fb-48e9-0af0-8a8ad8fe06df@10.31.142.10@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513078.951761:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.951763:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513078.951766:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:14.0:1616513078.951770:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688835526628800, offset 192 00000800:00000200:12.0:1616513078.951771:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.951775:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:14.0:1616513078.951776:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.10@tcp1 00000100:00000200:12.0:1616513078.951778:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513078.951782:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.10@tcp1 routed destination 00000400:00000200:14.0:1616513078.951795:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.10@tcp1(10.31.142.10@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513078.951801:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513078.951806:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513078.951809:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.951811:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513078.951815:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.951823:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513078.951831:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513078.951838:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513078.951841:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513078.951842:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513078.951848:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513078.951853:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:2.0:1616513078.951859:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513078.951883:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513078.951890:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513078.951892:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513078.951898:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513078.951902:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756dd0 00000800:00000200:0.0:1616513078.951906:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513078.951909:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.958201:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513078.958257:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513078.958267:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513078.958284:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.86@tcp1 : PUT - for me 00000400:00000200:22.0:1616513078.958294:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.86@tcp1 of length 224 into portal 26 MB=0x5fff25be079c0 00000400:00000200:22.0:1616513078.958300:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.86@tcp1 of length 224/224 into md 0xf97c7df9 [1] + 0 00000400:00000200:22.0:1616513078.958305:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82aa0 00000400:00000200:22.0:1616513078.958308:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513078.958311:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513078.958315:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.958315:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82aa0 00000800:00000200:22.0:1616513078.958318:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513078.958321:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513078.958322:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513078.958379:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688791272159680 00010000:00000200:12.0:1616513078.958389:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688791272159680/t0(0) o400->b1aa1ec9-7aae-ad38-527c-1d0e8fe60fe9@10.31.138.86@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.958403:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688791272159680, offset 192 00000400:00000200:12.0:1616513078.958409:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.86@tcp1 00000400:00000200:12.0:1616513078.958417:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.86@tcp1 routed destination 00000400:00000200:12.0:1616513078.958432:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.86@tcp1(10.31.138.86@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513078.958454:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513078.958460:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.958462:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513078.958466:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.958471:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.958476:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513078.958478:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513078.958530:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513078.958536:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513078.958539:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513078.958546:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513078.958551:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782416d8 00000800:00000200:18.0:1616513078.958555:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.958560:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513078.958613:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513078.958625:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.958632:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513078.958635:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513078.958642:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.86@tcp1 : PUT - for me 00000400:00000200:26.0:1616513078.958651:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.86@tcp1 of length 224 into portal 12 MB=0x5fff25be07b80 00000400:00000200:26.0:1616513078.958657:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.86@tcp1 of length 224/224 into md 0xf96450b9 [1] + 83232 00000400:00000200:26.0:1616513078.958662:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513078.958665:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513078.958671:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513078.958674:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513078.958675:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513078.958735:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688791272160128 00010000:00000200:14.0:1616513078.958747:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647de780 x1688791272160128/t0(0) o400->8cf05caa-daa8-dced-28cd-cdb804857ced@10.31.138.86@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513078.958762:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688791272160128, offset 192 00000400:00000200:14.0:1616513078.958769:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.86@tcp1 00000400:00000200:14.0:1616513078.958776:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.86@tcp1 routed destination 00000400:00000200:14.0:1616513078.958791:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.86@tcp1(10.31.138.86@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513078.958793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513078.958797:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:11.2:1616513078.958802:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.958803:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513078.958804:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513078.958808:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513078.958810:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513078.958810:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513078.958814:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.958815:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513078.958818:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.128@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.958826:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.128@tcp1 of length 224 into portal 26 MB=0x602bf46a16a00 00000400:00000200:18.0:1616513078.958833:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.128@tcp1 of length 224/224 into md 0xf97c7e09 [1] + 0 00000400:00000200:18.0:1616513078.958837:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82990 00000400:00000200:18.0:1616513078.958840:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.958843:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513078.958847:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82990 00000800:00000200:18.0:1616513078.958849:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.958852:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.958853:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513078.958857:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:12.0:1616513078.958860:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691870407256576 00010000:00000200:12.0:1616513078.958865:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1691870407256576/t0(0) o400->e9676053-df15-26d8-e6cf-39c7466270f1@10.31.129.128@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513078.958867:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513078.958876:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.128@tcp1 : PUT - for me 00000100:00000200:12.0:1616513078.958877:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691870407256576, offset 192 00000400:00000200:12.0:1616513078.958882:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.128@tcp1 00000400:00000200:24.0:1616513078.958885:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.128@tcp1 of length 224 into portal 12 MB=0x602bf46a16dc0 00000400:00000200:12.0:1616513078.958887:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.128@tcp1 routed destination 00000400:00000200:24.0:1616513078.958891:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.128@tcp1 of length 224/224 into md 0x865bbec1 [1] + 15232 00000400:00000200:24.0:1616513078.958896:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513078.958899:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513078.958901:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.128@tcp1(10.31.129.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513078.958907:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513078.958907:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513078.958910:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513078.958911:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:12.0:1616513078.958911:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513078.958914:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513078.958915:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.958918:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513078.958919:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513078.958920:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513078.958922:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513078.958923:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:12.0:1616513078.958923:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513078.958923:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513078.958930:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513078.958932:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513078.958934:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757ed0 00000800:00000200:22.0:1616513078.958935:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513078.958938:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513078.958938:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513078.958940:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513078.958943:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513078.958948:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241430 00000800:00000200:22.0:1616513078.958951:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513078.958974:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691870407257536 00010000:00000200:0.0:1616513078.958986:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc5e80 x1691870407257536/t0(0) o400->0f002b8b-6930-fc43-fd18-8b9ad1ccd464@10.31.129.128@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513078.959001:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691870407257536, offset 192 00000400:00000200:0.0:1616513078.959007:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.128@tcp1 00000400:00000200:0.0:1616513078.959014:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.128@tcp1 routed destination 00000400:00000200:0.0:1616513078.959029:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.128@tcp1(10.31.129.128@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513078.959036:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513078.959042:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513078.959044:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513078.959047:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513078.959051:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.959056:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513078.959110:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513078.959119:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513078.959122:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513078.959123:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513078.959129:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513078.959133:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000800:00000200:20.0:1616513078.959137:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513078.964537:0:49332:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513078.964592:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513078.964608:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513078.964617:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.24@tcp1 : PUT - for me 00000400:00000200:2.0:1616513078.964627:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.24@tcp1 of length 224 into portal 26 MB=0x5ffd5a5f14780 00000400:00000200:2.0:1616513078.964633:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.24@tcp1 of length 224/224 into md 0xf97c7e19 [1] + 0 00000400:00000200:2.0:1616513078.964638:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82cc0 00000400:00000200:2.0:1616513078.964641:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513078.964643:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513078.964648:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82cc0 00000800:00000200:2.0:1616513078.964651:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513078.964653:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513078.964655:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513078.964662:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513078.964664:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.964665:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513078.964674:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.24@tcp1 : PUT - for me 00000400:00000200:18.0:1616513078.964683:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.24@tcp1 of length 224 into portal 12 MB=0x5ffd5a5f14b40 00000400:00000200:18.0:1616513078.964690:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.24@tcp1 of length 224/224 into md 0xf96450b9 [1] + 83456 00000400:00000200:18.0:1616513078.964695:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513078.964697:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513078.964704:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513078.964707:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513078.964709:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513078.964711:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667960723328 00010000:00000200:12.0:1616513078.964722:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688667960723328/t0(0) o400->cd13b13c-fc3e-3ae0-a4a1-4150a6a53029@10.31.136.24@tcp1:35/0 lens 224/224 e 0 to 0 dl 1616513170 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513078.964736:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667960723328, offset 192 00000400:00000200:12.0:1616513078.964742:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.24@tcp1 00000400:00000200:12.0:1616513078.964750:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.24@tcp1 routed destination 00000400:00000200:12.0:1616513078.964765:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.24@tcp1(10.31.136.24@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:14.0:1616513078.964767:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667960724288 00000800:00000200:12.0:1616513078.964772:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513078.964777:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:14.0:1616513078.964779:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebd100 x1688667960724288/t0(0) o400->c61c1a2d-7e88-12f1-d4a5-5ff76d30c89f@10.31.136.24@tcp1:704/0 lens 224/224 e 0 to 0 dl 1616513084 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513078.964780:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513078.964782:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513078.964787:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513078.964792:0:49332:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:14.0:1616513078.964793:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667960724288, offset 192 00000100:00000200:12.0:1616513078.964794:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513078.964801:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.24@tcp1 00000400:00000200:14.0:1616513078.964808:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.24@tcp1 routed destination 00000800:00000200:22.0:1616513078.964819:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513078.964823:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.24@tcp1(10.31.136.24@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513078.964827:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513078.964830:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513078.964830:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513078.964831:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513078.964835:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513078.964836:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:14.0:1616513078.964838:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513078.964840:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513078.964842:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241b18 00000800:00000200:22.0:1616513078.964845:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:14.0:1616513078.964845:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513078.964851:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513078.964905:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513078.964912:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513078.964915:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513078.964922:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513078.964926:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47575c8 00000800:00000200:28.0:1616513078.964930:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513078.964933:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513079.034689:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925424, portal 4 00000100:00000200:30.0:1616513079.034695:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925424, offset 0 00000400:00000200:30.0:1616513079.034701:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:30.0:1616513079.034708:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:30.0:1616513079.034721:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513079.034727:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:30.0:1616513079.034734:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513079.034737:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:30.0:1616513079.034739:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:30.0:1616513079.034746:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513079.034794:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.034813:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513079.034819:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513079.034822:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513079.034829:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513079.034835:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74647d9b00 x1693067725925424/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513079.034847:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efd38 00000800:00000200:27.0:1616513079.034851:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513079.034854:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513079.034966:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513079.034975:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.034981:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513079.034989:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513079.034997:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626c30 00000400:00000200:27.0:1616513079.035004:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166a9d [1] + 192 00000400:00000200:27.0:1616513079.035009:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513079.035013:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74647d9b00 x1693067725925424/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513079.035028:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513079.035028:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513079.035031:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513079.035032:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:30.0:1616513079.035035:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74647d9b00 x1693067725925424/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513079.035047:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74647d9b00 x1693067725925424/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513079.035057:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:11.2:1616513079.040648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513079.040673:0:49340:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513079.040705:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.040715:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.040725:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.10@tcp1 : PUT - for me 00000800:00000200:18.0:1616513079.040732:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513079.040735:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.10@tcp1 of length 224 into portal 26 MB=0x6000d700bb6c0 00000800:00000200:18.0:1616513079.040742:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513079.040743:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.10@tcp1 of length 224/224 into md 0xf97c7e29 [1] + 0 00000400:00000200:24.0:1616513079.040748:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82bb0 00000400:00000200:24.0:1616513079.040751:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.040752:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.10@tcp1 : PUT - for me 00000100:00000200:24.0:1616513079.040754:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513079.040758:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.040759:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82bb0 00000400:00000200:18.0:1616513079.040761:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.10@tcp1 of length 224 into portal 12 MB=0x6000d700bb740 00000800:00000200:24.0:1616513079.040763:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.040766:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.040768:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513079.040768:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.10@tcp1 of length 224/224 into md 0xf96450b9 [1] + 83680 00000400:00000200:18.0:1616513079.040773:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.040776:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.040784:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.040787:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.040789:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.040794:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513079.040849:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688907574654656 00010000:00000200:12.0:1616513079.040861:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688907574654656/t0(0) o400->4a101940-f76d-c56b-ce2e-06a879a8b362@10.31.141.10@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:14.0:1616513079.040865:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688907574654784 00000100:00000200:12.0:1616513079.040878:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688907574654656, offset 192 00010000:00000200:14.0:1616513079.040879:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebe780 x1688907574654784/t0(0) o400->ddd8fae7-3e0b-1ad0-b646-5e18987dddc5@10.31.141.10@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513079.040885:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.10@tcp1 00000400:00000200:12.0:1616513079.040892:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.10@tcp1 routed destination 00000100:00000200:14.0:1616513079.040893:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688907574654784, offset 192 00000400:00000200:14.0:1616513079.040899:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.10@tcp1 00000400:00000200:12.0:1616513079.040908:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.10@tcp1(10.31.141.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.040915:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513079.040916:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.10@tcp1 routed destination 00000800:00000200:12.0:1616513079.040921:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.040924:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.040926:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:14.0:1616513079.040931:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.10@tcp1(10.31.141.10@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.040932:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513079.040937:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513079.040938:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.040940:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513079.040942:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.040945:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.040947:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.040951:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.040957:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.040968:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.040975:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.040977:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.040984:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.040988:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757100 00000800:00000200:28.0:1616513079.040993:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513079.040994:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513079.040996:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513079.041001:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.041004:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.041006:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513079.041010:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.041016:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240000 00000800:00000200:2.0:1616513079.041019:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.063722:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.063778:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.063788:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.063797:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.218@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.063806:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.218@tcp1 of length 224 into portal 26 MB=0x6001c33214500 00000400:00000200:20.0:1616513079.063813:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.218@tcp1 of length 224/224 into md 0xf97c7e39 [1] + 0 00000400:00000200:20.0:1616513079.063818:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83078 00000400:00000200:20.0:1616513079.063821:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.063824:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513079.063828:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83078 00000800:00000200:20.0:1616513079.063831:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.063834:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.063836:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513079.063840:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.063894:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688970977166592 00010000:00000200:12.0:1616513079.063907:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688970977166592/t0(0) o400->7599b145-0e6d-64c3-cd6b-1087fe55b363@10.31.128.218@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.063922:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688970977166592, offset 192 00000800:00000200:7.2:1616513079.063927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513079.063928:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.218@tcp1 00000400:00000200:12.0:1616513079.063936:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.218@tcp1 routed destination 00000400:00000200:12.0:1616513079.063951:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.218@tcp1(10.31.128.218@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.063958:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.063964:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.063968:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.063970:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513079.063975:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:12.0:1616513079.063982:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513079.063983:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.063992:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513079.064001:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.218@tcp1 : PUT - for me 00000400:00000200:26.0:1616513079.064010:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.218@tcp1 of length 224 into portal 12 MB=0x6001c33214940 00000400:00000200:26.0:1616513079.064017:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.218@tcp1 of length 224/224 into md 0xf96450b9 [1] + 83904 00000400:00000200:26.0:1616513079.064023:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.064025:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513079.064033:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513079.064036:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513079.064037:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513079.064045:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513079.064046:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513079.064048:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.064049:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.064055:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.064061:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241188 00000800:00000200:18.0:1616513079.064065:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.064099:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688970977167680 00010000:00000200:14.0:1616513079.064112:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dc800 x1688970977167680/t0(0) o400->1cbed702-c742-0b90-8934-b2fad431a9dc@10.31.128.218@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.064144:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688970977167680, offset 192 00000400:00000200:14.0:1616513079.064150:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.218@tcp1 00000400:00000200:14.0:1616513079.064157:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.218@tcp1 routed destination 00000400:00000200:14.0:1616513079.064172:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.218@tcp1(10.31.128.218@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.064179:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.064184:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.064187:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.064189:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.064194:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.064200:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.064252:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.064258:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.064263:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.064269:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.064273:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47577e8 00000800:00000200:28.0:1616513079.064277:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.064279:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.064418:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.064430:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.064443:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.064450:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.064459:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.183@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.064472:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.183@tcp1 of length 224 into portal 26 MB=0x5ffd4486e84c0 00000400:00000200:24.0:1616513079.064479:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.183@tcp1 of length 224/224 into md 0xf97c7e49 [1] + 0 00000400:00000200:24.0:1616513079.064484:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83760 00000400:00000200:24.0:1616513079.064487:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.064489:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.064494:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83760 00000800:00000200:24.0:1616513079.064497:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.064500:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.064502:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.064506:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662096905408 00010000:00000200:12.0:1616513079.064512:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688662096905408/t0(0) o400->9ab0067c-4828-2a17-fb59-5a063412d313@10.31.136.183@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.064523:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662096905408, offset 192 00000400:00000200:12.0:1616513079.064528:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.183@tcp1 00000400:00000200:12.0:1616513079.064534:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.183@tcp1 routed destination 00000400:00000200:12.0:1616513079.064549:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.183@tcp1(10.31.136.183@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.064555:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.064560:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.064562:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.064564:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.064567:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513079.064572:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.064572:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.064629:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.064636:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.064638:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513079.064640:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513079.064645:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.064651:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ff0 00000800:00000200:2.0:1616513079.064654:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.064832:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.064842:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.064847:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513079.064852:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.064855:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.183@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.064863:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.183@tcp1 of length 224 into portal 12 MB=0x5ffd4486e85c0 00000400:00000200:28.0:1616513079.064869:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.183@tcp1 of length 224/224 into md 0x865bbec1 [1] + 15456 00000400:00000200:28.0:1616513079.064874:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.064876:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513079.064883:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.064885:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.064887:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.064949:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662096905664 00010000:00000200:0.0:1616513079.064961:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6722d00 x1688662096905664/t0(0) o400->f496748e-c024-2c1f-00a4-d2261f12d4a4@10.31.136.183@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.064976:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662096905664, offset 192 00000400:00000200:0.0:1616513079.064982:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.183@tcp1 00000400:00000200:0.0:1616513079.064992:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.183@tcp1 routed destination 00000400:00000200:0.0:1616513079.065006:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.183@tcp1(10.31.136.183@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.065013:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.065019:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.065021:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.065024:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.065029:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.065034:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.065043:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.065047:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513079.065049:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.065051:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513079.065055:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.065058:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:20.0:1616513079.065061:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.077971:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513079.078027:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.078036:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513079.078045:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.102@tcp1 : PUT - for me 00000400:00000200:4.0:1616513079.078055:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.102@tcp1 of length 224 into portal 26 MB=0x5ffdc5dcef040 00000400:00000200:4.0:1616513079.078061:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.102@tcp1 of length 224/224 into md 0xf97c7e59 [1] + 0 00000400:00000200:4.0:1616513079.078066:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82a18 00000400:00000200:4.0:1616513079.078069:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513079.078072:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513079.078076:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82a18 00000800:00000200:28.0:1616513079.078078:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.078079:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.078082:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513079.078085:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.078088:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513079.078088:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.078096:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.102@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.078104:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.102@tcp1 of length 224 into portal 12 MB=0x5ffdc5dcef0c0 00000400:00000200:28.0:1616513079.078110:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.102@tcp1 of length 224/224 into md 0x865bbec1 [1] + 15680 00000400:00000200:28.0:1616513079.078115:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.078117:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513079.078123:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.078126:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.078128:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.078139:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696815284288 00010000:00000200:12.0:1616513079.078149:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688696815284288/t0(0) o400->ecda86c5-2483-923b-11cb-7263de902105@10.31.141.102@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.078164:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696815284288, offset 192 00000400:00000200:12.0:1616513079.078170:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.102@tcp1 00000400:00000200:12.0:1616513079.078177:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.102@tcp1 routed destination 00000400:00000200:12.0:1616513079.078192:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.102@tcp1(10.31.141.102@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.078199:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:0.0:1616513079.078201:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696815284416 00000800:00000200:12.0:1616513079.078205:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.078207:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.078209:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.078214:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513079.078214:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6724c80 x1688696815284416/t0(0) o400->7056dda9-f4f2-b268-9dff-d9dee8af0181@10.31.141.102@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513079.078221:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.078223:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513079.078228:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696815284416, offset 192 00000400:00000200:0.0:1616513079.078235:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.102@tcp1 00000400:00000200:0.0:1616513079.078242:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.102@tcp1 routed destination 00000400:00000200:0.0:1616513079.078257:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.102@tcp1(10.31.141.102@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.078264:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.078269:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.078272:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.078274:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513079.078277:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513079.078279:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513079.078284:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513079.078284:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513079.078287:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.078293:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.078298:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782405d8 00000800:00000200:18.0:1616513079.078301:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.078304:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513079.078351:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.078358:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513079.078361:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513079.078362:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513079.078367:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.078371:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05078 00000800:00000200:20.0:1616513079.078375:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513079.082576:0:6482:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925440, portal 4 00000100:00000200:16.0:1616513079.082582:0:6482:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925440, offset 0 00000400:00000200:16.0:1616513079.082589:0:6482:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:16.0:1616513079.082595:0:6482:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:16.0:1616513079.082608:0:6482:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513079.082615:0:6482:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:16.0:1616513079.082621:0:6482:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513079.082624:0:6482:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:16.0:1616513079.082626:0:6482:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:16.0:1616513079.082632:0:6482:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513079.082677:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.082695:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513079.082701:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513079.082704:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513079.082710:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513079.082717:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb238d80 x1693067725925440/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513079.082729:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a08 00000800:00000200:27.0:1616513079.082734:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513079.082736:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513079.082844:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513079.082853:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.082859:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513079.082867:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513079.082874:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626c40 00000400:00000200:27.0:1616513079.082881:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166aad [1] + 192 00000400:00000200:27.0:1616513079.082885:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513079.082890:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb238d80 x1693067725925440/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513079.082904:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513079.082905:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513079.082907:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513079.082909:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:16.0:1616513079.082914:0:6482:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb238d80 x1693067725925440/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513079.082928:0:6482:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb238d80 x1693067725925440/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513079.082938:0:6482:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42dd0 00000800:00000200:11.2:1616513079.095919:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.095974:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.095983:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.095993:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.28@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.096002:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.28@tcp1 of length 224 into portal 26 MB=0x604680dd58340 00000400:00000200:28.0:1616513079.096009:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.28@tcp1 of length 224/224 into md 0xf97c7e79 [1] + 0 00000400:00000200:28.0:1616513079.096014:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82b28 00000400:00000200:28.0:1616513079.096018:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.096021:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513079.096026:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82b28 00000800:00000200:24.0:1616513079.096028:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.096029:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.096032:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.096033:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.096090:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693694815470400 00010000:00000200:12.0:1616513079.096100:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1693694815470400/t0(0) o400->f216a061-9d49-2726-183a-611d20563f1b@10.31.140.28@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.096114:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693694815470400, offset 192 00000400:00000200:12.0:1616513079.096121:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.28@tcp1 00000400:00000200:12.0:1616513079.096128:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.28@tcp1 routed destination 00000400:00000200:12.0:1616513079.096143:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.28@tcp1(10.31.140.28@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.096150:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.096155:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.096160:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.096162:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.096168:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513079.096174:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.096174:0:49346:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.096229:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.096236:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.096246:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.096253:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.096259:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241298 00000800:00000200:26.0:1616513079.096262:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.096265:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.096271:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513079.096276:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.096278:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.096279:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.096282:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.096489:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.096554:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.096570:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513079.096571:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.096578:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.28@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.096587:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.28@tcp1 of length 224 into portal 12 MB=0x604680dd59180 00000400:00000200:22.0:1616513079.096594:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.28@tcp1 of length 224/224 into md 0xf96450b9 [1] + 84128 00000400:00000200:22.0:1616513079.096599:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.096602:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513079.096609:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.096612:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.096616:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.096636:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693694815474048 00010000:00000200:14.0:1616513079.096648:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec201200 x1693694815474048/t0(0) o400->067fe77c-a142-15d7-82b2-4d68836982c2@10.31.140.28@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.096662:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693694815474048, offset 192 00000400:00000200:14.0:1616513079.096668:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.28@tcp1 00000400:00000200:14.0:1616513079.096675:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.28@tcp1 routed destination 00000400:00000200:14.0:1616513079.096690:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.28@tcp1(10.31.140.28@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.096697:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.096703:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.096705:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.096708:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.096712:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.096718:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.096774:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.096781:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513079.096783:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513079.096790:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513079.096794:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756110 00000800:00000200:4.0:1616513079.096798:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513079.096801:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.123343:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.123408:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.123417:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.123426:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.236@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.123448:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.236@tcp1 of length 224 into portal 26 MB=0x5ffdeb3e22600 00000400:00000200:24.0:1616513079.123455:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.236@tcp1 of length 224/224 into md 0xf97c7e89 [1] + 0 00000800:00000200:28.0:1616513079.123460:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.123460:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83d38 00000400:00000200:24.0:1616513079.123463:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.123466:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.123471:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83d38 00000800:00000200:24.0:1616513079.123474:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.123477:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.123479:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.123534:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706849318400 00010000:00000200:12.0:1616513079.123545:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688706849318400/t0(0) o400->20521096-a296-5463-1053-31dbbe9a64a9@10.31.128.236@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.123559:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706849318400, offset 192 00000400:00000200:12.0:1616513079.123565:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.236@tcp1 00000800:00000200:11.2:1616513079.123568:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:12.0:1616513079.123573:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.236@tcp1 routed destination 00000800:00000200:4.0:1616513079.123579:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513079.123588:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.236@tcp1(10.31.128.236@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513079.123588:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.123592:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.123595:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:4.0:1616513079.123597:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.236@tcp1 : PUT - for me 00000800:00000200:12.0:1616513079.123601:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.123604:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:4.0:1616513079.123604:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.236@tcp1 of length 224 into portal 12 MB=0x5ffdeb3e22a40 00000800:00000200:12.0:1616513079.123606:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513079.123610:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.236@tcp1 of length 224/224 into md 0x865bbec1 [1] + 15904 00000800:00000200:12.0:1616513079.123611:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513079.123615:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513079.123617:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:4.0:1616513079.123618:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513079.123619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513079.123624:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.123627:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513079.123630:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.123689:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706849319488 00000800:00000200:2.0:1616513079.123693:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.123700:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:0.0:1616513079.123701:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23ad00 x1688706849319488/t0(0) o400->7cbf2033-1b8f-dd99-229a-17410ddefb8b@10.31.128.236@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513079.123703:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.123710:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.123715:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a08 00000800:00000200:2.0:1616513079.123719:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513079.123719:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706849319488, offset 192 00000800:00000200:2.0:1616513079.123724:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513079.123724:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.236@tcp1 00000400:00000200:0.0:1616513079.123731:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.236@tcp1 routed destination 00000400:00000200:0.0:1616513079.123745:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.236@tcp1(10.31.128.236@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.123751:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.123756:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.123759:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.123762:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.123767:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.123772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.123782:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.123787:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.123790:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.123792:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.123796:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.123800:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05f58 00000800:00000200:28.0:1616513079.123803:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.143059:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.143113:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.143123:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.143134:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.115@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.143144:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.115@tcp1 of length 224 into portal 26 MB=0x601837e6ef300 00000400:00000200:20.0:1616513079.143151:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.115@tcp1 of length 224/224 into md 0xf97c7ea9 [1] + 0 00000400:00000200:20.0:1616513079.143156:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83c28 00000400:00000200:20.0:1616513079.143160:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.143162:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:4.0:1616513079.143166:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513079.143167:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83c28 00000800:00000200:20.0:1616513079.143170:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.143173:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.143175:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.143231:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514133807872 00010000:00000200:12.0:1616513079.143242:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1690514133807872/t0(0) o400->5f2e02cc-5240-1398-3d19-0e9c1fa6c454@10.31.136.115@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.143256:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514133807872, offset 192 00000400:00000200:12.0:1616513079.143262:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.115@tcp1 00000400:00000200:12.0:1616513079.143270:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.115@tcp1 routed destination 00000400:00000200:12.0:1616513079.143285:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.115@tcp1(10.31.136.115@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.143292:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.143298:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.143301:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.143303:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.143308:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513079.143315:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.143317:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.143373:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.143380:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.143383:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.143390:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.143395:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240088 00000800:00000200:26.0:1616513079.143399:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.143402:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.143447:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.143502:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.143511:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.143520:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.115@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.143529:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.115@tcp1 of length 224 into portal 12 MB=0x601837e6ef400 00000400:00000200:24.0:1616513079.143535:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.115@tcp1 of length 224/224 into md 0x865bbec1 [1] + 16128 00000400:00000200:24.0:1616513079.143540:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.143543:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513079.143549:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.143554:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.143556:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.143560:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.143614:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514133808128 00010000:00000200:0.0:1616513079.143626:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc6780 x1690514133808128/t0(0) o400->1863bad0-942d-78ea-c6b8-90e71d6d31c0@10.31.136.115@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.143640:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514133808128, offset 192 00000400:00000200:0.0:1616513079.143647:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.115@tcp1 00000400:00000200:0.0:1616513079.143654:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.115@tcp1 routed destination 00000400:00000200:0.0:1616513079.143669:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.115@tcp1(10.31.136.115@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.143676:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.143681:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.143684:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.143686:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.143691:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.143696:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.143707:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.143713:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513079.143715:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513079.143717:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513079.143722:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513079.143726:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05c28 00000800:00000200:4.0:1616513079.143730:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.156605:0:49352:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.156661:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.156671:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.156680:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.179@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.156690:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.179@tcp1 of length 224 into portal 26 MB=0x5ffd4480cf980 00000400:00000200:2.0:1616513079.156697:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.179@tcp1 of length 224/224 into md 0xf97c7ec9 [1] + 0 00000400:00000200:2.0:1616513079.156701:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83870 00000400:00000200:2.0:1616513079.156704:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.156707:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.156711:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83870 00000800:00000200:2.0:1616513079.156714:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.156717:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.156718:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.156722:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513079.156775:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662090512768 00010000:00000200:12.0:1616513079.156786:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688662090512768/t0(0) o400->511e59b5-1377-b045-043f-33ed18b058b1@10.31.136.179@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.156800:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662090512768, offset 192 00000400:00000200:12.0:1616513079.156806:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.179@tcp1 00000400:00000200:12.0:1616513079.156814:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.179@tcp1 routed destination 00000800:00000200:11.2:1616513079.156822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:12.0:1616513079.156829:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.179@tcp1(10.31.136.179@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.156835:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.156842:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.156844:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.156847:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.156851:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.156857:0:49352:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.156858:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513079.156877:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.156886:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.156895:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.179@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.156904:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.179@tcp1 of length 224 into portal 12 MB=0x5ffd4480cfa00 00000400:00000200:28.0:1616513079.156911:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.179@tcp1 of length 224/224 into md 0x865bbec1 [1] + 16352 00000800:00000200:26.0:1616513079.156911:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513079.156916:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.156918:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513079.156918:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.156921:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.156926:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513079.156928:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513079.156929:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.156931:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513079.156933:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241078 00000800:00000200:28.0:1616513079.156934:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513079.156937:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.156940:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513079.157006:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662090512896 00010000:00000200:0.0:1616513079.157018:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b98480 x1688662090512896/t0(0) o400->d651c503-2516-8390-2c6d-39dd4db3f68b@10.31.136.179@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.157033:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662090512896, offset 192 00000400:00000200:0.0:1616513079.157039:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.179@tcp1 00000400:00000200:0.0:1616513079.157047:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.179@tcp1 routed destination 00000400:00000200:0.0:1616513079.157062:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.179@tcp1(10.31.136.179@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.157068:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.157074:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.157077:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.157079:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.157084:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.157089:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.157147:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.157154:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513079.157157:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513079.157163:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.157167:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:20.0:1616513079.157172:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.157174:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.161264:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.161321:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.161330:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.161339:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.13@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.161348:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.13@tcp1 of length 224 into portal 26 MB=0x5ffd417973900 00000400:00000200:24.0:1616513079.161355:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.13@tcp1 of length 224/224 into md 0xf97c7ee9 [1] + 0 00000400:00000200:24.0:1616513079.161359:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83a08 00000400:00000200:24.0:1616513079.161362:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.161365:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.161369:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83a08 00000800:00000200:24.0:1616513079.161372:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.161374:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.161375:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.161379:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.161432:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661277489408 00010000:00000200:12.0:1616513079.161455:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688661277489408/t0(0) o400->ced300b5-5bb2-21fa-4740-108b2b42bf2e@10.31.129.13@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.161469:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661277489408, offset 192 00000400:00000200:12.0:1616513079.161475:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.13@tcp1 00000400:00000200:12.0:1616513079.161483:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.13@tcp1 routed destination 00000400:00000200:12.0:1616513079.161498:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.13@tcp1(10.31.129.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.161504:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.161510:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.161513:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.161515:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.161520:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513079.161526:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.161526:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513079.161575:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513079.161581:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.161588:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.161591:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.161597:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.161603:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241540 00000800:00000200:18.0:1616513079.161606:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.161609:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513079.161628:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.161637:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.161646:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.13@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.161654:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.13@tcp1 of length 224 into portal 12 MB=0x5ffd417973c80 00000400:00000200:20.0:1616513079.161660:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.13@tcp1 of length 224/224 into md 0x865bbec1 [1] + 16576 00000400:00000200:20.0:1616513079.161666:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.161668:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513079.161676:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.161679:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513079.161681:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513079.161683:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.161740:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661277490304 00010000:00000200:0.0:1616513079.161752:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e00900 x1688661277490304/t0(0) o400->93fb4c9e-3ef2-92e3-f679-3327cef03e5a@10.31.129.13@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.161765:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661277490304, offset 192 00000400:00000200:0.0:1616513079.161772:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.13@tcp1 00000400:00000200:0.0:1616513079.161779:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.13@tcp1 routed destination 00000400:00000200:0.0:1616513079.161793:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.13@tcp1(10.31.129.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.161799:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.161804:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.161807:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.161809:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.161813:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.161818:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.161829:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.161834:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.161837:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.161839:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.161844:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.161848:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04088 00000800:00000200:28.0:1616513079.161852:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.164213:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.164266:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.164275:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.164284:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.202@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.164294:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.202@tcp1 of length 224 into portal 26 MB=0x601aa32d4c900 00000400:00000200:20.0:1616513079.164300:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.202@tcp1 of length 224/224 into md 0xf97c7f09 [1] + 0 00000400:00000200:20.0:1616513079.164307:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd833a8 00000400:00000200:20.0:1616513079.164310:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.164312:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513079.164317:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd833a8 00000800:00000200:4.0:1616513079.164318:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.164320:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.164323:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.164324:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.164379:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690680369137920 00010000:00000200:12.0:1616513079.164389:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1690680369137920/t0(0) o400->4a2d4021-bef8-850c-a9c3-078c75a4e1b1@10.31.140.202@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.164404:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690680369137920, offset 192 00000400:00000200:12.0:1616513079.164412:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.202@tcp1 00000400:00000200:12.0:1616513079.164419:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.202@tcp1 routed destination 00000400:00000200:12.0:1616513079.164448:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.202@tcp1(10.31.140.202@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.164454:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.164460:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.164463:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.164465:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.164469:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513079.164476:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.164477:0:49353:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.164532:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.164538:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.164541:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.164548:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.164553:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241760 00000800:00000200:26.0:1616513079.164556:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.164559:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.164607:0:49353:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513079.164661:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.164670:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513079.164679:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.202@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.164689:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.202@tcp1 of length 224 into portal 12 MB=0x601aa32d4cb40 00000400:00000200:18.0:1616513079.164695:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.202@tcp1 of length 224/224 into md 0xf96450b9 [1] + 84352 00000400:00000200:18.0:1616513079.164700:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.164703:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.164710:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.164713:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.164715:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.164719:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.164791:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690680369138496 00010000:00000200:14.0:1616513079.164803:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebc380 x1690680369138496/t0(0) o400->cbe924ad-76bc-1dfd-b6b7-fd97d9eef1ff@10.31.140.202@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.164817:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690680369138496, offset 192 00000400:00000200:14.0:1616513079.164824:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.202@tcp1 00000400:00000200:14.0:1616513079.164831:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.202@tcp1 routed destination 00000400:00000200:14.0:1616513079.164847:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.202@tcp1(10.31.140.202@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.164855:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.164861:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.164864:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.164866:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.164870:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.164875:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.164928:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.164934:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.164937:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.164944:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.164948:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756088 00000800:00000200:24.0:1616513079.164952:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.164955:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.167753:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513079.167773:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513079.167789:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.167797:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513079.167806:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.177@tcp1 : PUT - for me 00000800:00000200:26.0:1616513079.167810:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513079.167816:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.177@tcp1 of length 224 into portal 12 MB=0x5ffdb36b3d780 00000800:00000200:26.0:1616513079.167820:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:4.0:1616513079.167822:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.177@tcp1 of length 224/224 into md 0x865bbec1 [1] + 16800 00000400:00000200:4.0:1616513079.167827:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.167828:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.177@tcp1 : PUT - for me 00000100:00000200:4.0:1616513079.167830:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513079.167837:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513079.167838:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.177@tcp1 of length 224 into portal 26 MB=0x5ffdb36b3d540 00000800:00000200:4.0:1616513079.167840:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513079.167842:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513079.167844:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.177@tcp1 of length 224/224 into md 0xf97c7f19 [1] + 0 00000800:00000200:20.0:1616513079.167844:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513079.167849:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83320 00000400:00000200:26.0:1616513079.167852:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.167854:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513079.167860:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83320 00000800:00000200:26.0:1616513079.167864:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.167866:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.167868:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.167872:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513079.167903:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691864229760 00010000:00000200:0.0:1616513079.167915:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23c380 x1688691864229760/t0(0) o400->d3401524-2bcf-1f93-4aa5-86220f53c55a@10.31.140.177@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513079.167927:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691864229184 00000100:00000200:0.0:1616513079.167928:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688691864229760, offset 192 00000400:00000200:0.0:1616513079.167934:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.177@tcp1 00010000:00000200:12.0:1616513079.167938:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688691864229184/t0(0) o400->8b3c5287-5d58-9443-0f54-06698711fffe@10.31.140.177@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513079.167941:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.177@tcp1 routed destination 00000100:00000200:12.0:1616513079.167952:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688691864229184, offset 192 00000400:00000200:0.0:1616513079.167955:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.177@tcp1(10.31.140.177@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513079.167958:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.177@tcp1 00000800:00000200:0.0:1616513079.167961:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513079.167965:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.177@tcp1 routed destination 00000400:00000200:12.0:1616513079.167982:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.177@tcp1(10.31.140.177@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.167988:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.167989:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.167991:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.167993:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.167994:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.167997:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.167998:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513079.167999:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.168003:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513079.168003:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513079.168009:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.168010:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513079.168057:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.168064:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.168066:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.168073:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513079.168077:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513079.168077:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04b28 00000800:00000200:2.0:1616513079.168081:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513079.168084:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513079.168084:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513079.168087:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.168093:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.168093:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.168099:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241870 00000800:00000200:24.0:1616513079.168102:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:5.0:1616513079.179900:0:6494:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925456, portal 4 00000100:00000200:5.0:1616513079.179906:0:6494:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925456, offset 0 00000400:00000200:5.0:1616513079.179912:0:6494:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:5.0:1616513079.179918:0:6494:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:5.0:1616513079.179932:0:6494:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:5.0:1616513079.179939:0:6494:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:5.0:1616513079.179945:0:6494:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513079.179947:0:6494:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:5.0:1616513079.179949:0:6494:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:5.0:1616513079.179955:0:6494:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513079.180000:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.180019:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513079.180024:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513079.180027:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513079.180033:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513079.180040:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8302320900 x1693067725925456/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513079.180052:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55fa90 00000800:00000200:27.0:1616513079.180055:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513079.180058:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513079.180141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513079.180151:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.180157:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513079.180165:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513079.180173:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626c50 00000400:00000200:27.0:1616513079.180181:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166abd [1] + 192 00000400:00000200:27.0:1616513079.180185:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513079.180190:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8302320900 x1693067725925456/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513079.180203:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513079.180205:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513079.180206:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513079.180209:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:5.0:1616513079.180211:0:6494:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8302320900 x1693067725925456/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:5.0:1616513079.180223:0:6494:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8302320900 x1693067725925456/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513079.180232:0:6494:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55e7f8 00000800:00000200:11.2:1616513079.193597:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513079.193627:0:49356:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513079.193652:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.193662:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513079.193672:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.128@tcp1 : PUT - for me 00000400:00000200:4.0:1616513079.193681:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.128@tcp1 of length 224 into portal 26 MB=0x5ffe114799bc0 00000800:00000200:22.0:1616513079.193685:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513079.193688:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.128@tcp1 of length 224/224 into md 0xf97c7f39 [1] + 0 00000400:00000200:4.0:1616513079.193693:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83cb0 00000800:00000200:22.0:1616513079.193695:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:4.0:1616513079.193696:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513079.193699:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.193704:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.12@tcp1 : PUT - for me 00000400:00000200:4.0:1616513079.193704:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83cb0 00000800:00000200:4.0:1616513079.193707:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.193710:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513079.193712:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513079.193714:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.12@tcp1 of length 224 into portal 26 MB=0x603e15c6744c0 00000800:00000200:4.0:1616513079.193714:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513079.193721:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.12@tcp1 of length 224/224 into md 0xf97c7f59 [1] + 0 00000800:00000200:20.0:1616513079.193721:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513079.193721:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513079.193726:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd826e8 00000400:00000200:4.0:1616513079.193728:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.128@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.193729:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.193732:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513079.193734:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.128@tcp1 of length 224 into portal 12 MB=0x5ffe114799e00 00000400:00000200:22.0:1616513079.193737:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd826e8 00000400:00000200:4.0:1616513079.193739:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.128@tcp1 of length 224/224 into md 0x865bbec1 [1] + 17024 00000800:00000200:22.0:1616513079.193740:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.193743:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:4.0:1616513079.193743:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.193745:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513079.193745:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513079.193747:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513079.193751:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513079.193752:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:4.0:1616513079.193753:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513079.193754:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513079.193759:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.193759:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.12@tcp1 : PUT - for me 00000100:00000200:12.0:1616513079.193763:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717059791808 00000400:00000200:22.0:1616513079.193764:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.12@tcp1 of length 224 into portal 12 MB=0x603e15c674740 00000100:00000200:8.0:1616513079.193768:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116313060544 00000400:00000200:22.0:1616513079.193770:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.12@tcp1 of length 224/224 into md 0xf96450b9 [1] + 84576 00000400:00000200:22.0:1616513079.193774:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.193776:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:12.0:1616513079.193776:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688717059791808/t0(0) o400->a030e8ca-e72e-e029-605b-87e8484affaf@10.31.141.128@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:8.0:1616513079.193779:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1693116313060544/t0(0) o400->a3d28584-1501-5d47-e48b-c4bb9b01dede@10.31.130.12@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513079.193782:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.193784:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.193785:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513079.193791:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688717059791808, offset 192 00000100:00000200:8.0:1616513079.193794:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693116313060544, offset 192 00000400:00000200:12.0:1616513079.193797:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.128@tcp1 00000400:00000200:8.0:1616513079.193802:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.12@tcp1 00000400:00000200:12.0:1616513079.193805:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.128@tcp1 routed destination 00000400:00000200:8.0:1616513079.193815:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.12@tcp1 routed destination 00000400:00000200:12.0:1616513079.193821:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.128@tcp1(10.31.141.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513079.193823:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717059792384 00000400:00000200:8.0:1616513079.193831:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.12@tcp1(10.31.130.12@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.193832:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513079.193837:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:0.0:1616513079.193837:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23a880 x1688717059792384/t0(0) o400->634dd347-c599-8533-b7e8-7a055ec4ffb6@10.31.141.128@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513079.193839:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.193842:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513079.193843:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.193844:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513079.193846:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.193848:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.193849:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513079.193850:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116313061184 00000100:00000200:0.0:1616513079.193851:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688717059792384, offset 192 00000800:00000200:8.0:1616513079.193853:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513079.193855:0:49356:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.193857:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513079.193858:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513079.193858:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.128@tcp1 00000100:00000200:8.0:1616513079.193860:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:14.0:1616513079.193863:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec202d00 x1693116313061184/t0(0) o400->4738a77f-c172-4702-173c-f5bc8366971a@10.31.130.12@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513079.193865:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.128@tcp1 routed destination 00000100:00000200:14.0:1616513079.193877:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693116313061184, offset 192 00000400:00000200:0.0:1616513079.193881:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.128@tcp1(10.31.141.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513079.193884:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.12@tcp1 00000800:00000200:0.0:1616513079.193887:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513079.193891:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.12@tcp1 routed destination 00000800:00000200:0.0:1616513079.193893:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.193898:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513079.193900:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:0.0:1616513079.193905:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513079.193906:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.12@tcp1(10.31.130.12@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513079.193909:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513079.193912:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513079.193917:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513079.193918:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513079.193920:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:14.0:1616513079.193921:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513079.193922:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513079.193923:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:26.0:1616513079.193926:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513079.193928:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:26.0:1616513079.193929:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.193929:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513079.193930:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:22.0:1616513079.193931:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.193934:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240bb0 00000400:00000200:26.0:1616513079.193936:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513079.193938:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:18.0:1616513079.193938:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513079.193940:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05430 00000400:00000200:24.0:1616513079.193940:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513079.193941:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:26.0:1616513079.193944:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513079.193947:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513079.193948:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:20.0:1616513079.193950:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.193952:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71320 00000800:00000200:4.0:1616513079.193952:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.193956:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513079.193957:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.193961:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47562a8 00000800:00000200:20.0:1616513079.193965:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.204475:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.204529:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.204548:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.204557:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.185@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.204567:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.185@tcp1 of length 224 into portal 26 MB=0x5ffd6efe08300 00000400:00000200:28.0:1616513079.204573:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.185@tcp1 of length 224/224 into md 0xf97c7f69 [1] + 0 00000400:00000200:28.0:1616513079.204578:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82198 00000400:00000200:28.0:1616513079.204581:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.204583:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513079.204588:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82198 00000800:00000200:28.0:1616513079.204591:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.204593:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513079.204595:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.204599:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.204609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:8.0:1616513079.204658:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673496105728 00010000:00000200:8.0:1616513079.204668:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688673496105728/t0(0) o400->c5e633f9-a871-b641-cc08-6a94155c0557@10.31.128.185@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513079.204681:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513079.204681:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688673496105728, offset 192 00000400:00000200:8.0:1616513079.204687:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.185@tcp1 00000800:00000200:24.0:1616513079.204690:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.204695:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.185@tcp1 routed destination 00000400:00000200:24.0:1616513079.204705:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.185@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.204709:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.185@tcp1(10.31.128.185@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513079.204714:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.185@tcp1 of length 224 into portal 12 MB=0x5ffd6efe08640 00000800:00000200:8.0:1616513079.204715:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513079.204720:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.185@tcp1 of length 224/224 into md 0x865bbec1 [1] + 17248 00000800:00000200:8.0:1616513079.204721:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513079.204724:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513079.204726:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:8.0:1616513079.204727:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513079.204728:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513079.204732:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513079.204735:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.204735:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.204738:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513079.204738:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.204739:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513079.204740:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.204799:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673496106560 00000800:00000200:2.0:1616513079.204804:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.204811:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:0.0:1616513079.204812:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc1680 x1688673496106560/t0(0) o400->9b1e46e2-0fe2-82a6-6015-707a57665e67@10.31.128.185@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513079.204814:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.204821:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.204827:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71b18 00000800:00000200:2.0:1616513079.204830:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513079.204830:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688673496106560, offset 192 00000800:00000200:2.0:1616513079.204833:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513079.204836:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.185@tcp1 00000400:00000200:0.0:1616513079.204844:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.185@tcp1 routed destination 00000400:00000200:0.0:1616513079.204859:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.185@tcp1(10.31.128.185@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.204865:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.204871:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.204874:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.204876:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.204881:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.204885:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.204895:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.204901:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513079.204903:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.204905:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513079.204910:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513079.204914:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:4.0:1616513079.204918:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.205696:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.205707:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.205715:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513079.205717:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513079.205724:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.233@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.205733:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.233@tcp1 of length 224 into portal 12 MB=0x5ffdc3fe9da00 00000400:00000200:20.0:1616513079.205740:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.233@tcp1 of length 224/224 into md 0x865bbec1 [1] + 17472 00000400:00000200:20.0:1616513079.205745:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.205747:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513079.205754:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.205757:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.205759:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.205765:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696313731584 00010000:00000200:0.0:1616513079.205771:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06c00 x1688696313731584/t0(0) o400->2b7b9c8e-3700-4630-3564-01ad30cdddb5@10.31.141.233@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.205782:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696313731584, offset 192 00000400:00000200:0.0:1616513079.205786:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.233@tcp1 00000400:00000200:0.0:1616513079.205792:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.233@tcp1 routed destination 00000400:00000200:0.0:1616513079.205804:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.233@tcp1(10.31.141.233@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.205810:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513079.205814:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.205817:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.205818:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.205821:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.205827:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.205882:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.205888:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.205891:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.205898:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.205902:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb047f8 00000800:00000200:18.0:1616513079.205906:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.205908:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.205915:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513079.205917:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.205917:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.205919:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.205920:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.205955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.205968:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.205978:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513079.205979:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.205986:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.233@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.205995:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.233@tcp1 of length 224 into portal 26 MB=0x5ffdc3fe9d980 00000400:00000200:22.0:1616513079.206002:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.233@tcp1 of length 224/224 into md 0xf97c7f89 [1] + 0 00000400:00000200:22.0:1616513079.206007:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd835c8 00000400:00000200:22.0:1616513079.206009:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.206014:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.206019:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd835c8 00000800:00000200:22.0:1616513079.206022:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.206025:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.206027:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513079.206035:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696313731456 00010000:00000200:8.0:1616513079.206043:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688696313731456/t0(0) o400->23b1f17d-a0df-dca3-7887-a2b21ecae33a@10.31.141.233@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513079.206055:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696313731456, offset 192 00000400:00000200:8.0:1616513079.206060:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.233@tcp1 00000400:00000200:8.0:1616513079.206066:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.233@tcp1 routed destination 00000400:00000200:8.0:1616513079.206079:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.233@tcp1(10.31.141.233@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513079.206085:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513079.206090:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513079.206092:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.206094:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.206098:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.206103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513079.206103:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513079.206154:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.206161:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.206164:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513079.206165:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.206170:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.206175:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70dd0 00000800:00000200:28.0:1616513079.206179:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.242265:0:49361:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513079.242322:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.242333:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513079.242341:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.163@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.242351:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.163@tcp1 of length 224 into portal 26 MB=0x5ffd6ac165480 00000400:00000200:18.0:1616513079.242358:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.163@tcp1 of length 224/224 into md 0xf97c7f99 [1] + 0 00000400:00000200:18.0:1616513079.242363:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83100 00000400:00000200:18.0:1616513079.242366:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.242369:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513079.242374:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83100 00000800:00000200:26.0:1616513079.242378:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.242378:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.242381:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.242383:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513079.242397:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672358782080 00010000:00000200:8.0:1616513079.242408:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688672358782080/t0(0) o400->44a30dac-78e1-3d1f-5266-8e2470097b3a@10.31.138.163@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513079.242424:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688672358782080, offset 192 00000400:00000200:8.0:1616513079.242430:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.163@tcp1 00000400:00000200:8.0:1616513079.242444:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.163@tcp1 routed destination 00000400:00000200:8.0:1616513079.242459:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.163@tcp1(10.31.138.163@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513079.242466:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513079.242467:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.242473:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513079.242475:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513079.242478:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513079.242483:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.242488:0:49361:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513079.242491:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513079.242527:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.242537:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.242546:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.163@tcp1 : PUT - for me 00000800:00000200:2.0:1616513079.242549:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513079.242555:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.163@tcp1 of length 224 into portal 12 MB=0x5ffd6ac165500 00000800:00000200:2.0:1616513079.242556:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.242559:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.242561:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.163@tcp1 of length 224/224 into md 0x865bbec1 [1] + 17696 00000400:00000200:24.0:1616513079.242566:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.242566:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:24.0:1616513079.242569:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513079.242577:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513079.242578:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71f58 00000800:00000200:24.0:1616513079.242580:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513079.242582:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513079.242582:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.242585:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513079.242587:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.242628:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672358782208 00010000:00000200:0.0:1616513079.242641:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc7500 x1688672358782208/t0(0) o400->78c856fc-5795-192a-0712-4c5ad7c61eec@10.31.138.163@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.242665:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688672358782208, offset 192 00000400:00000200:0.0:1616513079.242670:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.163@tcp1 00000400:00000200:0.0:1616513079.242678:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.163@tcp1 routed destination 00000400:00000200:0.0:1616513079.242692:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.163@tcp1(10.31.138.163@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.242698:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.242704:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.242706:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.242708:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.242714:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.242719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.242782:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.242789:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513079.242792:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513079.242798:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513079.242803:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:4.0:1616513079.242806:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513079.242809:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.253822:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.253886:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.253895:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.253904:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.126@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.253914:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.126@tcp1 of length 224 into portal 26 MB=0x60022b3709900 00000400:00000200:20.0:1616513079.253920:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.126@tcp1 of length 224/224 into md 0xf97c7fa9 [1] + 0 00000400:00000200:20.0:1616513079.253925:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82550 00000400:00000200:20.0:1616513079.253930:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.253932:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513079.253937:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82550 00000800:00000200:24.0:1616513079.253940:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.253940:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.253943:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513079.253945:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513079.253955:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688998899652864 00010000:00000200:8.0:1616513079.253964:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688998899652864/t0(0) o400->8079a9e0-af4c-895f-3444-7d73ad1c5b86@10.31.141.126@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513079.253978:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688998899652864, offset 192 00000400:00000200:8.0:1616513079.253983:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.126@tcp1 00000400:00000200:8.0:1616513079.253992:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.126@tcp1 routed destination 00000400:00000200:8.0:1616513079.254006:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.126@tcp1(10.31.141.126@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513079.254012:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513079.254018:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513079.254021:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513079.254023:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513079.254028:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513079.254034:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.254034:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.254090:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.254097:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.254100:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.254107:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.254112:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71c28 00000800:00000200:26.0:1616513079.254116:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.254119:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.254270:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.254324:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.254334:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.254343:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.126@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.254351:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.126@tcp1 of length 224 into portal 12 MB=0x60022b370a440 00000400:00000200:2.0:1616513079.254358:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.126@tcp1 of length 224/224 into md 0xf96450b9 [1] + 84800 00000400:00000200:2.0:1616513079.254363:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.254366:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513079.254373:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.254376:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.254378:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.254381:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.254462:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688998899655744 00010000:00000200:14.0:1616513079.254475:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464624380 x1688998899655744/t0(0) o400->66fc4b40-5d9d-3ce6-3be3-b7f3c4ab6c1f@10.31.141.126@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.254490:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688998899655744, offset 192 00000400:00000200:14.0:1616513079.254496:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.126@tcp1 00000400:00000200:14.0:1616513079.254504:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.126@tcp1 routed destination 00000400:00000200:14.0:1616513079.254519:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.126@tcp1(10.31.141.126@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.254526:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.254532:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.254534:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.254537:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.254542:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.254547:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.254604:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.254611:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513079.254614:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513079.254620:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513079.254624:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756000 00000800:00000200:4.0:1616513079.254628:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513079.254631:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.261866:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.261922:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.261932:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.261941:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.9@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.261951:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.9@tcp1 of length 224 into portal 26 MB=0x601cab7301000 00000400:00000200:24.0:1616513079.261958:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.9@tcp1 of length 224/224 into md 0xf97c7fc9 [1] + 0 00000400:00000200:24.0:1616513079.261962:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82880 00000400:00000200:24.0:1616513079.261965:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.261968:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.261974:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82880 00000800:00000200:20.0:1616513079.261975:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.261977:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.261981:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513079.261983:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.261986:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.261986:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.261994:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.9@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.262003:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.9@tcp1 of length 224 into portal 12 MB=0x601cab73010c0 00000400:00000200:20.0:1616513079.262010:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.9@tcp1 of length 224/224 into md 0x865bbec1 [1] + 17920 00000400:00000200:20.0:1616513079.262015:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.262017:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513079.262025:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.262028:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.262030:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.262039:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690820028665856 00010000:00000200:12.0:1616513079.262050:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1690820028665856/t0(0) o400->0feaeaba-fb0c-860a-e113-7a5046511fb4@10.31.130.9@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.262064:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690820028665856, offset 192 00000400:00000200:12.0:1616513079.262070:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.9@tcp1 00000400:00000200:12.0:1616513079.262077:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.9@tcp1 routed destination 00000400:00000200:12.0:1616513079.262092:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.9@tcp1(10.31.130.9@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513079.262093:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690820028666048 00000800:00000200:12.0:1616513079.262099:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.262105:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513079.262106:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b105a00 x1690820028666048/t0(0) o400->7dbc17fb-dc82-28e3-4c02-1ce7b8875c84@10.31.130.9@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513079.262108:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.262110:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.262115:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513079.262121:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513079.262121:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690820028666048, offset 192 00000800:00000200:7.2:1616513079.262123:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513079.262127:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.9@tcp1 00000400:00000200:0.0:1616513079.262135:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.9@tcp1 routed destination 00000400:00000200:0.0:1616513079.262150:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.9@tcp1(10.31.130.9@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.262157:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.262162:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.262165:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.262167:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.262172:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.262177:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513079.262180:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.262186:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.262189:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.262196:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.262201:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241d38 00000800:00000200:26.0:1616513079.262205:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.262208:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513079.262231:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.262238:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.262240:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.262242:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.262247:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.262251:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04000 00000800:00000200:28.0:1616513079.262255:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:7.0:1616513079.274544:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925472, portal 4 00000100:00000200:7.0:1616513079.274548:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925472, offset 0 00000400:00000200:7.0:1616513079.274554:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:7.0:1616513079.274558:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:7.0:1616513079.274567:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513079.274572:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:7.0:1616513079.274577:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513079.274579:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:7.0:1616513079.274581:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:10.0:1616513079.274582:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925488, portal 4 00000800:00000200:7.0:1616513079.274584:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:10.0:1616513079.274588:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925488, offset 0 00000400:00000200:10.0:1616513079.274594:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:10.0:1616513079.274599:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:10.0:1616513079.274611:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513079.274617:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:10.0:1616513079.274623:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513079.274626:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:10.0:1616513079.274628:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.2:1616513079.274631:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:10.0:1616513079.274632:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:27.0:1616513079.274651:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513079.274656:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513079.274659:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513079.274662:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513079.274665:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513079.274671:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd604c80 x1693067725925472/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.2:1616513079.274678:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:27.0:1616513079.274683:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f298 00000800:00000200:27.0:1616513079.274687:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:20.0:1616513079.274735:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513079.274742:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513079.274745:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513079.274751:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513079.274757:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dec203a80 x1693067725925488/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513079.274769:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82550 00000800:00000200:27.2:1616513079.274772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:20.0:1616513079.274774:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513079.274777:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.0:1616513079.274782:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.274788:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513079.274792:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513079.274796:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513079.274803:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626c60 00000400:00000200:27.0:1616513079.274810:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166acd [1] + 192 00000400:00000200:27.0:1616513079.274814:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513079.274819:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd604c80 x1693067725925472/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513079.274832:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.274835:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513079.274836:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:7.0:1616513079.274836:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd604c80 x1693067725925472/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513079.274845:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd604c80 x1693067725925472/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513079.274852:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f7e8 00000800:00000200:5.2:1616513079.274881:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:24.0:1616513079.274935:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513079.274944:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:24.0:1616513079.274953:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:24.0:1616513079.274961:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626c70 00000400:00000200:24.0:1616513079.274968:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d51c1 [1] + 192 00000400:00000200:24.0:1616513079.274973:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.274978:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dec203a80 x1693067725925488/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513079.274989:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513079.274992:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513079.274995:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513079.274997:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:10.0:1616513079.275000:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dec203a80 x1693067725925488/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513079.275012:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dec203a80 x1693067725925488/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513079.275024:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82bb0 00000800:00000200:7.2:1616513079.276621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.276676:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.276685:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.276695:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.46@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.276703:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.46@tcp1 of length 224 into portal 26 MB=0x5ffdc0e23bd00 00000400:00000200:2.0:1616513079.276710:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.46@tcp1 of length 224/224 into md 0xf97c7fe9 [1] + 0 00000400:00000200:2.0:1616513079.276714:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83ed0 00000400:00000200:2.0:1616513079.276717:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.276720:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.276724:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83ed0 00000800:00000200:2.0:1616513079.276728:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.276729:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.276731:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.276733:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513079.276789:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695478664448 00010000:00000200:12.0:1616513079.276800:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688695478664448/t0(0) o400->142700c5-a8ab-278a-2ffb-8df2c7d7c5cd@10.31.143.46@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.276814:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695478664448, offset 192 00000400:00000200:12.0:1616513079.276820:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.46@tcp1 00000400:00000200:12.0:1616513079.276828:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.46@tcp1 routed destination 00000400:00000200:12.0:1616513079.276843:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.46@tcp1(10.31.143.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.276849:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.276856:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.276858:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.276862:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.276867:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.276873:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.276874:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513079.276926:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.276933:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.276936:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.276942:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.276948:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241d38 00000800:00000200:26.0:1616513079.276951:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.276954:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.276960:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513079.276962:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513079.276968:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.46@tcp1 : PUT - for me 00000400:00000200:26.0:1616513079.276976:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.46@tcp1 of length 224 into portal 12 MB=0x5ffdc0e23bf40 00000400:00000200:26.0:1616513079.276984:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.46@tcp1 of length 224/224 into md 0xf96450b9 [1] + 85024 00000400:00000200:26.0:1616513079.276989:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.276992:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513079.276998:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.277000:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.277002:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.277068:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695478665024 00010000:00000200:14.0:1616513079.277080:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dbf00 x1688695478665024/t0(0) o400->66acd142-8c68-6c3a-6503-aa58ce7773ce@10.31.143.46@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.277095:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695478665024, offset 192 00000400:00000200:14.0:1616513079.277101:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.46@tcp1 00000400:00000200:14.0:1616513079.277108:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.46@tcp1 routed destination 00000400:00000200:14.0:1616513079.277123:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.46@tcp1(10.31.143.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.277130:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.277136:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.277138:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.277141:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.277145:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.277152:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.277208:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.277215:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513079.277217:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513079.277224:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.277228:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757c28 00000800:00000200:20.0:1616513079.277232:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.277235:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:31.2:1616513079.286514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:2.0:1616513079.286572:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:2.0:1616513079.286581:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:2.0:1616513079.286589:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513079.286598:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 224 into portal 24 MB=0x603db0154c100 00000400:00000200:2.0:1616513079.286605:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 18 from 12345-10.242.46.234@o2ib1 of length 224/224 into md 0xf97369b9 [1] + 0 00000400:00000200:2.0:1616513079.286610:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd4757ba0 00000400:00000200:2.0:1616513079.286612:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.286617:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt_out 00000400:00000200:2.0:1616513079.286623:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757ba0 00000800:00000200:22.0:1616513079.286625:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:2.0:1616513079.286626:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:2.0:1616513079.286629:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:2.0:1616513079.286631:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:10.0:1616513079.286689:0:185125:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693089015316736 00010000:00000200:10.0:1616513079.286711:0:185125:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d741ca80850 x1693089015316736/t0(0) o41->boslfs02-MDT0000-mdtlov_UUID@10.242.46.234@o2ib1:705/0 lens 224/368 e 0 to 0 dl 1616513085 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513079.286725:0:185125:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 4, xid 1693089015316736, offset 192 00000400:00000200:10.0:1616513079.286732:0:185125:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:10.0:1616513079.286739:0:185125:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:10.0:1616513079.286753:0:185125:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513079.286759:0:185125:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:10.0:1616513079.286765:0:185125:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513079.286768:0:185125:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:10.0:1616513079.286770:0:185125:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:10.0:1616513079.286775:0:185125:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513079.286781:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:10.0:1616513079.286783:0:185125:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000800:00000200:18.0:1616513079.286835:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:18.0:1616513079.286842:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:18.0:1616513079.286845:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513079.286848:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:18.0:1616513079.286852:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.286857:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82bb0 00000800:00000200:18.0:1616513079.286861:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:7.2:1616513079.287839:0:49366:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513079.287879:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513079.287893:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.287902:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513079.287911:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.157@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.287921:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.157@tcp1 of length 224 into portal 26 MB=0x5ffd7a0e9a900 00000400:00000200:22.0:1616513079.287928:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.157@tcp1 of length 224/224 into md 0xf97c8009 [1] + 0 00000800:00000200:28.0:1616513079.287933:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513079.287933:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82000 00000400:00000200:22.0:1616513079.287935:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.287938:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513079.287942:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513079.287942:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82000 00000800:00000200:22.0:1616513079.287945:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.287948:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.287950:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513079.287951:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.157@tcp1 : PUT - for me 00000800:00000200:2.0:1616513079.287953:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513079.287961:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.157@tcp1 of length 224 into portal 12 MB=0x5ffd7a0e9ac40 00000400:00000200:28.0:1616513079.287967:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.157@tcp1 of length 224/224 into md 0x865bbec1 [1] + 18144 00000400:00000200:28.0:1616513079.287972:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.287975:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513079.287982:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.287987:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.287988:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.287991:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.288023:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676466272512 00010000:00000200:12.0:1616513079.288033:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688676466272512/t0(0) o400->31294238-412d-59ba-b9d4-b1ee3318b1d1@10.31.129.157@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.288048:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688676466272512, offset 192 00000400:00000200:12.0:1616513079.288054:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.157@tcp1 00000400:00000200:12.0:1616513079.288061:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.157@tcp1 routed destination 00000100:00000200:0.0:1616513079.288065:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676466273344 00000400:00000200:12.0:1616513079.288076:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.157@tcp1(10.31.129.157@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513079.288078:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9f080 x1688676466273344/t0(0) o400->6ece035f-9fc3-fc2a-d517-3e0ce97e4c93@10.31.129.157@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513079.288083:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.288089:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.288092:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513079.288092:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688676466273344, offset 192 00000800:00000200:12.0:1616513079.288094:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513079.288098:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.157@tcp1 00000800:00000200:12.0:1616513079.288099:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.288103:0:49366:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.288105:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513079.288106:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.157@tcp1 routed destination 00000400:00000200:0.0:1616513079.288120:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.157@tcp1(10.31.129.157@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.288129:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.288135:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.288137:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.288139:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.288144:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.288150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513079.288158:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513079.288161:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513079.288165:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.288168:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513079.288168:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513079.288173:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.288174:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.288179:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.288180:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241870 00000800:00000200:26.0:1616513079.288183:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513079.288183:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04198 00000800:00000200:26.0:1616513079.288186:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513079.288187:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.288190:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.297590:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.297645:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.297655:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.297664:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.75@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.297674:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.75@tcp1 of length 224 into portal 26 MB=0x5f91d65e14b80 00000400:00000200:28.0:1616513079.297681:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.75@tcp1 of length 224/224 into md 0xf97c8019 [1] + 0 00000400:00000200:28.0:1616513079.297686:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83ba0 00000400:00000200:28.0:1616513079.297689:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.297691:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513079.297697:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83ba0 00000800:00000200:24.0:1616513079.297699:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.297700:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.297702:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.297707:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.297710:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.297711:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.297719:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.75@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.297729:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.75@tcp1 of length 224 into portal 12 MB=0x5f91d65e14c00 00000400:00000200:24.0:1616513079.297735:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.75@tcp1 of length 224/224 into md 0x865bbec1 [1] + 18368 00000400:00000200:24.0:1616513079.297740:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.297743:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513079.297750:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.297753:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.297755:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.297771:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681279542184832 00010000:00000200:12.0:1616513079.297782:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1681279542184832/t0(0) o400->e071e473-b185-6b87-e323-2c25978c1ece@10.31.131.75@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.297796:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681279542184832, offset 192 00000400:00000200:12.0:1616513079.297802:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.75@tcp1 00000400:00000200:12.0:1616513079.297810:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.75@tcp1 routed destination 00000100:00000200:0.0:1616513079.297815:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681279542184960 00000400:00000200:12.0:1616513079.297824:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.75@tcp1(10.31.131.75@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513079.297827:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc3f00 x1681279542184960/t0(0) o400->998a9580-5064-c6a4-e071-602b973f7f33@10.31.131.75@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513079.297831:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.297837:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.297839:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.297841:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513079.297842:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681279542184960, offset 192 00000800:00000200:12.0:1616513079.297846:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513079.297848:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.75@tcp1 00000100:00000200:12.0:1616513079.297853:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.297854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513079.297856:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.75@tcp1 routed destination 00000400:00000200:0.0:1616513079.297871:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.75@tcp1(10.31.131.75@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.297877:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.297883:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.297886:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.297888:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.297893:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.297898:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513079.297908:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513079.297911:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513079.297916:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.297918:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513079.297919:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.297923:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513079.297923:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513079.297925:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:4.0:1616513079.297929:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.297931:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241540 00000800:00000200:22.0:1616513079.297934:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513079.297934:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05298 00000800:00000200:22.0:1616513079.297937:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513079.297939:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.317638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.317694:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.317704:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.317713:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.22@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.317722:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.22@tcp1 of length 224 into portal 26 MB=0x60183be35fe40 00000400:00000200:24.0:1616513079.317729:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.22@tcp1 of length 224/224 into md 0xf97c8039 [1] + 0 00000400:00000200:24.0:1616513079.317734:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82908 00000400:00000200:24.0:1616513079.317738:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.317741:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.317746:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82908 00000800:00000200:20.0:1616513079.317748:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.317749:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.317751:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.317755:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.317758:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.317759:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.317768:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.22@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.317777:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.22@tcp1 of length 224 into portal 12 MB=0x60183be35ff00 00000400:00000200:20.0:1616513079.317783:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.22@tcp1 of length 224/224 into md 0x865bbec1 [1] + 18592 00000400:00000200:20.0:1616513079.317788:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.317791:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513079.317798:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.317801:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.317803:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.317808:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515203817024 00010000:00000200:12.0:1616513079.317819:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1690515203817024/t0(0) o400->5b180203-3184-4a93-b2d6-ecda14892c9f@10.31.132.22@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.317835:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690515203817024, offset 192 00000400:00000200:12.0:1616513079.317842:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.22@tcp1 00000400:00000200:12.0:1616513079.317849:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.22@tcp1 routed destination 00000100:00000200:0.0:1616513079.317862:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515203817216 00000400:00000200:12.0:1616513079.317864:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.22@tcp1(10.31.132.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.317871:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:0.0:1616513079.317875:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b104380 x1690515203817216/t0(0) o400->02f045bc-1658-e604-d13d-216685f88df9@10.31.132.22@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513079.317876:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.317879:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.317881:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.317886:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513079.317889:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690515203817216, offset 192 00000100:00000200:12.0:1616513079.317893:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.317895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513079.317895:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.22@tcp1 00000400:00000200:0.0:1616513079.317903:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.22@tcp1 routed destination 00000400:00000200:0.0:1616513079.317918:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.22@tcp1(10.31.132.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.317925:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.317930:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.317933:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.317935:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.317944:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.317945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513079.317953:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.317960:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.317963:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.317969:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.317975:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240088 00000800:00000200:26.0:1616513079.317978:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.317981:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513079.317996:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.318003:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513079.318006:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.318007:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513079.318012:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513079.318015:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05980 00000800:00000200:4.0:1616513079.318019:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513079.322643:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925504, portal 4 00000100:00000200:15.0:1616513079.322649:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925504, offset 0 00000400:00000200:15.0:1616513079.322656:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:15.0:1616513079.322663:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:15.0:1616513079.322676:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513079.322683:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:15.0:1616513079.322689:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513079.322692:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:15.0:1616513079.322694:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:15.0:1616513079.322700:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513079.322747:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513079.322804:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513079.322811:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513079.322814:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513079.322821:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513079.322827:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded6780 x1693067725925504/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513079.322849:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0ad48 00000800:00000200:22.0:1616513079.322852:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513079.322856:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513079.322915:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513079.322969:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513079.322979:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:26.0:1616513079.322987:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513079.322995:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626c80 00000400:00000200:26.0:1616513079.323003:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c86b9 [1] + 192 00000400:00000200:26.0:1616513079.323008:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.323013:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded6780 x1693067725925504/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513079.323023:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513079.323029:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513079.323031:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513079.323033:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:15.0:1616513079.323038:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded6780 x1693067725925504/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513079.323051:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded6780 x1693067725925504/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513079.323062:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000800:00000200:3.2:1616513079.326330:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5cbeecba00] (20)++ 00000800:00000200:20.0:1616513079.326388:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5cbeecba00] (21)++ 00000800:00000200:20.0:1616513079.326397:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.149@o2ib1 00000400:00000200:20.0:1616513079.326406:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.149@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513079.326414:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.149@o2ib1 of length 224 into portal 26 MB=0x604b9d49acc00 00000400:00000200:20.0:1616513079.326420:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.149@o2ib1 of length 224/224 into md 0xf97c8059 [1] + 0 00000400:00000200:20.0:1616513079.326425:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83980 00000400:00000200:20.0:1616513079.326428:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.326430:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513079.326448:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83980 00000800:00000200:20.0:1616513079.326451:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5cbeecba00] (22)++ 00000800:00000200:28.0:1616513079.326455:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5cbeecba00] (23)++ 00000800:00000200:20.0:1616513079.326455:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5cbeecba00] (23)-- 00000800:00000200:20.0:1616513079.326462:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5cbeecba00] (23)-- 00000800:00000200:20.0:1616513079.326464:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5cbeecba00] (22)-- 00000800:00000200:28.0:1616513079.326465:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.149@o2ib1 00000400:00000200:28.0:1616513079.326474:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.149@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513079.326482:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.149@o2ib1 of length 224 into portal 12 MB=0x604b9d49acc80 00000400:00000200:28.0:1616513079.326488:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.149@o2ib1 of length 224/224 into md 0x865bbec1 [1] + 18816 00000400:00000200:28.0:1616513079.326494:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.326496:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513079.326503:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5cbeecba00] (21)++ 00000800:00000200:28.0:1616513079.326506:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5cbeecba00] (22)-- 00000800:00000200:28.0:1616513079.326508:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5cbeecba00] (21)-- 00000100:00000200:0.0:1616513079.326527:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694046042639488 00010000:00000200:0.0:1616513079.326539:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b99680 x1694046042639488/t0(0) o400->9a1a450b-f481-d311-3e6f-87c8e8c59438@10.242.46.149@o2ib1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513079.326541:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694046042639360 00010000:00000200:12.0:1616513079.326552:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1694046042639360/t0(0) o400->988767f1-0aa4-45ae-1037-5d1717164136@10.242.46.149@o2ib1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513079.326552:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694046042639488, offset 192 00000400:00000200:0.0:1616513079.326558:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.149@o2ib1 00000400:00000200:0.0:1616513079.326565:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.149@o2ib1 local destination 00000100:00000200:12.0:1616513079.326566:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694046042639360, offset 192 00000400:00000200:12.0:1616513079.326573:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.149@o2ib1 00000400:00000200:0.0:1616513079.326578:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.149@o2ib1(10.242.46.149@o2ib1:10.242.46.149@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.326584:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.149@o2ib1 00000400:00000200:12.0:1616513079.326586:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.149@o2ib1 local destination 00000400:00000200:12.0:1616513079.326600:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.149@o2ib1(10.242.46.149@o2ib1:10.242.46.149@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.326606:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.149@o2ib1 00000800:00000200:0.0:1616513079.326607:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d56406b3600] -> 10.242.46.149@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.326609:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5cbeecba00] (20)++ 00000800:00000200:0.0:1616513079.326611:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5cbeecba00] (21)++ 00000800:00000200:12.0:1616513079.326612:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d56406b3600] -> 10.242.46.149@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.326615:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5cbeecba00] (22)++ 00000800:00000200:0.0:1616513079.326616:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5cbeecba00] (22)-- 00000800:00000200:12.0:1616513079.326617:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5cbeecba00] (23)++ 00000800:00000200:12.0:1616513079.326621:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5cbeecba00] (23)-- 00000800:00000200:3.2:1616513079.326622:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5cbeecba00] (23)++ 00000100:00000200:12.0:1616513079.326628:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513079.326640:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5cbeecba00] (23)++ 00000800:00000200:4.0:1616513079.326647:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5cbeecba00] (24)-- 00000400:00000200:4.0:1616513079.326650:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513079.326651:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5cbeecba00] (23)++ 00000800:00000200:20.0:1616513079.326655:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5cbeecba00] (24)-- 00000400:00000200:4.0:1616513079.326656:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.149@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.326657:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.326659:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5cbeecba00] (23)-- 00000400:00000200:4.0:1616513079.326660:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb043b8 00000400:00000200:20.0:1616513079.326662:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.149@o2ib1: PUT: OK 00000800:00000200:4.0:1616513079.326663:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5cbeecba00] (22)-- 00000400:00000200:20.0:1616513079.326667:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241298 00000800:00000200:20.0:1616513079.326672:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5cbeecba00] (21)-- 00000800:00000200:11.2:1616513079.327774:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.327862:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.327871:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.327880:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.185@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.327890:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.185@tcp1 of length 224 into portal 26 MB=0x5ffd4d64a8880 00000400:00000200:24.0:1616513079.327897:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.185@tcp1 of length 224/224 into md 0xf97c8079 [1] + 0 00000400:00000200:24.0:1616513079.327901:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd834b8 00000400:00000200:24.0:1616513079.327904:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.327907:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.327911:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd834b8 00000800:00000200:24.0:1616513079.327914:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.327916:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.327918:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.327921:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.327942:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513079.327953:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.327960:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513079.327968:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.185@tcp1 : PUT - for me 00000100:00000200:12.0:1616513079.327972:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664476911744 00000400:00000200:4.0:1616513079.327977:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.185@tcp1 of length 224 into portal 12 MB=0x5ffd4d64a8940 00010000:00000200:12.0:1616513079.327983:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688664476911744/t0(0) o400->9a0b399c-944a-2030-7059-892729fee858@10.31.136.185@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:4.0:1616513079.327983:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.185@tcp1 of length 224/224 into md 0x865bbec1 [1] + 19040 00000400:00000200:4.0:1616513079.327988:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513079.327990:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513079.327996:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:12.0:1616513079.327997:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664476911744, offset 192 00000800:00000200:4.0:1616513079.327999:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513079.328001:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513079.328004:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.185@tcp1 00000800:00000200:4.0:1616513079.328004:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:12.0:1616513079.328012:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.185@tcp1 routed destination 00000400:00000200:12.0:1616513079.328029:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.185@tcp1(10.31.136.185@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.328035:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.328041:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.328043:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.328045:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.328050:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.328055:0:49370:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.328057:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513079.328061:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664476911936 00010000:00000200:0.0:1616513079.328073:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23cc80 x1688664476911936/t0(0) o400->013f68da-b81a-9ba9-ccd2-8796679b0ba6@10.31.136.185@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.328086:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664476911936, offset 192 00000400:00000200:0.0:1616513079.328092:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.185@tcp1 00000400:00000200:0.0:1616513079.328099:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.185@tcp1 routed destination 00000800:00000200:2.0:1616513079.328112:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513079.328113:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.185@tcp1(10.31.136.185@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513079.328121:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513079.328123:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513079.328124:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513079.328128:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.328130:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513079.328131:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513079.328132:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513079.328136:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ff0 00000800:00000200:0.0:1616513079.328137:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513079.328140:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513079.328142:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513079.328143:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.328149:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513079.328151:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513079.328152:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.328154:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513079.328155:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513079.328155:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513079.328157:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.328158:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513079.328159:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.328162:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.328165:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb042a8 00000800:00000200:24.0:1616513079.328168:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.339091:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.339146:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.339155:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513079.339156:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513079.339164:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.107@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.339173:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.107@tcp1 of length 224 into portal 26 MB=0x5ffdbd08116c0 00000400:00000200:20.0:1616513079.339179:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.107@tcp1 of length 224/224 into md 0xf97c8089 [1] + 0 00000800:00000200:7.2:1616513079.339180:0:49371:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513079.339183:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83540 00000400:00000200:20.0:1616513079.339186:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.339189:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513079.339193:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83540 00000800:00000200:20.0:1616513079.339196:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.339198:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.339200:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513079.339239:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.339252:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513079.339260:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.107@tcp1 : PUT - for me 00000100:00000200:12.0:1616513079.339267:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694444594880 00000400:00000200:26.0:1616513079.339270:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.107@tcp1 of length 224 into portal 12 MB=0x5ffdbd0811740 00000400:00000200:26.0:1616513079.339277:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.107@tcp1 of length 224/224 into md 0xf96450b9 [1] + 85248 00010000:00000200:12.0:1616513079.339277:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688694444594880/t0(0) o400->b3dcc14a-a7b1-c5a7-585c-b9a54bc15dcb@10.31.136.107@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513079.339282:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.339285:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:12.0:1616513079.339290:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694444594880, offset 192 00000800:00000200:26.0:1616513079.339292:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.339295:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:12.0:1616513079.339296:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.107@tcp1 00000800:00000200:2.0:1616513079.339297:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.339301:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513079.339302:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.107@tcp1 routed destination 00000400:00000200:12.0:1616513079.339316:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.107@tcp1(10.31.136.107@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.339322:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.339328:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.339330:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.339332:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.339337:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.339341:0:49371:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.339343:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:14.0:1616513079.339361:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694444595008 00010000:00000200:14.0:1616513079.339374:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8900 x1688694444595008/t0(0) o400->748c3106-18ca-5aa2-10bb-9083b556cb93@10.31.136.107@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.339391:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694444595008, offset 192 00000800:00000200:22.0:1616513079.339394:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513079.339397:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.107@tcp1 00000800:00000200:22.0:1616513079.339401:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.339404:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513079.339404:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.107@tcp1 routed destination 00000400:00000200:22.0:1616513079.339410:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.339416:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240000 00000400:00000200:14.0:1616513079.339418:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.107@tcp1(10.31.136.107@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513079.339420:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.339422:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:14.0:1616513079.339425:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.339430:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.339433:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.339449:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.339454:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.339459:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.339511:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.339517:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.339520:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.339528:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.339532:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756a18 00000800:00000200:28.0:1616513079.339535:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.339538:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.355317:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.355372:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.355382:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.355391:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.10@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.355402:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.10@tcp1 of length 224 into portal 26 MB=0x5ffd6a2247280 00000400:00000200:2.0:1616513079.355408:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.10@tcp1 of length 224/224 into md 0xf97c8099 [1] + 0 00000400:00000200:2.0:1616513079.355413:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82c38 00000400:00000200:2.0:1616513079.355415:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.355418:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.355423:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82c38 00000800:00000200:2.0:1616513079.355426:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.355428:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.355429:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.355435:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.355454:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513079.355456:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513079.355463:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.10@tcp1 : PUT - for me 00000400:00000200:26.0:1616513079.355472:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.10@tcp1 of length 224 into portal 12 MB=0x5ffd6a2247300 00000400:00000200:26.0:1616513079.355479:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.10@tcp1 of length 224/224 into md 0xf96450b9 [1] + 85472 00000400:00000200:26.0:1616513079.355484:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513079.355486:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672191935104 00000100:00000200:26.0:1616513079.355487:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513079.355493:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.355496:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:12.0:1616513079.355496:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688672191935104/t0(0) o400->ccde19b4-1cb0-33db-99dc-fe010226311d@10.31.138.10@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513079.355498:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513079.355511:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688672191935104, offset 192 00000400:00000200:12.0:1616513079.355517:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.10@tcp1 00000400:00000200:12.0:1616513079.355524:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.10@tcp1 routed destination 00000400:00000200:12.0:1616513079.355540:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.10@tcp1(10.31.138.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.355546:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.355552:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.355556:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.355559:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.355563:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513079.355564:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672191935232 00000800:00000200:7.2:1616513079.355568:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.355570:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:14.0:1616513079.355577:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8d80 x1688672191935232/t0(0) o400->50209d51-012b-4fd1-4354-0aaca0364f4a@10.31.138.10@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.355591:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688672191935232, offset 192 00000400:00000200:14.0:1616513079.355598:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.10@tcp1 00000400:00000200:14.0:1616513079.355605:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.10@tcp1 routed destination 00000400:00000200:14.0:1616513079.355622:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.10@tcp1(10.31.138.10@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513079.355623:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513079.355629:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513079.355630:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.355633:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513079.355634:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513079.355634:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513079.355637:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513079.355639:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:14.0:1616513079.355639:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.355644:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513079.355645:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241430 00000800:00000200:18.0:1616513079.355648:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.355650:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.355705:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.355712:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513079.355715:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513079.355721:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513079.355725:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47567f8 00000800:00000200:4.0:1616513079.355729:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513079.355734:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.367516:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.367572:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.367581:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.367590:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.171@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.367599:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.171@tcp1 of length 224 into portal 12 MB=0x60185f9747980 00000400:00000200:28.0:1616513079.367605:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.171@tcp1 of length 224/224 into md 0x865bbec1 [1] + 19264 00000400:00000200:28.0:1616513079.367611:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.367613:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513079.367620:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.367623:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.367625:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.367628:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.367638:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.367647:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.171@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.367655:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.171@tcp1 of length 224 into portal 26 MB=0x60185f97475c0 00000400:00000200:24.0:1616513079.367662:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.171@tcp1 of length 224/224 into md 0xf97c80a9 [1] + 0 00000400:00000200:24.0:1616513079.367667:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82ff0 00000400:00000200:24.0:1616513079.367669:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.367674:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.367678:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82ff0 00000100:00000200:0.0:1616513079.367680:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690524787702144 00000800:00000200:24.0:1616513079.367681:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.367684:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513079.367684:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.367688:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513079.367691:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b98d80 x1690524787702144/t0(0) o400->9e3230cc-ab0c-3ec6-ca88-d72ef29505a8@10.31.129.171@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.367704:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690524787702144, offset 192 00000400:00000200:0.0:1616513079.367709:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.171@tcp1 00000400:00000200:0.0:1616513079.367716:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.171@tcp1 routed destination 00000400:00000200:0.0:1616513079.367730:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.171@tcp1(10.31.129.171@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.367736:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513079.367741:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.367743:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.367745:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.367750:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.367758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.367759:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690524787701184 00010000:00000200:12.0:1616513079.367770:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1690524787701184/t0(0) o400->ab98f97a-9e0f-b496-57d4-46ff1dfd5243@10.31.129.171@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.367784:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690524787701184, offset 192 00000400:00000200:12.0:1616513079.367790:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.171@tcp1 00000400:00000200:12.0:1616513079.367797:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.171@tcp1 routed destination 00000400:00000200:12.0:1616513079.367813:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.171@tcp1(10.31.129.171@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.367819:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513079.367824:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513079.367825:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.367827:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.367830:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513079.367831:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.367833:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513079.367834:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.367839:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513079.367840:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:12.0:1616513079.367841:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513079.367844:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04550 00000800:00000200:26.0:1616513079.367848:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.367851:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513079.367852:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.367859:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513079.367861:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.367863:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513079.367868:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.367874:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:20.0:1616513079.367877:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:3.0:1616513079.370671:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925520, portal 4 00000100:00000200:3.0:1616513079.370677:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925520, offset 0 00000400:00000200:3.0:1616513079.370684:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:3.0:1616513079.370690:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:3.0:1616513079.370704:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513079.370711:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513079.370717:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513079.370720:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:3.0:1616513079.370722:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:3.0:1616513079.370727:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513079.370775:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513079.370831:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513079.370838:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513079.370841:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513079.370847:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513079.370854:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456eb600 x1693067725925520/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513079.370866:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2550 00000800:00000200:4.0:1616513079.370870:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513079.370873:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513079.370949:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513079.371004:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513079.371013:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:28.0:1616513079.371021:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513079.371029:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626c90 00000400:00000200:28.0:1616513079.371036:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d52c1 [1] + 192 00000400:00000200:28.0:1616513079.371042:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.371047:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456eb600 x1693067725925520/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513079.371056:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513079.371062:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513079.371065:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513079.371067:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:3.0:1616513079.371072:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456eb600 x1693067725925520/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513079.371085:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456eb600 x1693067725925520/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513079.371098:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:11.2:1616513079.391841:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513079.391912:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513079.391923:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513079.391932:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.192@tcp1 : PUT - for me 00000400:00000200:4.0:1616513079.391941:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.192@tcp1 of length 224 into portal 26 MB=0x6039103314ac0 00000400:00000200:4.0:1616513079.391948:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.192@tcp1 of length 224/224 into md 0xf97c80b9 [1] + 0 00000400:00000200:4.0:1616513079.391953:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82440 00000400:00000200:4.0:1616513079.391956:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513079.391958:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513079.391963:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82440 00000800:00000200:24.0:1616513079.391967:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.391967:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.391970:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513079.391974:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513079.391977:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.391978:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.391987:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.192@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.391995:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.192@tcp1 of length 224 into portal 12 MB=0x6039103314b80 00000400:00000200:24.0:1616513079.392002:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.192@tcp1 of length 224/224 into md 0x865bbec1 [1] + 19488 00000400:00000200:24.0:1616513079.392007:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.392010:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513079.392017:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.392020:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.392022:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.392025:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692771218967232 00010000:00000200:12.0:1616513079.392036:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1692771218967232/t0(0) o400->ca416fcb-29ce-8ff5-cd85-70aa8db39d62@10.31.131.192@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.392049:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692771218967232, offset 192 00000400:00000200:12.0:1616513079.392055:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.192@tcp1 00000400:00000200:12.0:1616513079.392062:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.192@tcp1 routed destination 00000400:00000200:12.0:1616513079.392077:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.192@tcp1(10.31.131.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513079.392082:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692771218967424 00000800:00000200:12.0:1616513079.392083:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.392089:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.392091:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.392093:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513079.392095:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc0d80 x1692771218967424/t0(0) o400->bd039ee2-dadc-cc06-542d-49ee4a6b51f0@10.31.131.192@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513079.392098:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513079.392104:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513079.392109:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692771218967424, offset 192 00000400:00000200:0.0:1616513079.392115:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.192@tcp1 00000400:00000200:0.0:1616513079.392123:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.192@tcp1 routed destination 00000400:00000200:0.0:1616513079.392138:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.192@tcp1(10.31.131.192@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.392144:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513079.392145:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.392150:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.392153:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.392155:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.392160:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.392165:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513079.392203:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.392211:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513079.392221:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513079.392222:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.392229:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513079.392231:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.392234:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.392234:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240aa0 00000800:00000200:8.0:1616513079.392235:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513079.392238:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513079.392240:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513079.392241:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513079.392244:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb056d8 00000800:00000200:28.0:1616513079.392248:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.401952:0:49377:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513079.402010:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.402019:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513079.402028:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:26.0:1616513079.402038:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d0322ac480 00000400:00000200:26.0:1616513079.402045:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0xf96450b9 [1] + 85696 00000400:00000200:26.0:1616513079.402050:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.402055:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513079.402063:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.402064:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.402066:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.402068:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.402127:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042590008448 00010000:00000200:14.0:1616513079.402222:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dec00 x1693042590008448/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:705/0 lens 440/368 e 0 to 0 dl 1616513085 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:14.0:1616513079.402236:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042590008448, offset 192 00000400:00000200:14.0:1616513079.402241:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:14.0:1616513079.402249:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:14.0:1616513079.402262:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.402268:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.402274:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.402276:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.402278:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.402283:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.402287:0:49377:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.402341:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.402348:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.402351:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.402357:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.402361:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756330 00000800:00000200:2.0:1616513079.402365:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.402368:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.405786:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.405799:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.405807:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.405816:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.152@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.405824:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.152@tcp1 of length 224 into portal 26 MB=0x601983fd18f80 00000400:00000200:24.0:1616513079.405830:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.152@tcp1 of length 224/224 into md 0xf97c80c9 [1] + 0 00000400:00000200:24.0:1616513079.405835:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83b18 00000400:00000200:24.0:1616513079.405837:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.405840:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.405845:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83b18 00000800:00000200:24.0:1616513079.405848:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.405850:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513079.405851:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.405855:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.405907:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690603277619072 00010000:00000200:12.0:1616513079.405918:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1690603277619072/t0(0) o400->7d8e5071-a861-ecfb-bdbe-93d527112486@10.31.131.152@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.405932:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690603277619072, offset 192 00000400:00000200:12.0:1616513079.405938:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.152@tcp1 00000400:00000200:12.0:1616513079.405945:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.152@tcp1 routed destination 00000400:00000200:12.0:1616513079.405960:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.152@tcp1(10.31.131.152@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.405966:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.405972:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.405975:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.405977:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.405981:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.405986:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.405987:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513079.405996:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.406002:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.406004:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.406010:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.406015:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ee0 00000800:00000200:8.0:1616513079.406018:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.406020:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.406216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.406232:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.406240:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513079.406241:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513079.406249:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.152@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.406258:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.152@tcp1 of length 224 into portal 12 MB=0x601983fd19340 00000400:00000200:20.0:1616513079.406264:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.152@tcp1 of length 224/224 into md 0x865bbec1 [1] + 19712 00000400:00000200:20.0:1616513079.406269:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.406272:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513079.406279:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.406282:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.406284:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.406346:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690603277620032 00010000:00000200:0.0:1616513079.406358:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b101680 x1690603277620032/t0(0) o400->0e9fad80-c731-4099-68ba-72b1d695fb80@10.31.131.152@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.406373:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690603277620032, offset 192 00000400:00000200:0.0:1616513079.406379:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.152@tcp1 00000400:00000200:0.0:1616513079.406386:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.152@tcp1 routed destination 00000400:00000200:0.0:1616513079.406401:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.152@tcp1(10.31.131.152@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.406407:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513079.406413:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.406416:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.406420:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.406424:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.406432:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.406488:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.406495:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.406497:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513079.406503:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.406504:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.406508:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05188 00000800:00000200:22.0:1616513079.406512:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.413769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.413781:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.413788:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.413794:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.210@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.413802:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.210@tcp1 of length 224 into portal 26 MB=0x60195387e7480 00000400:00000200:8.0:1616513079.413807:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.210@tcp1 of length 224/224 into md 0xf97c80e9 [1] + 0 00000400:00000200:8.0:1616513079.413811:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd837e8 00000400:00000200:8.0:1616513079.413814:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.413816:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513079.413820:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd837e8 00000800:00000200:8.0:1616513079.413822:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.413824:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.413826:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.413828:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.413882:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590269830272 00010000:00000200:12.0:1616513079.413893:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1690590269830272/t0(0) o400->2b4e9bbe-b631-d2f4-16e3-d992cf6851d1@10.31.140.210@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.413907:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690590269830272, offset 192 00000400:00000200:12.0:1616513079.413913:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.210@tcp1 00000400:00000200:12.0:1616513079.413921:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.210@tcp1 routed destination 00000400:00000200:12.0:1616513079.413936:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.210@tcp1(10.31.140.210@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.413942:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.413948:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.413951:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.413953:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.413957:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.413963:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.413964:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513079.414015:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.414021:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.414024:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.414030:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.414035:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241320 00000800:00000200:24.0:1616513079.414040:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.414043:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.414049:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513079.414049:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.414056:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.210@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.414063:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.210@tcp1 of length 224 into portal 12 MB=0x60195387e7940 00000400:00000200:24.0:1616513079.414069:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.210@tcp1 of length 224/224 into md 0x865bbec1 [1] + 19936 00000400:00000200:24.0:1616513079.414073:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.414075:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513079.414081:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.414084:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.414085:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.414147:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590269831488 00010000:00000200:0.0:1616513079.414159:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc7980 x1690590269831488/t0(0) o400->e752dc4d-d535-ac2a-2a09-a731d047d42d@10.31.140.210@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.414174:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690590269831488, offset 192 00000400:00000200:0.0:1616513079.414180:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.210@tcp1 00000400:00000200:0.0:1616513079.414187:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.210@tcp1 routed destination 00000400:00000200:0.0:1616513079.414202:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.210@tcp1(10.31.140.210@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.414209:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513079.414215:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.414218:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.414220:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.414225:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.414234:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.414289:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.414296:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.414299:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.414305:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.414309:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a08 00000800:00000200:2.0:1616513079.414313:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.414316:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.416236:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.416294:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.416303:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513079.416303:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.416307:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.416312:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.190@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.416313:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.190@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.416321:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.190@tcp1 of length 224 into portal 12 MB=0x5ffd4a39bf300 00000400:00000200:28.0:1616513079.416323:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.190@tcp1 of length 224 into portal 26 MB=0x5ffd4a39bf1c0 00000400:00000200:8.0:1616513079.416326:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.190@tcp1 of length 224/224 into md 0x865bbec1 [1] + 20160 00000400:00000200:28.0:1616513079.416329:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.190@tcp1 of length 224/224 into md 0xf97c8129 [1] + 0 00000400:00000200:8.0:1616513079.416329:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.416331:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513079.416334:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd823b8 00000800:00000200:8.0:1616513079.416335:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:28.0:1616513079.416337:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:8.0:1616513079.416337:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513079.416338:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:28.0:1616513079.416339:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513079.416340:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.416344:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd823b8 00000800:00000200:28.0:1616513079.416347:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.416350:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.416352:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.416400:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663626609408 00010000:00000200:0.0:1616513079.416413:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3a80 x1688663626609408/t0(0) o400->2ddae934-a9ae-1fe3-1f72-6e5b5fafaf50@10.31.132.190@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513079.416418:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663626609088 00000100:00000200:0.0:1616513079.416427:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663626609408, offset 192 00010000:00000200:12.0:1616513079.416429:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688663626609088/t0(0) o400->cbda3584-4262-844d-81e5-59dc30ec137a@10.31.132.190@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513079.416444:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.190@tcp1 00000400:00000200:0.0:1616513079.416452:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.190@tcp1 routed destination 00000100:00000200:12.0:1616513079.416459:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663626609088, offset 192 00000400:00000200:0.0:1616513079.416466:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.190@tcp1(10.31.132.190@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513079.416468:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.190@tcp1 00000800:00000200:0.0:1616513079.416473:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513079.416476:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.190@tcp1 routed destination 00000800:00000200:0.0:1616513079.416483:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.416486:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.416488:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513079.416491:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.190@tcp1(10.31.132.190@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.416493:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.416497:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513079.416498:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.416503:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.416506:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.416508:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.416513:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.416518:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.416519:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513079.416533:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.416540:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.416543:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.416550:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513079.416550:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513079.416555:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240770 00000800:00000200:24.0:1616513079.416557:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513079.416558:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513079.416559:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.416560:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513079.416561:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513079.416566:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.416569:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb054b8 00000800:00000200:24.0:1616513079.416573:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.416577:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.416589:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.416600:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513079.416603:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.416606:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.416607:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.416609:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.420463:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513079.420517:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.420526:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513079.420535:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.64@tcp1 : PUT - for me 00000400:00000200:26.0:1616513079.420544:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.64@tcp1 of length 224 into portal 26 MB=0x5ffd4ae1a0200 00000400:00000200:26.0:1616513079.420550:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.64@tcp1 of length 224/224 into md 0xf97c8139 [1] + 0 00000400:00000200:26.0:1616513079.420555:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82d48 00000400:00000200:26.0:1616513079.420558:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.420561:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513079.420565:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82d48 00000800:00000200:26.0:1616513079.420568:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.420571:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.420571:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.420575:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513079.420627:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663802642944 00010000:00000200:12.0:1616513079.420637:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688663802642944/t0(0) o400->9a4befc3-25e6-503c-d70d-1c3ddcff6595@10.31.138.64@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.420651:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663802642944, offset 192 00000400:00000200:12.0:1616513079.420656:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.64@tcp1 00000400:00000200:12.0:1616513079.420664:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.64@tcp1 routed destination 00000400:00000200:12.0:1616513079.420677:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.64@tcp1(10.31.138.64@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.420683:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.420688:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.420691:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.420693:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.420697:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513079.420701:0:49379:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:12.0:1616513079.420703:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513079.420743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513079.420756:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.420765:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.420774:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.64@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.420782:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.64@tcp1 of length 224 into portal 12 MB=0x5ffd4ae1a0340 00000400:00000200:2.0:1616513079.420788:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.64@tcp1 of length 224/224 into md 0xf96450b9 [1] + 86136 00000400:00000200:2.0:1616513079.420794:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.420796:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513079.420797:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.420803:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513079.420803:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.420806:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513079.420808:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.420808:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.420811:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513079.420814:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.420819:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ee0 00000800:00000200:20.0:1616513079.420822:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.420825:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513079.420866:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663802643264 00010000:00000200:14.0:1616513079.420878:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464622400 x1688663802643264/t0(0) o400->01c8c56f-e27e-38d9-b42e-5416a9895830@10.31.138.64@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.420892:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663802643264, offset 192 00000400:00000200:14.0:1616513079.420898:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.64@tcp1 00000400:00000200:14.0:1616513079.420905:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.64@tcp1 routed destination 00000400:00000200:14.0:1616513079.420920:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.64@tcp1(10.31.138.64@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.420927:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.420933:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.420935:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.420939:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.420944:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.420950:0:49379:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.420961:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.420966:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.420968:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513079.420970:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.420975:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.420979:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756c38 00000800:00000200:22.0:1616513079.420983:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.423193:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.423204:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.423209:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.423215:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.24@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.423221:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.24@tcp1 of length 224 into portal 12 MB=0x5ffd38352fe00 00000400:00000200:8.0:1616513079.423225:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.24@tcp1 of length 224/224 into md 0x865bbec1 [1] + 20384 00000400:00000200:8.0:1616513079.423228:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.423230:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513079.423234:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.423235:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.423237:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.423238:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.423283:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513079.423300:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658789989888 00010000:00000200:0.0:1616513079.423312:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d2400 x1688658789989888/t0(0) o400->0ff4431c-cb40-f61f-c793-a0481007e91d@10.31.138.24@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.423325:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658789989888, offset 192 00000400:00000200:0.0:1616513079.423332:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.24@tcp1 00000800:00000200:2.0:1616513079.423337:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513079.423339:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.24@tcp1 routed destination 00000800:00000200:2.0:1616513079.423347:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513079.423354:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.24@tcp1(10.31.138.24@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513079.423356:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.24@tcp1 : PUT - for me 00000800:00000200:0.0:1616513079.423360:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513079.423364:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.24@tcp1 of length 224 into portal 26 MB=0x5ffd38352fcc0 00000800:00000200:0.0:1616513079.423365:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.423368:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.423370:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513079.423373:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.24@tcp1 of length 224/224 into md 0xf97c8149 [1] + 0 00000800:00000200:0.0:1616513079.423374:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513079.423377:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd824c8 00000800:00000200:11.2:1616513079.423379:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513079.423380:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.423383:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.423387:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd824c8 00000800:00000200:20.0:1616513079.423390:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513079.423390:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.423392:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.423393:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.423396:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513079.423397:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513079.423399:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513079.423405:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.423409:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb045d8 00000800:00000200:20.0:1616513079.423413:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.423415:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.423457:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658789989568 00010000:00000200:12.0:1616513079.423467:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688658789989568/t0(0) o400->de91d1c1-c8ac-8f01-63f7-85dd4f651def@10.31.138.24@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.423481:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658789989568, offset 192 00000400:00000200:12.0:1616513079.423487:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.24@tcp1 00000400:00000200:12.0:1616513079.423494:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.24@tcp1 routed destination 00000400:00000200:12.0:1616513079.423508:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.24@tcp1(10.31.138.24@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.423514:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.423521:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.423523:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513079.423525:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.423530:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.423535:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.423536:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513079.423589:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.423596:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.423599:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.423605:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.423611:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:26.0:1616513079.423614:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.423617:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.429055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513079.429107:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.429115:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513079.429124:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.116@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.429132:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.116@tcp1 of length 224 into portal 26 MB=0x6000122790f00 00000400:00000200:18.0:1616513079.429138:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.116@tcp1 of length 224/224 into md 0xf97c8159 [1] + 0 00000400:00000200:18.0:1616513079.429142:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83f58 00000400:00000200:18.0:1616513079.429144:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.429147:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513079.429151:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83f58 00000800:00000200:18.0:1616513079.429154:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.429156:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.429158:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.429160:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.429171:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.429180:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.116@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.429188:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.116@tcp1 of length 224 into portal 12 MB=0x6000122790f80 00000400:00000200:2.0:1616513079.429194:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.116@tcp1 of length 224/224 into md 0xf96450b9 [1] + 86360 00000400:00000200:2.0:1616513079.429199:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.429202:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513079.429208:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:12.0:1616513079.429211:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688854733590272 00000800:00000200:2.0:1616513079.429211:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.429213:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.429216:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:12.0:1616513079.429221:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688854733590272/t0(0) o400->8d2c69e8-f6cb-963f-baf7-ac644ebf05aa@10.31.138.116@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.429234:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688854733590272, offset 192 00000400:00000200:12.0:1616513079.429240:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.116@tcp1 00000400:00000200:12.0:1616513079.429246:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.116@tcp1 routed destination 00000400:00000200:12.0:1616513079.429260:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.116@tcp1(10.31.138.116@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.429266:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:14.0:1616513079.429271:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688854733590400 00000800:00000200:12.0:1616513079.429271:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.429274:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.429276:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.429280:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:14.0:1616513079.429283:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464627980 x1688854733590400/t0(0) o400->0e5850ae-b57d-9a5c-852a-e25d770fd345@10.31.138.116@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513079.429286:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:14.0:1616513079.429297:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688854733590400, offset 192 00000400:00000200:14.0:1616513079.429303:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.116@tcp1 00000400:00000200:14.0:1616513079.429310:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.116@tcp1 routed destination 00000400:00000200:14.0:1616513079.429325:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.116@tcp1(10.31.138.116@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513079.429328:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.429332:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.429338:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.429340:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513079.429340:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513079.429342:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513079.429344:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.429346:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513079.429347:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513079.429350:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513079.429352:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513079.429354:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240000 00000800:00000200:8.0:1616513079.429357:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.429359:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513079.429412:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.429419:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.429422:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513079.429423:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.429428:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.429449:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756d48 00000800:00000200:22.0:1616513079.429454:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513079.434554:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925536, portal 4 00000100:00000200:10.0:1616513079.434559:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925536, offset 0 00000400:00000200:10.0:1616513079.434565:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:10.0:1616513079.434570:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:10.0:1616513079.434580:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513079.434586:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:10.0:1616513079.434591:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513079.434594:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513079.434596:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513079.434600:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513079.434647:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513079.434661:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513079.434667:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513079.434671:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.434672:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:26.0:1616513079.434677:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513079.434683:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6a7d100 x1693067725925536/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513079.434694:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82770 00000800:00000200:26.0:1616513079.434698:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513079.434722:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513079.434776:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513079.434784:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513079.434793:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513079.434800:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626ca0 00000400:00000200:18.0:1616513079.434806:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c87d9 [1] + 192 00000400:00000200:18.0:1616513079.434811:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.434815:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6a7d100 x1693067725925536/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513079.434827:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513079.434829:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513079.434832:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513079.434833:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:10.0:1616513079.434833:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6a7d100 x1693067725925536/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513079.434844:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6a7d100 x1693067725925536/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513079.434853:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83078 00000800:00000200:11.2:1616513079.441253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.441309:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.441328:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.441337:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.58@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.441345:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.58@tcp1 of length 224 into portal 26 MB=0x5ffd507650340 00000400:00000200:20.0:1616513079.441351:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.58@tcp1 of length 224/224 into md 0xf97c8179 [1] + 0 00000400:00000200:20.0:1616513079.441356:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83a90 00000400:00000200:20.0:1616513079.441358:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.441361:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513079.441365:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83a90 00000800:00000200:20.0:1616513079.441368:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.441371:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513079.441373:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.441375:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.441427:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665300730688 00010000:00000200:12.0:1616513079.441448:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688665300730688/t0(0) o400->48d93325-0628-c6b5-859b-1899a6848427@10.31.138.58@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.441461:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665300730688, offset 192 00000400:00000200:12.0:1616513079.441468:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.58@tcp1 00000400:00000200:12.0:1616513079.441474:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.58@tcp1 routed destination 00000400:00000200:12.0:1616513079.441488:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.58@tcp1(10.31.138.58@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.441494:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.441499:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.441502:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.441504:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.441508:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.441514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.441516:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513079.441523:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.441527:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.441530:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.441534:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.441538:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241298 00000800:00000200:8.0:1616513079.441541:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.441543:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.442444:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.442508:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.442518:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.442526:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.58@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.442534:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.58@tcp1 of length 224 into portal 12 MB=0x5ffd5076503c0 00000400:00000200:2.0:1616513079.442540:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.58@tcp1 of length 224/224 into md 0xf96450b9 [1] + 86584 00000400:00000200:2.0:1616513079.442545:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.442547:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513079.442553:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.442556:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.442558:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.442561:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.442620:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665300730816 00010000:00000200:14.0:1616513079.442632:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621680 x1688665300730816/t0(0) o400->67d00b4e-70da-df8b-27d8-702a8e762ce6@10.31.138.58@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.442647:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665300730816, offset 192 00000400:00000200:14.0:1616513079.442653:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.58@tcp1 00000400:00000200:14.0:1616513079.442660:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.58@tcp1 routed destination 00000800:00000200:7.2:1616513079.442667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:14.0:1616513079.442674:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.58@tcp1(10.31.138.58@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.442680:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.442686:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.442689:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.442691:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513079.442696:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.442721:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.442736:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513079.442744:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.72@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.442753:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.72@tcp1 of length 224 into portal 26 MB=0x603ff7e8a9080 00000400:00000200:22.0:1616513079.442759:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.72@tcp1 of length 224/224 into md 0xf97c8189 [1] + 0 00000400:00000200:22.0:1616513079.442763:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82088 00000400:00000200:22.0:1616513079.442766:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.442768:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.442772:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82088 00000800:00000200:22.0:1616513079.442775:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513079.442777:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513079.442777:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:22.0:1616513079.442784:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513079.442786:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513079.442787:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513079.442790:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513079.442791:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:22.0:1616513079.442792:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513079.442797:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.442800:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757cb0 00000400:00000200:18.0:1616513079.442800:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.72@tcp1 : PUT - for me 00000800:00000200:22.0:1616513079.442802:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.442809:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.72@tcp1 of length 224 into portal 12 MB=0x603ff7e8a9140 00000400:00000200:18.0:1616513079.442815:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.72@tcp1 of length 224/224 into md 0xf96450b9 [1] + 86808 00000400:00000200:18.0:1616513079.442820:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.442822:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.442829:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.442832:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.442834:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513079.442834:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245734817920 00000100:00000200:14.0:1616513079.442840:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245734818112 00010000:00000200:12.0:1616513079.442845:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1693245734817920/t0(0) o400->d2157a44-8f9f-536c-c1c7-64b3b4cdfcdd@10.31.132.72@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:14.0:1616513079.442847:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebb600 x1693245734818112/t0(0) o400->e829d8f2-67cb-83f7-6175-38fb71242a76@10.31.132.72@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.442858:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245734818112, offset 192 00000100:00000200:12.0:1616513079.442859:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245734817920, offset 192 00000400:00000200:14.0:1616513079.442865:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.72@tcp1 00000400:00000200:12.0:1616513079.442865:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.72@tcp1 00000400:00000200:14.0:1616513079.442871:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.72@tcp1 routed destination 00000400:00000200:12.0:1616513079.442873:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.72@tcp1 routed destination 00000400:00000200:14.0:1616513079.442888:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.72@tcp1(10.31.132.72@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513079.442889:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.72@tcp1(10.31.132.72@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.442894:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.442896:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.442899:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.442901:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.442902:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.442903:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.442904:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.442906:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.442906:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.442911:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.442916:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513079.442917:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513079.442925:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.442929:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.442931:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513079.442933:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513079.442936:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.442941:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241540 00000800:00000200:2.0:1616513079.442944:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.442952:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.443007:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.443014:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513079.443017:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513079.443024:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.443028:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47576d8 00000800:00000200:20.0:1616513079.443032:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.443035:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.455710:0:49382:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.455765:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.455775:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513079.455795:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.190@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.455804:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.190@tcp1 of length 224 into portal 26 MB=0x601869113c680 00000400:00000200:22.0:1616513079.455810:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.190@tcp1 of length 224/224 into md 0xf97c81a9 [1] + 0 00000400:00000200:22.0:1616513079.455815:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82e58 00000400:00000200:22.0:1616513079.455817:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.455820:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.455824:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82e58 00000800:00000200:22.0:1616513079.455827:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.455830:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.455832:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.455836:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513079.455886:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690527331501696 00010000:00000200:12.0:1616513079.455895:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1690527331501696/t0(0) o400->a7cfbcdc-36c3-689c-612a-23ac77c39896@10.31.141.190@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.455909:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690527331501696, offset 192 00000400:00000200:12.0:1616513079.455914:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.190@tcp1 00000400:00000200:12.0:1616513079.455921:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.190@tcp1 routed destination 00000400:00000200:12.0:1616513079.455935:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.190@tcp1(10.31.141.190@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.455941:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.455947:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.455949:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.455951:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.455955:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513079.455961:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513079.455964:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.455974:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.455977:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.455979:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.455983:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.455986:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241d38 00000800:00000200:8.0:1616513079.455989:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.455990:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.456324:0:49382:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513079.456386:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.456395:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513079.456408:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.190@tcp1 : PUT - for me 00000400:00000200:26.0:1616513079.456416:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.190@tcp1 of length 224 into portal 12 MB=0x601869113c940 00000400:00000200:26.0:1616513079.456423:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.190@tcp1 of length 224/224 into md 0xf96450b9 [1] + 87032 00000400:00000200:26.0:1616513079.456428:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.456431:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513079.456448:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.456450:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.456454:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.456455:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.456513:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690527331502400 00010000:00000200:14.0:1616513079.456524:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647df500 x1690527331502400/t0(0) o400->62d990dd-3f07-78c9-15f0-f56d5d74d10a@10.31.141.190@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.456538:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690527331502400, offset 192 00000400:00000200:14.0:1616513079.456544:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.190@tcp1 00000400:00000200:14.0:1616513079.456551:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.190@tcp1 routed destination 00000400:00000200:14.0:1616513079.456565:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.190@tcp1(10.31.141.190@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.456571:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.456576:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.456579:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.456581:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.456585:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.456590:0:49382:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.456600:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.456604:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.456606:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513079.456608:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.456611:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.456614:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47564c8 00000800:00000200:22.0:1616513079.456617:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.466866:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.466921:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.466931:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.466940:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.40@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.466952:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.40@tcp1 of length 224 into portal 26 MB=0x5ffd38cdc07c0 00000400:00000200:20.0:1616513079.466959:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.40@tcp1 of length 224/224 into md 0xf97c81c9 [1] + 0 00000400:00000200:20.0:1616513079.466964:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd836d8 00000400:00000200:20.0:1616513079.466967:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.466970:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513079.466974:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd836d8 00000800:00000200:28.0:1616513079.466975:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.466977:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.466980:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.466982:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.467038:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658949965760 00010000:00000200:12.0:1616513079.467048:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688658949965760/t0(0) o400->48d801c0-c3fe-8285-d668-cc0ff72cf8b8@10.31.138.40@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.467063:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658949965760, offset 192 00000400:00000200:12.0:1616513079.467071:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.40@tcp1 00000400:00000200:12.0:1616513079.467078:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.40@tcp1 routed destination 00000400:00000200:12.0:1616513079.467093:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.40@tcp1(10.31.138.40@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.467100:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.467106:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.467108:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.467110:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.467115:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.467121:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.467122:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513079.467131:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.467136:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.467138:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.467144:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.467148:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241298 00000800:00000200:8.0:1616513079.467151:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.467154:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.467616:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.467628:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.467636:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.467638:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.467645:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.40@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.467654:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.40@tcp1 of length 224 into portal 12 MB=0x5ffd38cdc08c0 00000400:00000200:28.0:1616513079.467660:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.40@tcp1 of length 224/224 into md 0x865bbec1 [1] + 20608 00000400:00000200:28.0:1616513079.467666:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.467668:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513079.467675:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.467678:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.467680:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.467739:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658949966016 00010000:00000200:0.0:1616513079.467752:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9f500 x1688658949966016/t0(0) o400->b3855186-3ebc-ee93-f7ce-574151ee8122@10.31.138.40@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.467766:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658949966016, offset 192 00000400:00000200:0.0:1616513079.467773:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.40@tcp1 00000400:00000200:0.0:1616513079.467780:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.40@tcp1 routed destination 00000400:00000200:0.0:1616513079.467795:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.40@tcp1(10.31.138.40@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.467802:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513079.467808:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.467811:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.467813:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.467818:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.467826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.467883:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.467891:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.467894:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.467900:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.467904:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a90 00000800:00000200:2.0:1616513079.467908:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.467911:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.472924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.472935:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.472940:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.472947:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.12@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.472953:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.12@tcp1 of length 224 into portal 26 MB=0x603d0b53f5a40 00000400:00000200:8.0:1616513079.472959:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.21.12@tcp1 of length 224/224 into md 0xf97c81f9 [1] + 0 00000400:00000200:8.0:1616513079.472962:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd827f8 00000400:00000200:8.0:1616513079.472965:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.472967:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513079.472970:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd827f8 00000800:00000200:8.0:1616513079.472972:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.472974:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.472976:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.472977:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.472982:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513079.472986:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513079.472987:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.12@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.472992:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.12@tcp1 of length 224 into portal 12 MB=0x603d0b53f5dc0 00000400:00000200:8.0:1616513079.472996:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.21.12@tcp1 of length 224/224 into md 0x865bbec1 [1] + 20832 00000400:00000200:8.0:1616513079.472999:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.473000:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513079.473004:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.473006:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.473007:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.473031:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693044789172800 00010000:00000200:12.0:1616513079.473041:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1693044789172800/t0(0) o400->9f1f4653-39ac-3ab5-4000-6f772afb1757@10.31.21.12@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.473054:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693044789172800, offset 192 00000400:00000200:12.0:1616513079.473060:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.12@tcp1 00000400:00000200:12.0:1616513079.473067:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.12@tcp1 routed destination 00000100:00000200:0.0:1616513079.473074:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693044789173696 00000400:00000200:12.0:1616513079.473080:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.12@tcp1(10.31.21.12@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513079.473086:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4380 x1693044789173696/t0(0) o400->48c1cb4f-762b-9ca6-0d15-05214f46b3a2@10.31.21.12@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513079.473087:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.473092:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.473094:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.473096:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513079.473099:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693044789173696, offset 192 00000800:00000200:12.0:1616513079.473101:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513079.473105:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.12@tcp1 00000800:00000200:11.2:1616513079.473106:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.473107:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513079.473112:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.12@tcp1 routed destination 00000400:00000200:0.0:1616513079.473125:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.12@tcp1(10.31.21.12@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.473131:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513079.473136:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.473140:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.473142:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.473147:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.473152:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513079.473160:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513079.473165:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513079.473167:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.473170:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.473171:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513079.473171:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.473173:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.473177:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.473180:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.473182:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:28.0:1616513079.473186:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513079.473186:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000800:00000200:18.0:1616513079.473190:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.473193:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:31.0:1616513079.482545:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925568, portal 4 00000100:00000200:19.0:1616513079.482545:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925552, portal 4 00000100:00000200:31.0:1616513079.482551:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925568, offset 0 00000100:00000200:19.0:1616513079.482551:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925552, offset 0 00000400:00000200:31.0:1616513079.482558:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:31.0:1616513079.482564:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:19.0:1616513079.482564:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:19.0:1616513079.482572:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:31.0:1616513079.482579:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:19.0:1616513079.482585:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513079.482586:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:19.0:1616513079.482592:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:31.0:1616513079.482594:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513079.482597:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:31.0:1616513079.482599:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:19.0:1616513079.482599:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513079.482602:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:19.0:1616513079.482604:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513079.482605:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:19.0:1616513079.482609:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:22.0:1616513079.482633:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925584, portal 4 00000100:00000200:22.0:1616513079.482639:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925584, offset 0 00000400:00000200:22.0:1616513079.482646:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:25.2:1616513079.482652:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000400:00000200:22.0:1616513079.482652:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:22.0:1616513079.482665:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513079.482668:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513079.482671:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.2:1616513079.482672:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:2.0:1616513079.482675:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000800:00000200:22.0:1616513079.482678:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513079.482678:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513079.482680:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513079.482681:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:22.0:1616513079.482683:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:2.0:1616513079.482685:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:22.0:1616513079.482689:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (24)-- 00000800:00000200:27.0:1616513079.482691:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000100:00000200:2.0:1616513079.482692:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456eec00 x1693067725925568/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513079.482697:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:27.0:1616513079.482700:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513079.482702:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:2.0:1616513079.482704:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c22a8 00000400:00000200:27.0:1616513079.482707:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:2.0:1616513079.482708:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:31.0:1616513079.482709:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:31.0:1616513079.482712:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:27.0:1616513079.482713:0:6476:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63da00 x1693067725925552/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513079.482718:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:31.0:1616513079.482724:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bd9b00 x1693067725925584/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513079.482738:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707320 00000800:00000200:27.0:1616513079.482742:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513079.482745:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:31.0:1616513079.482747:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:31.0:1616513079.482750:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:25.2:1616513079.482828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:27.2:1616513079.482845:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513079.482855:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.482860:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513079.482863:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513079.482868:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:31.0:1616513079.482869:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:3.0:1616513079.482873:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:31.0:1616513079.482876:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513079.482876:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626cb0 00000800:00000200:18.0:1616513079.482882:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000400:00000200:27.0:1616513079.482883:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166b0d [1] + 192 00000400:00000200:31.0:1616513079.482885:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626cd0 00000400:00000200:27.0:1616513079.482888:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:31.0:1616513079.482891:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166b3d [1] + 192 00000800:00000200:18.0:1616513079.482891:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513079.482893:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:31.0:1616513079.482897:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513079.482898:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63da00 x1693067725925552/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513079.482900:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513079.482908:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626cc0 00000800:00000200:27.0:1616513079.482912:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000100:00000200:31.0:1616513079.482913:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bd9b00 x1693067725925584/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513079.482915:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:18.0:1616513079.482915:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c8879 [1] + 192 00000800:00000200:27.0:1616513079.482917:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:18.0:1616513079.482920:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.482925:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456eec00 x1693067725925568/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513079.482927:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513079.482929:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:19.0:1616513079.482930:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63da00 x1693067725925552/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513079.482931:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:18.0:1616513079.482940:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000100:00000200:19.0:1616513079.482943:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63da00 x1693067725925552/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513079.482943:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513079.482945:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:31.0:1616513079.482951:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456eec00 x1693067725925568/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513079.482953:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000100:00000200:22.0:1616513079.482961:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bd9b00 x1693067725925584/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513079.482962:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456eec00 x1693067725925568/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513079.482972:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000100:00000200:22.0:1616513079.482975:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bd9b00 x1693067725925584/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513079.482985:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b430 00000800:00000200:11.2:1616513079.485265:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.485276:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.485282:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.485289:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.179@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.485296:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.179@tcp1 of length 224 into portal 26 MB=0x5ffd3d752fa40 00000400:00000200:8.0:1616513079.485302:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.179@tcp1 of length 224/224 into md 0xf97c8209 [1] + 0 00000400:00000200:8.0:1616513079.485306:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd83650 00000400:00000200:8.0:1616513079.485307:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.485310:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513079.485314:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd83650 00000800:00000200:8.0:1616513079.485316:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.485318:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.485319:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.485322:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.485377:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660199275072 00010000:00000200:12.0:1616513079.485388:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688660199275072/t0(0) o400->439b94f2-2e79-f010-3b58-f7979d8827ae@10.31.129.179@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.485402:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660199275072, offset 192 00000400:00000200:12.0:1616513079.485408:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.179@tcp1 00000400:00000200:12.0:1616513079.485415:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.179@tcp1 routed destination 00000400:00000200:12.0:1616513079.485430:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.179@tcp1(10.31.129.179@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.485450:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.485457:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.485460:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.485462:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.485467:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.485473:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.485474:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513079.485527:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.485534:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.485537:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.485544:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.485549:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240770 00000800:00000200:24.0:1616513079.485552:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.485555:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.485561:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513079.485562:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.485569:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.179@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.485577:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.179@tcp1 of length 224 into portal 12 MB=0x5ffd3d752fe40 00000400:00000200:24.0:1616513079.485583:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.179@tcp1 of length 224/224 into md 0x865bbec1 [1] + 21056 00000400:00000200:24.0:1616513079.485588:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.485590:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513079.485596:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.485600:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.485602:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.485661:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660199276096 00010000:00000200:0.0:1616513079.485674:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc5a00 x1688660199276096/t0(0) o400->a1ab9212-b52c-d07e-7dc1-ed9434e0d934@10.31.129.179@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.485688:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660199276096, offset 192 00000400:00000200:0.0:1616513079.485694:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.179@tcp1 00000400:00000200:0.0:1616513079.485702:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.179@tcp1 routed destination 00000400:00000200:0.0:1616513079.485717:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.179@tcp1(10.31.129.179@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.485723:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513079.485729:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.485732:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.485734:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.485739:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.485745:0:49385:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.485805:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.485811:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.485814:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.485821:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.485825:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04c38 00000800:00000200:26.0:1616513079.485828:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.485832:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.485837:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513079.485840:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.485842:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.485843:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.485845:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.489474:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.489528:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.489538:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513079.489539:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513079.489546:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.65@tcp1 : PUT - for me 00000800:00000200:8.0:1616513079.489546:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.489553:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.65@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.489556:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.65@tcp1 of length 224 into portal 26 MB=0x5ffd453dcbf80 00000400:00000200:8.0:1616513079.489561:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.65@tcp1 of length 224 into portal 12 MB=0x5ffd453dcc240 00000400:00000200:20.0:1616513079.489562:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.65@tcp1 of length 224/224 into md 0xf97c8229 [1] + 0 00000400:00000200:20.0:1616513079.489567:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82220 00000400:00000200:8.0:1616513079.489567:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.65@tcp1 of length 224/224 into md 0x865bbec1 [1] + 21280 00000400:00000200:20.0:1616513079.489570:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.489571:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.489572:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:8.0:1616513079.489573:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513079.489577:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82220 00000800:00000200:8.0:1616513079.489578:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513079.489580:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:8.0:1616513079.489581:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513079.489582:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513079.489583:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513079.489585:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.489587:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.489640:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662288678784 00010000:00000200:12.0:1616513079.489650:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688662288678784/t0(0) o400->d917ea2f-d783-b4ca-7e0f-f8cb4f896cc8@10.31.138.65@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513079.489664:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662288679488 00000100:00000200:12.0:1616513079.489665:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662288678784, offset 192 00000400:00000200:12.0:1616513079.489671:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.65@tcp1 00010000:00000200:0.0:1616513079.489676:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0480 x1688662288679488/t0(0) o400->bf05676d-8cab-ece5-9bc9-293e682b25e8@10.31.138.65@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513079.489678:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.65@tcp1 routed destination 00000100:00000200:0.0:1616513079.489691:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662288679488, offset 192 00000400:00000200:12.0:1616513079.489693:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.65@tcp1(10.31.138.65@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513079.489697:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.65@tcp1 00000800:00000200:12.0:1616513079.489700:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513079.489705:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.65@tcp1 routed destination 00000800:00000200:12.0:1616513079.489713:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.489715:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.489717:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513079.489720:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.65@tcp1(10.31.138.65@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.489722:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.489726:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.489727:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:12.0:1616513079.489729:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513079.489733:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.489735:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.489738:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.489742:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.489749:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513079.489781:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.489788:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.489791:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513079.489791:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.489797:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.489803:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241298 00000800:00000200:24.0:1616513079.489806:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513079.489808:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.489814:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.489817:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.489824:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.489830:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb044c8 00000800:00000200:18.0:1616513079.489834:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.489837:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.499315:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.499370:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.499379:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.499388:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.217@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.499397:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.217@tcp1 of length 224 into portal 26 MB=0x6031cf1d5e540 00000400:00000200:2.0:1616513079.499404:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.217@tcp1 of length 224/224 into md 0xf97c8249 [1] + 0 00000400:00000200:2.0:1616513079.499408:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbd82330 00000400:00000200:2.0:1616513079.499411:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.499414:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.499418:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbd82330 00000800:00000200:2.0:1616513079.499421:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.499424:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513079.499424:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.499431:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.499462:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.499465:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513079.499473:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.217@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.499482:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.217@tcp1 of length 224 into portal 12 MB=0x6031cf1d5e700 00000400:00000200:22.0:1616513079.499488:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.217@tcp1 of length 224/224 into md 0xf96450b9 [1] + 87256 00000400:00000200:22.0:1616513079.499493:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.499495:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:12.0:1616513079.499497:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692272711558464 00000800:00000200:22.0:1616513079.499503:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.499506:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.499507:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:12.0:1616513079.499507:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1692272711558464/t0(0) o400->688f4fa8-885a-d473-0345-90655b30c021@10.31.141.217@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.499521:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692272711558464, offset 192 00000400:00000200:12.0:1616513079.499527:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.217@tcp1 00000400:00000200:12.0:1616513079.499533:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.217@tcp1 routed destination 00000400:00000200:12.0:1616513079.499547:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.217@tcp1(10.31.141.217@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.499553:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.499558:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.499561:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.499563:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.499567:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.499573:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.499575:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:14.0:1616513079.499576:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692272711558912 00010000:00000200:14.0:1616513079.499587:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec202880 x1692272711558912/t0(0) o400->82dd272c-a1d2-4874-05b9-cdc21ed5ba8d@10.31.141.217@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.499601:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692272711558912, offset 192 00000400:00000200:14.0:1616513079.499607:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.217@tcp1 00000400:00000200:14.0:1616513079.499614:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.217@tcp1 routed destination 00000400:00000200:14.0:1616513079.499627:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.217@tcp1(10.31.141.217@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.499633:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.499639:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.499641:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513079.499643:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.499645:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.499648:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513079.499649:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513079.499651:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513079.499655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513079.499657:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513079.499663:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241298 00000800:00000200:26.0:1616513079.499666:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513079.499666:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.499669:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513079.499673:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.499676:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.499677:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513079.499682:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.499688:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47566e8 00000800:00000200:26.0:1616513079.499692:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.508768:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.508780:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.508787:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.508804:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.210@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.508812:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.210@tcp1 of length 224 into portal 26 MB=0x5ffd3656a9a00 00000400:00000200:8.0:1616513079.508817:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.210@tcp1 of length 224/224 into md 0xf97c8259 [1] + 0 00000400:00000200:8.0:1616513079.508821:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c4a0b320 00000400:00000200:8.0:1616513079.508823:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.508825:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513079.508829:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b320 00000800:00000200:8.0:1616513079.508831:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.508833:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.508834:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.508836:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.508858:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:12.0:1616513079.508890:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658288220672 00010000:00000200:12.0:1616513079.508900:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688658288220672/t0(0) o400->bf93aec2-85b4-94cf-dba4-aec0ced8358a@10.31.136.210@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513079.508910:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.508914:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658288220672, offset 192 00000800:00000200:20.0:1616513079.508919:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513079.508920:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.210@tcp1 00000400:00000200:20.0:1616513079.508927:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.210@tcp1 : PUT - for me 00000400:00000200:12.0:1616513079.508927:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.210@tcp1 routed destination 00000400:00000200:12.0:1616513079.508941:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.210@tcp1(10.31.136.210@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.508947:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.508950:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.210@tcp1 of length 224 into portal 12 MB=0x5ffd3656a9a80 00000800:00000200:12.0:1616513079.508952:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.508955:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.508957:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:20.0:1616513079.508958:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.210@tcp1 of length 224/224 into md 0x865bbec1 [1] + 21504 00000400:00000200:20.0:1616513079.508962:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513079.508962:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:20.0:1616513079.508965:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513079.508966:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:12.0:1616513079.508967:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513079.508971:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:20.0:1616513079.508975:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:28.0:1616513079.508976:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:20.0:1616513079.508976:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513079.508978:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513079.508979:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.508987:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.508992:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241298 00000800:00000200:28.0:1616513079.508996:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.509036:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658288220800 00010000:00000200:0.0:1616513079.509048:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b100000 x1688658288220800/t0(0) o400->d6e87706-6e01-c7f5-50cc-647f71f3da68@10.31.136.210@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.509073:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658288220800, offset 192 00000400:00000200:0.0:1616513079.509079:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.210@tcp1 00000400:00000200:0.0:1616513079.509087:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.210@tcp1 routed destination 00000400:00000200:0.0:1616513079.509102:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.210@tcp1(10.31.136.210@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.509108:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513079.509114:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.509117:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.509119:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.509124:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.509131:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.509184:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.509190:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.509193:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.509199:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.509203:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb058f8 00000800:00000200:18.0:1616513079.509206:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.509209:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.512534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.512587:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.512596:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.512605:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.217@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.512615:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.217@tcp1 of length 224 into portal 26 MB=0x5ffede7594140 00000400:00000200:24.0:1616513079.512621:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.217@tcp1 of length 224/224 into md 0xf97c8289 [1] + 0 00000400:00000200:24.0:1616513079.512627:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c4a0ba08 00000400:00000200:24.0:1616513079.512629:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.512632:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.512636:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0ba08 00000800:00000200:24.0:1616513079.512640:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.512641:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.512642:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.512648:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.512651:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.512652:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.512660:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.217@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.512669:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.217@tcp1 of length 224 into portal 12 MB=0x5ffede75941c0 00000400:00000200:20.0:1616513079.512675:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.217@tcp1 of length 224/224 into md 0x865bbec1 [1] + 21728 00000400:00000200:20.0:1616513079.512681:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.512683:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513079.512690:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.512693:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.512695:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.512700:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688772137271616 00010000:00000200:12.0:1616513079.512710:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688772137271616/t0(0) o400->48a889a5-6177-aba9-efa1-a1f2f82f3ae0@10.31.136.217@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.512725:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688772137271616, offset 192 00000400:00000200:12.0:1616513079.512731:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.217@tcp1 00000400:00000200:12.0:1616513079.512738:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.217@tcp1 routed destination 00000400:00000200:12.0:1616513079.512753:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.217@tcp1(10.31.136.217@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513079.512755:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688772137271744 00000800:00000200:12.0:1616513079.512760:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.512765:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513079.512767:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b107080 x1688772137271744/t0(0) o400->8acfe3ab-9e31-ea44-84c1-43439ec9b148@10.31.136.217@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513079.512768:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.512770:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.512774:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.512779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.512781:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513079.512782:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688772137271744, offset 192 00000400:00000200:0.0:1616513079.512788:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.217@tcp1 00000400:00000200:0.0:1616513079.512795:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.217@tcp1 routed destination 00000400:00000200:0.0:1616513079.512810:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.217@tcp1(10.31.136.217@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.512816:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513079.512822:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.512825:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.512827:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.512832:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513079.512833:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513079.512838:0:49388:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513079.512841:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.512843:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513079.512843:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.512850:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.512856:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f650 00000800:00000200:28.0:1616513079.512859:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513079.512895:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.512902:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.512905:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.512912:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.512916:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05cb0 00000800:00000200:2.0:1616513079.512920:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.512923:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.519074:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513079.519093:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513079.519128:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.519137:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.519146:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.211@tcp1 : PUT - for me 00000800:00000200:18.0:1616513079.519150:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513079.519156:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.211@tcp1 of length 224 into portal 26 MB=0x5ffdef5bf6380 00000800:00000200:18.0:1616513079.519158:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513079.519163:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.211@tcp1 of length 224/224 into md 0xf97c8299 [1] + 0 00000400:00000200:18.0:1616513079.519167:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.84@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.519168:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c4a0a660 00000400:00000200:24.0:1616513079.519170:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.519173:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513079.519176:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.84@tcp1 of length 224 into portal 26 MB=0x5ffdfbea89440 00000400:00000200:24.0:1616513079.519178:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0a660 00000800:00000200:20.0:1616513079.519180:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.519181:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513079.519182:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.84@tcp1 of length 224/224 into md 0xf97c82c9 [1] + 0 00000800:00000200:24.0:1616513079.519183:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.519185:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513079.519188:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1706f68 00000400:00000200:18.0:1616513079.519190:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.519193:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513079.519198:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706f68 00000800:00000200:11.2:1616513079.519200:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513079.519201:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.519203:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.519204:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:8.0:1616513079.519208:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513079.519209:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513079.519213:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513079.519216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513079.519219:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.211@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.519226:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.211@tcp1 of length 224 into portal 12 MB=0x5ffdef5bf6740 00000400:00000200:8.0:1616513079.519230:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.211@tcp1 of length 224/224 into md 0x865bbec1 [1] + 21952 00000400:00000200:8.0:1616513079.519233:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.519235:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513079.519240:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.519242:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.519243:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.519245:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513079.519246:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707954336640 00000100:00000200:20.0:1616513079.519254:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711325062208 00010000:00000200:12.0:1616513079.519256:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688707954336640/t0(0) o400->e4381175-3871-b5b4-7aec-c3d9f31fbbc5@10.31.128.211@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:20.0:1616513079.519264:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688711325062208/t0(0) o400->fc796536-b7b7-6a80-e863-b83341bce853@10.31.128.84@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513079.519269:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688707954336640, offset 192 00000400:00000200:12.0:1616513079.519277:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.211@tcp1 00000100:00000200:20.0:1616513079.519280:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688711325062208, offset 192 00000400:00000200:12.0:1616513079.519284:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.211@tcp1 routed destination 00000400:00000200:20.0:1616513079.519285:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.84@tcp1 00000400:00000200:12.0:1616513079.519298:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.211@tcp1(10.31.128.211@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513079.519298:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.519304:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.519306:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.84@tcp1 routed destination 00000800:00000200:2.0:1616513079.519307:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:0.0:1616513079.519311:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707954337600 00000800:00000200:12.0:1616513079.519314:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513079.519316:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513079.519316:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.84@tcp1 : PUT - for me 00000800:00000200:12.0:1616513079.519318:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513079.519321:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.84@tcp1(10.31.128.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513079.519322:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6c00 x1688707954337600/t0(0) o400->ad698cd8-9707-8b6e-70c4-f24695a1e444@10.31.128.211@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513079.519323:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.519327:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513079.519328:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513079.519329:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513079.519330:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.84@tcp1 of length 224 into portal 12 MB=0x5ffdfbea894c0 00000800:00000200:20.0:1616513079.519333:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.519335:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:0.0:1616513079.519336:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688707954337600, offset 192 00000400:00000200:2.0:1616513079.519337:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.84@tcp1 of length 224/224 into md 0xf96450b9 [1] + 87480 00000800:00000200:20.0:1616513079.519341:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:0.0:1616513079.519342:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.211@tcp1 00000400:00000200:2.0:1616513079.519343:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513079.519345:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:2.0:1616513079.519345:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513079.519349:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.211@tcp1 routed destination 00000800:00000200:26.0:1616513079.519351:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513079.519352:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000100:00000200:20.0:1616513079.519354:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513079.519355:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:2.0:1616513079.519358:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513079.519359:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513079.519361:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.519362:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513079.519362:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513079.519363:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.211@tcp1(10.31.128.211@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513079.519369:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513079.519370:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513079.519371:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513079.519372:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.519374:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23210 00000800:00000200:24.0:1616513079.519374:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513079.519376:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513079.519377:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513079.519378:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513079.519379:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513079.519381:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:0.0:1616513079.519381:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.519383:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513079.519385:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.519390:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.519397:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.519401:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.519402:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.519406:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513079.519407:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.519410:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05540 00000800:00000200:8.0:1616513079.519412:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513079.519417:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711325062336 00010000:00000200:14.0:1616513079.519429:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621f80 x1688711325062336/t0(0) o400->15c32944-cd9f-492b-d836-6aa1e0e5bbbf@10.31.128.84@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.519484:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688711325062336, offset 192 00000400:00000200:14.0:1616513079.519491:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.84@tcp1 00000400:00000200:14.0:1616513079.519498:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.84@tcp1 routed destination 00000400:00000200:14.0:1616513079.519519:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.84@tcp1(10.31.128.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.519525:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.519531:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.519534:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.519536:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.519541:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.519545:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.519554:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.519558:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.519560:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.519563:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.519566:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.519568:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757e48 00000800:00000200:18.0:1616513079.519571:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.522316:0:49389:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.522371:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.522380:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513079.522390:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.62@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.522399:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.62@tcp1 of length 224 into portal 26 MB=0x5ffd3fff5f3c0 00000400:00000200:22.0:1616513079.522405:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.62@tcp1 of length 224/224 into md 0xf97c82d9 [1] + 0 00000400:00000200:22.0:1616513079.522410:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241c28 00000400:00000200:22.0:1616513079.522413:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.522416:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.522420:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241c28 00000800:00000200:22.0:1616513079.522423:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.522426:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.522426:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.522430:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.522481:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660881044416 00010000:00000200:20.0:1616513079.522491:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688660881044416/t0(0) o400->c9c7ccd7-a94e-5d59-7cc1-d47ec700da8b@10.31.128.62@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.522505:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660881044416, offset 192 00000400:00000200:20.0:1616513079.522511:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.62@tcp1 00000400:00000200:20.0:1616513079.522518:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.62@tcp1 routed destination 00000400:00000200:20.0:1616513079.522531:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.62@tcp1(10.31.128.62@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.522537:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.522543:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.522545:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.522547:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.522552:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.522557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513079.522558:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.522564:0:49389:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513079.522610:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.522617:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.522619:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513079.522620:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.522625:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.522631:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22b28 00000800:00000200:2.0:1616513079.522631:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513079.522634:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513079.522640:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.522649:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.62@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.522658:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.62@tcp1 of length 224 into portal 12 MB=0x5ffd3fff5f800 00000400:00000200:2.0:1616513079.522664:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.62@tcp1 of length 224/224 into md 0xf96450b9 [1] + 87704 00000400:00000200:2.0:1616513079.522669:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.522672:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513079.522679:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.522682:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.522684:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.522687:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.522747:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660881045504 00010000:00000200:14.0:1616513079.522759:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21de80 x1688660881045504/t0(0) o400->d08c5725-b78c-414a-77dd-9d432c319636@10.31.128.62@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.522774:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660881045504, offset 192 00000400:00000200:14.0:1616513079.522781:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.62@tcp1 00000400:00000200:14.0:1616513079.522788:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.62@tcp1 routed destination 00000400:00000200:14.0:1616513079.522803:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.62@tcp1(10.31.128.62@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.522809:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.522815:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.522818:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.522820:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.522825:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.522829:0:49389:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.522840:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.522845:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.522847:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.522849:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513079.522854:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.522858:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757210 00000800:00000200:26.0:1616513079.522862:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.524647:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.524702:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.524713:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.524722:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.83@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.524732:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.83@tcp1 of length 224 into portal 26 MB=0x6051ff877ee40 00000400:00000200:2.0:1616513079.524738:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.83@tcp1 of length 224/224 into md 0xf97c82e9 [1] + 0 00000400:00000200:2.0:1616513079.524742:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240a18 00000400:00000200:2.0:1616513079.524745:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.524748:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.524752:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240a18 00000800:00000200:2.0:1616513079.524755:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.524756:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.524758:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.524760:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.524813:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694484730998336 00000800:00000200:7.2:1616513079.524816:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:20.0:1616513079.524823:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1694484730998336/t0(0) o400->176c12c1-f09e-f89b-9f65-919419ef4d6c@10.31.133.83@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.524836:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694484730998336, offset 192 00000400:00000200:20.0:1616513079.524842:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.83@tcp1 00000400:00000200:20.0:1616513079.524849:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.83@tcp1 routed destination 00000400:00000200:20.0:1616513079.524863:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.83@tcp1(10.31.133.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.524869:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.524874:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.524877:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.524879:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513079.524881:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.524885:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513079.524890:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513079.524890:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513079.524891:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513079.524898:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.83@tcp1 : PUT - for me 00000800:00000200:28.0:1616513079.524906:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513079.524907:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.83@tcp1 of length 224 into portal 12 MB=0x6051ff877ef40 00000800:00000200:28.0:1616513079.524914:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513079.524914:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.83@tcp1 of length 224/224 into md 0xf96450b9 [1] + 87928 00000400:00000200:28.0:1616513079.524917:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513079.524919:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513079.524919:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513079.524922:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513079.524923:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513079.524928:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513079.524929:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23ed0 00000800:00000200:22.0:1616513079.524931:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513079.524932:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513079.524933:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.524937:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.524945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:14.0:1616513079.524952:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694484730998592 00000800:00000200:20.0:1616513079.524956:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.524961:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.524964:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:14.0:1616513079.524964:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec204800 x1694484730998592/t0(0) o400->6e38ca03-4512-0d68-b36b-63296f99c46f@10.31.133.83@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513079.524965:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513079.524967:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513079.524969:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513079.524990:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694484730998592, offset 192 00000400:00000200:14.0:1616513079.524996:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.83@tcp1 00000400:00000200:14.0:1616513079.525004:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.83@tcp1 routed destination 00000400:00000200:14.0:1616513079.525025:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.83@tcp1(10.31.133.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.525032:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.525038:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.525040:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.525043:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.525047:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.525052:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.525061:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.525066:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.525068:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.525070:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.525074:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.525078:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47565d8 00000800:00000200:18.0:1616513079.525083:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.531993:0:49390:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513079.532048:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.532058:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513079.532067:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.20@tcp1 : PUT - for me 00000400:00000200:26.0:1616513079.532076:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.20@tcp1 of length 224 into portal 26 MB=0x60006b45f1ec0 00000400:00000200:26.0:1616513079.532083:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.20@tcp1 of length 224/224 into md 0xf97c8309 [1] + 0 00000400:00000200:26.0:1616513079.532087:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241f58 00000400:00000200:26.0:1616513079.532090:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.532093:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513079.532097:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241f58 00000800:00000200:26.0:1616513079.532100:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.532101:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.532103:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.532105:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.532160:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688878656200384 00010000:00000200:20.0:1616513079.532171:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688878656200384/t0(0) o400->01e7868f-cb14-179b-1fbf-1550e9021f3f@10.31.141.20@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513079.532183:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513079.532185:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688878656200384, offset 192 00000400:00000200:20.0:1616513079.532192:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.20@tcp1 00000400:00000200:20.0:1616513079.532199:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.20@tcp1 routed destination 00000800:00000200:12.0:1616513079.532206:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513079.532214:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.20@tcp1(10.31.141.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513079.532216:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.532221:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513079.532224:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.20@tcp1 : PUT - for me 00000800:00000200:20.0:1616513079.532227:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.532229:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513079.532231:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:12.0:1616513079.532233:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.20@tcp1 of length 224 into portal 12 MB=0x60006b45f22c0 00000800:00000200:20.0:1616513079.532236:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:12.0:1616513079.532239:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.20@tcp1 of length 224/224 into md 0x865bbec1 [1] + 22176 00000100:00000200:20.0:1616513079.532243:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513079.532245:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513079.532247:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513079.532254:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513079.532257:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513079.532259:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.532262:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.532271:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.532273:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.532280:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.532286:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23430 00000800:00000200:28.0:1616513079.532289:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.532292:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.532320:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688878656201408 00010000:00000200:0.0:1616513079.532333:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c664800 x1688878656201408/t0(0) o400->6bf74ce9-ce0d-f2ea-50f3-8c8cde9683dd@10.31.141.20@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.532347:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688878656201408, offset 192 00000400:00000200:0.0:1616513079.532353:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.20@tcp1 00000400:00000200:0.0:1616513079.532361:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.20@tcp1 routed destination 00000400:00000200:0.0:1616513079.532376:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.20@tcp1(10.31.141.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.532382:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513079.532388:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.532391:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.532393:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.532398:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.532403:0:49390:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.532458:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.532465:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.532468:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.532475:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.532479:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04aa0 00000800:00000200:2.0:1616513079.532483:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.532486:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.560936:0:49393:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.560994:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.561004:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513079.561013:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.199@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.561023:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.199@tcp1 of length 224 into portal 26 MB=0x605a073a27880 00000400:00000200:22.0:1616513079.561030:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.199@tcp1 of length 224/224 into md 0xf97c8319 [1] + 0 00000400:00000200:22.0:1616513079.561035:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782415c8 00000400:00000200:22.0:1616513079.561038:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.561042:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.561047:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782415c8 00000800:00000200:26.0:1616513079.561049:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.561050:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.561053:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.561055:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.561086:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513079.561115:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036553197696 00010000:00000200:20.0:1616513079.561126:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1695036553197696/t0(0) o400->639b4b3c-f944-7cf0-977c-285846063c0b@10.31.129.199@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.561141:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036553197696, offset 192 00000800:00000200:2.0:1616513079.561141:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513079.561148:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.199@tcp1 00000800:00000200:2.0:1616513079.561151:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513079.561156:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.199@tcp1 routed destination 00000800:00000200:18.0:1616513079.561157:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513079.561160:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.199@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.561169:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.199@tcp1 of length 224 into portal 12 MB=0x605a073a27900 00000400:00000200:20.0:1616513079.561172:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.199@tcp1(10.31.129.199@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513079.561176:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.199@tcp1 of length 224/224 into md 0xf96450b9 [1] + 88152 00000800:00000200:20.0:1616513079.561179:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513079.561182:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513079.561184:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:2.0:1616513079.561184:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513079.561187:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.561191:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513079.561192:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.561195:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513079.561197:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513079.561197:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.561204:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513079.561216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.561230:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.561244:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.561247:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.561253:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.561259:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23430 00000100:00000200:14.0:1616513079.561259:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036553197824 00000800:00000200:24.0:1616513079.561262:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.561265:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:14.0:1616513079.561271:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21c800 x1695036553197824/t0(0) o400->96fb4064-57fa-fe0b-afa7-3c5e39440e7d@10.31.129.199@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.561286:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036553197824, offset 192 00000400:00000200:14.0:1616513079.561292:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.199@tcp1 00000400:00000200:14.0:1616513079.561300:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.199@tcp1 routed destination 00000400:00000200:14.0:1616513079.561314:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.199@tcp1(10.31.129.199@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.561321:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.561327:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.561330:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.561332:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.561337:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.561342:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.561352:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.561358:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.561360:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.561362:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513079.561367:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.561371:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757980 00000800:00000200:26.0:1616513079.561377:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.562060:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513079.562071:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.562079:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513079.562081:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.562088:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.19@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.562097:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.19@tcp1 of length 224 into portal 26 MB=0x5ffe456d5ab80 00000800:00000200:7.2:1616513079.562097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513079.562104:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.19@tcp1 of length 224/224 into md 0xf97c8329 [1] + 0 00000800:00000200:22.0:1616513079.562107:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513079.562108:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240550 00000400:00000200:18.0:1616513079.562111:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.562112:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513079.562114:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:18.0:1616513079.562114:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513079.562118:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240550 00000400:00000200:22.0:1616513079.562119:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.19@tcp1 : PUT - for me 00000800:00000200:18.0:1616513079.562121:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.562124:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.562125:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.562126:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.19@tcp1 of length 224 into portal 12 MB=0x5ffe456d5ad80 00000400:00000200:22.0:1616513079.562131:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.19@tcp1 of length 224/224 into md 0xf96450b9 [1] + 88376 00000100:00000200:20.0:1616513079.562131:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731058023296 00000400:00000200:22.0:1616513079.562136:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.562138:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:20.0:1616513079.562138:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688731058023296/t0(0) o400->f9380b0b-eec0-92a7-e136-0ea44edc9e26@10.31.140.19@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513079.562143:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.562145:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.562147:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.562149:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688731058023296, offset 192 00000400:00000200:20.0:1616513079.562154:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.19@tcp1 00000100:00000200:14.0:1616513079.562156:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731058023808 00000400:00000200:20.0:1616513079.562160:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.19@tcp1 routed destination 00010000:00000200:14.0:1616513079.562163:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec203f00 x1688731058023808/t0(0) o400->de961de4-868d-b45f-ec4c-e63e24105562@10.31.140.19@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513079.562173:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.19@tcp1(10.31.140.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:14.0:1616513079.562174:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688731058023808, offset 192 00000800:00000200:20.0:1616513079.562179:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513079.562180:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.19@tcp1 00000800:00000200:20.0:1616513079.562184:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.562186:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513079.562186:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.19@tcp1 routed destination 00000800:00000200:20.0:1616513079.562188:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.562191:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513079.562196:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513079.562197:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513079.562199:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.19@tcp1(10.31.140.19@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.562207:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.562211:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513079.562212:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513079.562214:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.562216:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513079.562219:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513079.562219:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513079.562221:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.562222:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513079.562224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513079.562227:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.562232:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22b28 00000800:00000200:2.0:1616513079.562233:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513079.562236:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513079.562237:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.562239:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513079.562240:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513079.562246:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.562249:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756f68 00000800:00000200:2.0:1616513079.562253:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.570576:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.570630:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.570640:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.570649:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.106@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.570658:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.106@tcp1 of length 224 into portal 26 MB=0x5ffdba66d49c0 00000400:00000200:24.0:1616513079.570665:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.106@tcp1 of length 224/224 into md 0xf97c8339 [1] + 0 00000400:00000200:24.0:1616513079.570670:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240110 00000400:00000200:24.0:1616513079.570672:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.570675:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.570679:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240110 00000800:00000200:24.0:1616513079.570682:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.570684:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.570685:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.570688:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.570694:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513079.570744:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693738654144 00000800:00000200:18.0:1616513079.570753:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:20.0:1616513079.570755:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688693738654144/t0(0) o400->d1201f80-8ede-1968-c17f-1dea639b45d1@10.31.136.106@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513079.570762:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:20.0:1616513079.570770:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693738654144, offset 192 00000400:00000200:18.0:1616513079.570771:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.106@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.570776:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.106@tcp1 00000400:00000200:18.0:1616513079.570781:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.106@tcp1 of length 224 into portal 12 MB=0x5ffdba66d4ac0 00000400:00000200:20.0:1616513079.570784:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.106@tcp1 routed destination 00000400:00000200:18.0:1616513079.570787:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.106@tcp1 of length 224/224 into md 0xf96450b9 [1] + 88600 00000400:00000200:18.0:1616513079.570792:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.570795:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513079.570799:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.106@tcp1(10.31.136.106@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513079.570801:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.570804:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513079.570806:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513079.570806:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.570809:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513079.570811:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.570814:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.570816:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.570821:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:14.0:1616513079.570826:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693738654400 00000800:00000200:11.2:1616513079.570826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513079.570828:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513079.570834:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.570838:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:14.0:1616513079.570839:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebf080 x1688693738654400/t0(0) o400->a9b8af43-aa56-2a5b-1fd5-7e952c50c7e1@10.31.136.106@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513079.570840:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.570846:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.570850:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23a08 00000800:00000200:24.0:1616513079.570853:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:14.0:1616513079.570853:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693738654400, offset 192 00000800:00000200:24.0:1616513079.570855:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513079.570859:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.106@tcp1 00000400:00000200:14.0:1616513079.570867:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.106@tcp1 routed destination 00000400:00000200:14.0:1616513079.570881:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.106@tcp1(10.31.136.106@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.570888:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.570894:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.570896:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.570899:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.570903:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.570908:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.570924:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.570931:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.570934:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.570941:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.570945:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757540 00000800:00000200:26.0:1616513079.570949:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.570952:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.572693:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.572747:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.572756:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513079.572765:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.83@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.572774:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.83@tcp1 of length 224 into portal 26 MB=0x5ffd49f9e5dc0 00000400:00000200:22.0:1616513079.572781:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.83@tcp1 of length 224/224 into md 0xf97c8359 [1] + 0 00000400:00000200:22.0:1616513079.572791:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70550 00000400:00000200:22.0:1616513079.572794:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.572796:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.572800:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70550 00000800:00000200:22.0:1616513079.572803:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.572806:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.572808:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.572811:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.572821:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513079.572829:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.83@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.572838:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.83@tcp1 of length 224 into portal 12 MB=0x5ffd49f9e5e40 00000400:00000200:18.0:1616513079.572843:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.83@tcp1 of length 224/224 into md 0xf96450b9 [1] + 88824 00000400:00000200:18.0:1616513079.572848:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.572851:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.572858:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.572861:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.572863:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513079.572865:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663559658944 00000800:00000200:18.0:1616513079.572865:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:20.0:1616513079.572875:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688663559658944/t0(0) o400->a6cd19c1-ab79-0beb-e133-402d9a20260c@10.31.128.83@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.572892:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663559658944, offset 192 00000400:00000200:20.0:1616513079.572898:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.83@tcp1 00000400:00000200:20.0:1616513079.572905:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.83@tcp1 routed destination 00000400:00000200:20.0:1616513079.572920:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.83@tcp1(10.31.128.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:14.0:1616513079.572920:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663559659072 00000800:00000200:20.0:1616513079.572927:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.572932:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:14.0:1616513079.572932:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebbf00 x1688663559659072/t0(0) o400->5791aefc-5a4a-0f65-5181-c1f26c16c0cf@10.31.128.83@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513079.572935:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.572937:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.572942:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:14.0:1616513079.572945:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663559659072, offset 192 00000100:00000200:20.0:1616513079.572948:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513079.572948:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513079.572951:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.83@tcp1 00000400:00000200:14.0:1616513079.572957:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.83@tcp1 routed destination 00000400:00000200:14.0:1616513079.572971:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.83@tcp1(10.31.128.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.572977:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.572982:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.572984:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.572986:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.572991:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.572996:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513079.573001:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.573008:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513079.573011:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.573013:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513079.573017:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513079.573023:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee227f8 00000800:00000200:12.0:1616513079.573026:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513079.573050:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.573057:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.573060:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.573062:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513079.573067:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.573071:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756ee0 00000800:00000200:2.0:1616513079.573075:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.573244:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513079.573252:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.573258:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513079.573261:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.573264:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.50@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.573271:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.50@tcp1 of length 224 into portal 12 MB=0x5ffe05e76c5c0 00000400:00000200:18.0:1616513079.573276:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.50@tcp1 of length 224/224 into md 0xf96450b9 [1] + 89048 00000400:00000200:18.0:1616513079.573279:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.573281:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.573285:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.573287:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.573289:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.573297:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714006152640 00010000:00000200:14.0:1616513079.573303:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebb180 x1688714006152640/t0(0) o400->9655f3ac-e9f2-b4ca-5852-18355697c978@10.31.129.50@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.573314:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688714006152640, offset 192 00000400:00000200:14.0:1616513079.573319:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.50@tcp1 00000400:00000200:14.0:1616513079.573324:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.50@tcp1 routed destination 00000400:00000200:14.0:1616513079.573337:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.50@tcp1(10.31.129.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.573342:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.573346:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.573349:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.573350:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.573353:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.573365:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.573374:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513079.573378:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513079.573379:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.573381:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.573387:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513079.573388:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.573391:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756770 00000800:00000200:22.0:1616513079.573392:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513079.573395:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513079.573395:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513079.573397:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513079.573399:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.50@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.573405:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.50@tcp1 of length 224 into portal 26 MB=0x5ffe05e76c2c0 00000400:00000200:22.0:1616513079.573411:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.50@tcp1 of length 224/224 into md 0xf97c8369 [1] + 0 00000400:00000200:22.0:1616513079.573415:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241a90 00000400:00000200:22.0:1616513079.573416:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.573418:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.573421:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a90 00000800:00000200:22.0:1616513079.573423:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.573425:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.573426:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.573445:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714006151872 00010000:00000200:20.0:1616513079.573451:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688714006151872/t0(0) o400->fce70f63-a7b3-d927-c078-817606d34e55@10.31.129.50@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.573461:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688714006151872, offset 192 00000400:00000200:20.0:1616513079.573466:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.50@tcp1 00000400:00000200:20.0:1616513079.573471:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.50@tcp1 routed destination 00000400:00000200:20.0:1616513079.573484:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.50@tcp1(10.31.129.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.573489:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.573494:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.573496:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.573498:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.573501:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513079.573505:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.573506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.573515:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.573520:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.573522:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513079.573523:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513079.573529:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.573534:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22220 00000800:00000200:26.0:1616513079.573537:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.578322:0:49395:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.578378:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.578389:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.578398:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.168@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.578406:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.168@tcp1 of length 224 into portal 26 MB=0x5ffde54981b40 00000400:00000200:2.0:1616513079.578413:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.168@tcp1 of length 224/224 into md 0xf97c8389 [1] + 0 00000400:00000200:2.0:1616513079.578417:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782406e8 00000400:00000200:2.0:1616513079.578420:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.578423:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.578427:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782406e8 00000800:00000200:2.0:1616513079.578430:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.578432:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.578433:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.578446:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.578491:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705250630464 00010000:00000200:20.0:1616513079.578502:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688705250630464/t0(0) o400->7020b2b6-33e5-695a-b599-1e595c9321f3@10.31.132.168@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.578516:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705250630464, offset 192 00000400:00000200:20.0:1616513079.578522:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.168@tcp1 00000400:00000200:20.0:1616513079.578530:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.168@tcp1 routed destination 00000400:00000200:20.0:1616513079.578547:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.168@tcp1(10.31.132.168@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.578553:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.578559:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.578561:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.578564:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.578568:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513079.578575:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513079.578575:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.578587:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.578594:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.578597:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.578603:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.578609:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22550 00000800:00000200:24.0:1616513079.578612:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.578615:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.578622:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513079.578624:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.578626:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.578628:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.578630:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.578702:0:49395:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513079.578755:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.578764:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513079.578772:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.168@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.578780:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.168@tcp1 of length 224 into portal 12 MB=0x5ffde54981d00 00000400:00000200:18.0:1616513079.578786:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.168@tcp1 of length 224/224 into md 0xf96450b9 [1] + 89272 00000400:00000200:18.0:1616513079.578790:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.578793:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.578799:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.578802:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.578804:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.578807:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.578862:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705250630912 00010000:00000200:14.0:1616513079.578874:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebd580 x1688705250630912/t0(0) o400->cfbf8b82-36ec-11e9-e218-62b8df87c497@10.31.132.168@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.578887:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705250630912, offset 192 00000400:00000200:14.0:1616513079.578893:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.168@tcp1 00000400:00000200:14.0:1616513079.578900:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.168@tcp1 routed destination 00000400:00000200:14.0:1616513079.578914:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.168@tcp1(10.31.132.168@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.578920:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.578925:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.578928:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.578930:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.578934:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.578939:0:49395:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.578950:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.578955:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.578957:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.578959:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.578963:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.578967:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756220 00000800:00000200:22.0:1616513079.578971:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.580923:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.580977:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.580987:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.580995:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.51@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.581004:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.51@tcp1 of length 224 into portal 26 MB=0x603e24c6271c0 00000400:00000200:28.0:1616513079.581011:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.51@tcp1 of length 224/224 into md 0xf97c8399 [1] + 0 00000400:00000200:28.0:1616513079.581017:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241e48 00000400:00000200:28.0:1616513079.581020:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.581023:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513079.581027:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241e48 00000800:00000200:28.0:1616513079.581030:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.581032:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.581033:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.581038:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.581041:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.581041:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.581049:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.51@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.581058:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.51@tcp1 of length 224 into portal 12 MB=0x603e24c627240 00000400:00000200:24.0:1616513079.581063:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.51@tcp1 of length 224/224 into md 0x865bbec1 [1] + 22400 00000400:00000200:24.0:1616513079.581068:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.581071:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513079.581078:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.581081:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.581083:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.581103:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693120339276224 00010000:00000200:20.0:1616513079.581114:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1693120339276224/t0(0) o400->11f9ff5f-87bb-8554-19c4-938e54be1288@10.31.136.51@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.581127:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693120339276224, offset 192 00000400:00000200:20.0:1616513079.581133:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.51@tcp1 00000400:00000200:20.0:1616513079.581139:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.51@tcp1 routed destination 00000100:00000200:0.0:1616513079.581141:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693120339276352 00000400:00000200:20.0:1616513079.581153:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.51@tcp1(10.31.136.51@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513079.581153:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d1680 x1693120339276352/t0(0) o400->c7961f4c-d2a6-3048-76e0-8d0e548b4568@10.31.136.51@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513079.581159:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.581165:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513079.581166:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693120339276352, offset 192 00000800:00000200:20.0:1616513079.581167:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.581169:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513079.581172:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.51@tcp1 00000800:00000200:20.0:1616513079.581173:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.581179:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513079.581179:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.51@tcp1 routed destination 00000100:00000200:20.0:1616513079.581180:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513079.581188:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.581193:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513079.581193:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.51@tcp1(10.31.136.51@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513079.581195:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513079.581199:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513079.581201:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513079.581204:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513079.581206:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22c38 00000800:00000200:0.0:1616513079.581207:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513079.581208:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513079.581209:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513079.581211:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513079.581213:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.581220:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.581272:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.581278:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.581281:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.581287:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.581291:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:26.0:1616513079.581294:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.581297:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.581758:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.581767:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.581773:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.581777:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.581780:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.581787:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x60399133e1440 00000400:00000200:24.0:1616513079.581794:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0x865bbec1 [1] + 22624 00000400:00000200:24.0:1616513079.581798:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.581800:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513079.581805:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.581807:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.581808:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.581818:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805847979072 00010000:00000200:0.0:1616513079.581888:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5580 x1692805847979072/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:705/0 lens 336/336 e 0 to 0 dl 1616513085 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513079.581900:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805847979072, offset 192 00000400:00000200:0.0:1616513079.581905:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:0.0:1616513079.581911:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:0.0:1616513079.581924:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.581930:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.581935:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.581938:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.581939:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.581943:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.581947:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.581956:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.581960:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.581961:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513079.581965:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.581966:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.581969:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:28.0:1616513079.581971:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.583599:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513079.583609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513079.583651:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.583660:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.583668:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.55@tcp1 : PUT - for me 00000800:00000200:12.0:1616513079.583669:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513079.583676:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.55@tcp1 of length 224 into portal 26 MB=0x5ffd52530d0c0 00000800:00000200:12.0:1616513079.583679:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513079.583682:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.55@tcp1 of length 224/224 into md 0xf97c83c9 [1] + 0 00000400:00000200:2.0:1616513079.583687:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241210 00000400:00000200:12.0:1616513079.583688:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.55@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.583689:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.583692:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.583695:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241210 00000400:00000200:12.0:1616513079.583696:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.55@tcp1 of length 224 into portal 12 MB=0x5ffd52530d140 00000800:00000200:2.0:1616513079.583698:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.583703:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:12.0:1616513079.583704:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.55@tcp1 of length 224/224 into md 0x865bbec1 [1] + 22960 00000800:00000200:2.0:1616513079.583704:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.583707:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513079.583710:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513079.583712:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513079.583719:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.583721:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.583723:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513079.583726:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.583760:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665800626368 00010000:00000200:20.0:1616513079.583770:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688665800626368/t0(0) o400->1efb29cd-274d-351d-2890-34d979cfb18d@10.31.138.55@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.583783:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665800626368, offset 192 00000100:00000200:0.0:1616513079.583785:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665800626496 00000400:00000200:20.0:1616513079.583789:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.55@tcp1 00000400:00000200:20.0:1616513079.583796:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.55@tcp1 routed destination 00010000:00000200:0.0:1616513079.583797:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c665e80 x1688665800626496/t0(0) o400->4888e326-d2df-dd95-cc37-8b858979ad56@10.31.138.55@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513079.583810:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.55@tcp1(10.31.138.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513079.583811:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665800626496, offset 192 00000800:00000200:20.0:1616513079.583816:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513079.583818:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.55@tcp1 00000800:00000200:20.0:1616513079.583821:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.583825:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513079.583825:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.55@tcp1 routed destination 00000800:00000200:20.0:1616513079.583827:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.583832:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.583837:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.583838:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513079.583840:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.55@tcp1(10.31.138.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.583846:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.583852:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.583854:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.583857:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.583861:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.583866:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513079.583891:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.583897:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.583900:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.583906:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.583911:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22990 00000800:00000200:26.0:1616513079.583914:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.583917:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513079.583922:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513079.583930:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513079.583932:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.583935:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513079.583939:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513079.583943:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04cc0 00000800:00000200:4.0:1616513079.583947:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.585175:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.585226:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.585234:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.585242:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.199@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.585251:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.199@tcp1 of length 224 into portal 26 MB=0x5ffd4d0ff0f80 00000400:00000200:24.0:1616513079.585257:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.199@tcp1 of length 224/224 into md 0xf97c83d9 [1] + 0 00000400:00000200:24.0:1616513079.585261:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782402a8 00000400:00000200:24.0:1616513079.585264:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.585266:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.585270:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782402a8 00000800:00000200:24.0:1616513079.585273:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.585275:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513079.585276:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.585280:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.585330:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664388079488 00010000:00000200:20.0:1616513079.585340:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688664388079488/t0(0) o400->43275600-c3ed-3940-2e24-9b669d1d50e0@10.31.132.199@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.585354:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664388079488, offset 192 00000400:00000200:20.0:1616513079.585359:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.199@tcp1 00000400:00000200:20.0:1616513079.585366:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.199@tcp1 routed destination 00000400:00000200:20.0:1616513079.585380:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.199@tcp1(10.31.132.199@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.585386:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.585391:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.585394:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.585396:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.585400:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.585405:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.585406:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513079.585457:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.585463:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.585466:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.585472:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.585477:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22f68 00000800:00000200:2.0:1616513079.585481:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.585483:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.585847:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.585902:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.585911:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.585919:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.199@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.585928:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.199@tcp1 of length 224 into portal 12 MB=0x5ffd4d0ff1100 00000400:00000200:28.0:1616513079.585934:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.199@tcp1 of length 224/224 into md 0x865bbec1 [1] + 23184 00000400:00000200:28.0:1616513079.585939:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.585942:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513079.585948:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.585951:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.585953:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513079.585956:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.586013:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664388079872 00010000:00000200:0.0:1616513079.586025:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9e300 x1688664388079872/t0(0) o400->a0b1a02e-58f9-63f8-adcc-787c767bb298@10.31.132.199@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.586039:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664388079872, offset 192 00000400:00000200:0.0:1616513079.586046:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.199@tcp1 00000400:00000200:0.0:1616513079.586053:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.199@tcp1 routed destination 00000400:00000200:0.0:1616513079.586070:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.199@tcp1(10.31.132.199@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.586077:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.586082:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.586085:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.586087:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.586092:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.586097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.586106:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.586111:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513079.586114:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.586115:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513079.586121:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513079.586125:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04908 00000800:00000200:12.0:1616513079.586128:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.590872:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513079.590876:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513079.590927:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.590936:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513079.590958:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513079.590959:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.90@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.590967:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.90@tcp1 of length 224 into portal 26 MB=0x5ffdd00c58000 00000800:00000200:26.0:1616513079.590967:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513079.590974:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.90@tcp1 of length 224/224 into md 0xf97c83e9 [1] + 0 00000400:00000200:26.0:1616513079.590976:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.90@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.590979:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241cb0 00000400:00000200:28.0:1616513079.590981:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.590984:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513079.590984:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.90@tcp1 of length 224 into portal 12 MB=0x5ffdd00c58080 00000400:00000200:28.0:1616513079.590988:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241cb0 00000800:00000200:4.0:1616513079.590990:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.590991:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.590993:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513079.590993:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.90@tcp1 of length 224/224 into md 0xf96450b9 [1] + 89496 00000800:00000200:28.0:1616513079.590995:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513079.590998:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.591000:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513079.591007:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.591009:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.591011:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.591014:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.591053:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699549351936 00010000:00000200:20.0:1616513079.591063:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688699549351936/t0(0) o400->41086db4-b5f7-b3f9-e7ca-0488bbacc049@10.31.138.90@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:14.0:1616513079.591075:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699549352064 00000100:00000200:20.0:1616513079.591077:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699549351936, offset 192 00000400:00000200:20.0:1616513079.591083:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.90@tcp1 00010000:00000200:14.0:1616513079.591087:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9b00 x1688699549352064/t0(0) o400->ffd3828a-3970-bc8c-b66b-ecf6c60dd1f6@10.31.138.90@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513079.591089:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.90@tcp1 routed destination 00000100:00000200:14.0:1616513079.591100:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699549352064, offset 192 00000400:00000200:20.0:1616513079.591103:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.90@tcp1(10.31.138.90@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513079.591105:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.90@tcp1 00000800:00000200:20.0:1616513079.591109:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513079.591112:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.90@tcp1 routed destination 00000800:00000200:20.0:1616513079.591114:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.591117:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.591119:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.591124:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513079.591128:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.90@tcp1(10.31.138.90@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513079.591128:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.591130:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513079.591134:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.591140:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.591142:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.591144:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.591149:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.591153:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.591165:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.591171:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.591173:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.591179:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513079.591181:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513079.591183:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756198 00000800:00000200:24.0:1616513079.591187:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513079.591188:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513079.591189:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513079.591190:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.591196:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.591202:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23650 00000800:00000200:2.0:1616513079.591205:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.591208:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.591269:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513079.591280:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.591289:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513079.591290:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.591297:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.148@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.591306:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.148@tcp1 of length 224 into portal 12 MB=0x5ffda11403c00 00000400:00000200:18.0:1616513079.591312:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.148@tcp1 of length 224/224 into md 0xf96450b9 [1] + 89720 00000400:00000200:18.0:1616513079.591317:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.591321:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.591328:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.591331:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.591333:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.591339:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688686940929024 00010000:00000200:14.0:1616513079.591345:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfd100 x1688686940929024/t0(0) o400->f1abf640-1f6f-97b5-ef45-4386335f5d6d@10.31.129.148@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.591356:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688686940929024, offset 192 00000400:00000200:14.0:1616513079.591361:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.148@tcp1 00000400:00000200:14.0:1616513079.591366:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.148@tcp1 routed destination 00000800:00000200:11.2:1616513079.591370:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513079.591379:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.148@tcp1(10.31.129.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513079.591380:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.591384:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.591389:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513079.591390:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513079.591391:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513079.591391:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.591393:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.591396:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513079.591398:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.148@tcp1 : PUT - for me 00000800:00000200:7.2:1616513079.591400:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513079.591407:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.148@tcp1 of length 224 into portal 26 MB=0x5ffda11403b80 00000800:00000200:2.0:1616513079.591409:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.591413:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:4.0:1616513079.591414:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.148@tcp1 of length 224/224 into md 0xf97c8409 [1] + 0 00000400:00000200:2.0:1616513079.591414:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.591418:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513079.591418:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241100 00000400:00000200:2.0:1616513079.591419:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:4.0:1616513079.591421:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.591422:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757d38 00000800:00000200:2.0:1616513079.591424:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513079.591425:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513079.591429:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241100 00000800:00000200:4.0:1616513079.591432:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513079.591451:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688686940928896 00000800:00000200:4.0:1616513079.591456:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:20.0:1616513079.591457:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688686940928896/t0(0) o400->5868999c-eda8-450a-f062-920ca4dda1db@10.31.129.148@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:4.0:1616513079.591458:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.591468:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688686940928896, offset 192 00000400:00000200:20.0:1616513079.591472:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.148@tcp1 00000400:00000200:20.0:1616513079.591478:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.148@tcp1 routed destination 00000400:00000200:20.0:1616513079.591490:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.148@tcp1(10.31.129.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.591495:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.591500:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.591502:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.591504:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.591507:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513079.591511:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513079.591511:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.591520:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.591523:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.591525:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513079.591528:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.591529:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.591533:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ff0 00000800:00000200:24.0:1616513079.591536:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513079.594525:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925600, portal 4 00000100:00000200:28.0:1616513079.594530:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925600, offset 0 00000400:00000200:28.0:1616513079.594536:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:28.0:1616513079.594541:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:28.0:1616513079.594553:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513079.594559:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513079.594564:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513079.594567:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513079.594569:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513079.594573:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513079.594619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513079.594635:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513079.594639:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513079.594641:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.594646:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513079.594651:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b08e780 x1693067725925600/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513079.594662:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd188 00000800:00000200:28.0:1616513079.594665:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513079.594667:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513079.594820:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:12.0:1616513079.594874:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513079.594882:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:12.0:1616513079.594890:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513079.594897:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626ce0 00000400:00000200:12.0:1616513079.594903:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d55f1 [1] + 192 00000400:00000200:12.0:1616513079.594908:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513079.594913:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b08e780 x1693067725925600/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:24.0:1616513079.594926:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513079.594926:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513079.594929:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:12.0:1616513079.594931:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:28.0:1616513079.594932:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b08e780 x1693067725925600/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513079.594953:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b08e780 x1693067725925600/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513079.594961:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc000 00000800:00000200:11.2:1616513079.624489:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.624544:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.624554:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513079.624556:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.624562:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.133@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.624572:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.133@tcp1 of length 224 into portal 26 MB=0x603fc2607b040 00000400:00000200:28.0:1616513079.624579:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.133@tcp1 of length 224/224 into md 0xf97c8449 [1] + 0 00000400:00000200:28.0:1616513079.624584:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240440 00000400:00000200:28.0:1616513079.624587:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.624589:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513079.624595:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240440 00000800:00000200:28.0:1616513079.624598:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.624601:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.624603:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.624619:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693231364943936 00010000:00000200:20.0:1616513079.624631:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1693231364943936/t0(0) o400->e7268b54-f2ca-2adb-cb92-b23269b232b9@10.31.138.133@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.624646:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693231364943936, offset 192 00000400:00000200:20.0:1616513079.624652:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.133@tcp1 00000400:00000200:20.0:1616513079.624661:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.133@tcp1 routed destination 00000400:00000200:20.0:1616513079.624677:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.133@tcp1(10.31.138.133@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.624684:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.624690:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.624693:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.624695:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.624700:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513079.624707:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.624735:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513079.624791:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.624801:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513079.624809:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.133@tcp1 : PUT - for me 00000400:00000200:26.0:1616513079.624818:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.133@tcp1 of length 224 into portal 12 MB=0x603fc2607b140 00000400:00000200:26.0:1616513079.624824:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.133@tcp1 of length 224/224 into md 0xf96450b9 [1] + 89944 00000400:00000200:26.0:1616513079.624829:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513079.624831:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513079.624840:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513079.624843:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513079.624844:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513079.624847:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513079.624851:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513079.624853:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513079.624858:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513079.624862:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22e58 00000800:00000200:26.0:1616513079.624877:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513079.624880:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.624925:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693231364944192 00010000:00000200:14.0:1616513079.624936:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc579680 x1693231364944192/t0(0) o400->cbf84f61-7414-b385-cc01-b7a67676ac5c@10.31.138.133@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.624949:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693231364944192, offset 192 00000400:00000200:14.0:1616513079.624954:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.133@tcp1 00000400:00000200:14.0:1616513079.624961:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.133@tcp1 routed destination 00000400:00000200:14.0:1616513079.624974:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.133@tcp1(10.31.138.133@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.624980:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.624985:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.624987:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.624991:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.624995:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.625001:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.625013:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.625019:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.625022:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.625029:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.625033:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756660 00000800:00000200:24.0:1616513079.625037:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.625040:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513079.626920:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925616, portal 4 00000100:00000200:26.0:1616513079.626926:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925616, offset 0 00000400:00000200:26.0:1616513079.626933:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:26.0:1616513079.626939:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:26.0:1616513079.626952:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513079.626958:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:26.0:1616513079.626965:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513079.626970:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:26.0:1616513079.626972:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:26.0:1616513079.626977:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513079.627025:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513079.627046:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513079.627053:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513079.627056:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513079.627063:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513079.627071:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bd9b00 x1693067725925616/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513079.627083:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae880 00000800:00000200:1.0:1616513079.627088:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513079.627091:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513079.627198:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513079.627208:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513079.627215:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513079.627218:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513079.627223:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513079.627231:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626cf0 00000400:00000200:1.0:1616513079.627238:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffb75 [1] + 192 00000400:00000200:1.0:1616513079.627244:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513079.627248:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bd9b00 x1693067725925616/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513079.627263:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513079.627266:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513079.627268:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:26.0:1616513079.627273:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bd9b00 x1693067725925616/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513079.627286:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bd9b00 x1693067725925616/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513079.627296:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa90 00000800:00000200:19.2F:1616513079.642216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da653200] (20)++ 00000800:00000200:2.0:1616513079.642277:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da653200] (21)++ 00000800:00000200:2.0:1616513079.642287:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.203@o2ib1 00000400:00000200:2.0:1616513079.642296:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.203@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513079.642304:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.203@o2ib1 of length 224 into portal 26 MB=0x60386f09a64c0 00000400:00000200:2.0:1616513079.642311:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.203@o2ib1 of length 224/224 into md 0xf97c8459 [1] + 0 00000400:00000200:2.0:1616513079.642316:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782404c8 00000400:00000200:2.0:1616513079.642318:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.642321:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.642326:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782404c8 00000800:00000200:2.0:1616513079.642329:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da653200] (22)++ 00000800:00000200:18.0:1616513079.642332:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da653200] (23)++ 00000800:00000200:2.0:1616513079.642332:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da653200] (23)-- 00000800:00000200:2.0:1616513079.642339:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da653200] (23)-- 00000800:00000200:18.0:1616513079.642343:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.203@o2ib1 00000800:00000200:2.0:1616513079.642344:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da653200] (22)-- 00000400:00000200:18.0:1616513079.642351:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.203@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513079.642359:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.203@o2ib1 of length 224 into portal 12 MB=0x60386f09a6580 00000400:00000200:18.0:1616513079.642365:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.203@o2ib1 of length 224/224 into md 0xf96450b9 [1] + 90168 00000400:00000200:18.0:1616513079.642370:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.642373:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.642380:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da653200] (21)++ 00000800:00000200:18.0:1616513079.642383:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da653200] (22)-- 00000800:00000200:18.0:1616513079.642385:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da653200] (21)-- 00000100:00000200:20.0:1616513079.642391:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692727957415104 00010000:00000200:20.0:1616513079.642402:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1692727957415104/t0(0) o400->1f6f4d8f-dfa8-472d-a56c-e3fa7194f6d6@10.242.46.203@o2ib1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.642416:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692727957415104, offset 192 00000400:00000200:20.0:1616513079.642424:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.203@o2ib1 00000400:00000200:20.0:1616513079.642431:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.203@o2ib1 local destination 00000100:00000200:14.0:1616513079.642458:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692727957415296 00000400:00000200:20.0:1616513079.642459:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.203@o2ib1(10.242.46.203@o2ib1:10.242.46.203@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.642465:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.203@o2ib1 00010000:00000200:14.0:1616513079.642470:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfad00 x1692727957415296/t0(0) o400->6e78e7cf-341a-73e0-0c21-7394fbf4d812@10.242.46.203@o2ib1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513079.642471:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9d00] -> 10.242.46.203@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.642474:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da653200] (20)++ 00000800:00000200:20.0:1616513079.642476:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da653200] (21)++ 00000800:00000200:20.0:1616513079.642481:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da653200] (22)-- 00000100:00000200:26.0:1616513079.642484:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925632, portal 4 00000100:00000200:14.0:1616513079.642484:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692727957415296, offset 192 00000800:00000200:19.2:1616513079.642486:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da653200] (21)++ 00000100:00000200:20.0:1616513079.642488:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513079.642491:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.203@o2ib1 00000100:00000200:26.0:1616513079.642493:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925632, offset 0 00000400:00000200:14.0:1616513079.642497:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.203@o2ib1 local destination 00000400:00000200:26.0:1616513079.642500:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:14.0:1616513079.642511:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.203@o2ib1(10.242.46.203@o2ib1:10.242.46.203@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.642518:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.203@o2ib1 00000400:00000200:26.0:1616513079.642519:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:22.0:1616513079.642522:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da653200] (22)++ 00000800:00000200:14.0:1616513079.642527:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9d00] -> 10.242.46.203@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513079.642529:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da653200] (23)-- 00000800:00000200:14.0:1616513079.642529:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da653200] (23)++ 00000800:00000200:14.0:1616513079.642531:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da653200] (23)++ 00000400:00000200:26.0:1616513079.642532:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513079.642532:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.642534:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da653200] (24)-- 00000800:00000200:14.0:1616513079.642536:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da653200] (23)-- 00000800:00000200:26.0:1616513079.642539:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:22.0:1616513079.642539:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.203@o2ib1: PUT: OK 00000800:00000200:19.2:1616513079.642541:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da653200] (22)++ 00000800:00000200:26.0:1616513079.642545:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513079.642545:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23078 00000800:00000200:26.0:1616513079.642548:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:22.0:1616513079.642548:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da653200] (23)-- 00000800:00000200:26.0:1616513079.642550:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:22.0:1616513079.642551:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da653200] (22)++ 00000800:00000200:26.0:1616513079.642556:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:22.0:1616513079.642556:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da653200] (23)-- 00000400:00000200:22.0:1616513079.642558:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513079.642562:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da653200] (22)-- 00000400:00000200:22.0:1616513079.642563:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.203@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.642566:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757f58 00000800:00000200:22.0:1616513079.642569:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da653200] (21)-- 00000800:00000200:27.2:1616513079.642602:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.642623:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513079.642629:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513079.642632:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513079.642639:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513079.642645:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c9bf8480 x1693067725925632/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513079.642657:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af6d8 00000800:00000200:27.0:1616513079.642661:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513079.642664:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513079.642771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513079.642780:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.642786:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513079.642794:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513079.642801:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626d00 00000400:00000200:27.0:1616513079.642808:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166b4d [1] + 192 00000400:00000200:27.0:1616513079.642813:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513079.642818:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c9bf8480 x1693067725925632/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513079.642832:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513079.642843:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513079.642846:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513079.642848:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:26.0:1616513079.642852:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c9bf8480 x1693067725925632/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513079.642866:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c9bf8480 x1693067725925632/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513079.642876:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa90 00000800:00000200:11.2:1616513079.644684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.644696:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.644704:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.644711:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.91@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.644720:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.91@tcp1 of length 224 into portal 26 MB=0x5ffd46d3e2b40 00000400:00000200:8.0:1616513079.644725:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.91@tcp1 of length 224/224 into md 0xf97c8479 [1] + 0 00000400:00000200:8.0:1616513079.644729:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240990 00000400:00000200:8.0:1616513079.644731:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.644733:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513079.644737:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240990 00000800:00000200:8.0:1616513079.644739:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.644742:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.644743:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.644745:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.644749:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513079.644750:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513079.644754:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.91@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.644759:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.91@tcp1 of length 224 into portal 12 MB=0x5ffd46d3e2c80 00000400:00000200:8.0:1616513079.644763:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.91@tcp1 of length 224/224 into md 0x865bbec1 [1] + 23408 00000400:00000200:8.0:1616513079.644766:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.644768:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513079.644773:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.644774:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.644775:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.644800:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662714493760 00010000:00000200:20.0:1616513079.644810:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688662714493760/t0(0) o400->80e5e892-b639-4973-00ff-154f414c24a3@10.31.138.91@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.644823:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662714493760, offset 192 00000400:00000200:20.0:1616513079.644829:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.91@tcp1 00000400:00000200:20.0:1616513079.644836:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.91@tcp1 routed destination 00000100:00000200:0.0:1616513079.644848:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662714494080 00000400:00000200:20.0:1616513079.644851:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.91@tcp1(10.31.138.91@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.644856:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:0.0:1616513079.644860:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3600 x1688662714494080/t0(0) o400->df41eb8d-3dc0-8112-cc43-8f3a9b1b90b9@10.31.138.91@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513079.644862:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.644864:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.644866:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.644871:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513079.644874:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662714494080, offset 192 00000100:00000200:20.0:1616513079.644877:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.644878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513079.644881:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.91@tcp1 00000400:00000200:0.0:1616513079.644889:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.91@tcp1 routed destination 00000400:00000200:0.0:1616513079.644904:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.91@tcp1(10.31.138.91@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.644911:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.644916:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.644919:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.644921:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.644926:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.644931:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513079.644933:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.644940:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.644942:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.644949:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.644954:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22198 00000800:00000200:2.0:1616513079.644958:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.644961:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513079.644985:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.644992:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513079.644995:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.644996:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513079.645001:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513079.645005:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04440 00000800:00000200:12.0:1616513079.645009:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.647148:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.647159:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.647163:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.647169:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.201@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.647174:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.201@tcp1 of length 224 into portal 26 MB=0x605a073007440 00000400:00000200:8.0:1616513079.647178:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.201@tcp1 of length 224/224 into md 0xf97c8499 [1] + 0 00000400:00000200:8.0:1616513079.647181:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782407f8 00000400:00000200:8.0:1616513079.647182:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.647183:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513079.647186:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782407f8 00000800:00000200:8.0:1616513079.647188:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.647190:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.647191:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.647192:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.647196:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.647202:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.201@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.647205:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.201@tcp1 of length 224 into portal 12 MB=0x605a0730074c0 00000800:00000200:24.0:1616513079.647209:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513079.647209:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.201@tcp1 of length 224/224 into md 0x865bbec1 [1] + 23632 00000400:00000200:8.0:1616513079.647212:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.647213:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513079.647216:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.647218:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.647219:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.647243:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036542579904 00010000:00000200:0.0:1616513079.647256:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5100 x1695036542579904/t0(0) o400->52d418a3-0f25-a8ec-0c41-9aefac21cbcd@10.31.129.201@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513079.647265:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036542579776 00000100:00000200:0.0:1616513079.647270:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036542579904, offset 192 00010000:00000200:20.0:1616513079.647275:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1695036542579776/t0(0) o400->c4d5b994-8d32-6fab-903c-09c84fd96bca@10.31.129.201@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513079.647276:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.201@tcp1 00000400:00000200:0.0:1616513079.647284:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.201@tcp1 routed destination 00000100:00000200:20.0:1616513079.647288:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036542579776, offset 192 00000400:00000200:20.0:1616513079.647294:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.201@tcp1 00000400:00000200:0.0:1616513079.647299:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.201@tcp1(10.31.129.201@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513079.647301:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.201@tcp1 routed destination 00000800:00000200:0.0:1616513079.647305:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513079.647315:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.201@tcp1(10.31.129.201@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.647325:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.647326:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.647329:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.647331:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.647333:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.647333:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.647335:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.647338:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513079.647340:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513079.647343:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513079.647345:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513079.647346:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513079.647352:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.647357:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513079.647359:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513079.647365:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513079.647368:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:30.0:1616513079.647372:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.647374:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513079.647410:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.647417:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.647420:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513079.647421:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513079.647428:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513079.647446:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22110 00000800:00000200:12.0:1616513079.647450:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.655628:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.655683:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.655699:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.655708:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.244@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.655717:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.244@tcp1 of length 224 into portal 26 MB=0x5ffd43c167f80 00000400:00000200:2.0:1616513079.655724:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.244@tcp1 of length 224/224 into md 0xf97c84c9 [1] + 0 00000400:00000200:2.0:1616513079.655729:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240c38 00000400:00000200:2.0:1616513079.655731:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.655734:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.655738:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240c38 00000800:00000200:2.0:1616513079.655742:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.655743:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.655745:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.655747:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.655800:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661889810304 00010000:00000200:20.0:1616513079.655810:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688661889810304/t0(0) o400->35b12651-7a5e-1cce-55cf-9b42246c34c8@10.31.132.244@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.655823:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661889810304, offset 192 00000400:00000200:20.0:1616513079.655829:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.244@tcp1 00000800:00000200:7.2:1616513079.655830:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513079.655836:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.244@tcp1 routed destination 00000800:00000200:30.0:1616513079.655841:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.655847:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513079.655850:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.244@tcp1(10.31.132.244@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.655856:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:30.0:1616513079.655858:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.244@tcp1 : PUT - for me 00000800:00000200:20.0:1616513079.655862:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.655864:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:30.0:1616513079.655866:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.244@tcp1 of length 224 into portal 12 MB=0x5ffd43c168080 00000800:00000200:20.0:1616513079.655866:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:30.0:1616513079.655871:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.244@tcp1 of length 224/224 into md 0xf96450b9 [1] + 90392 00000800:00000200:20.0:1616513079.655871:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:30.0:1616513079.655876:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.655877:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:30.0:1616513079.655878:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513079.655891:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513079.655893:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513079.655895:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.655898:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.655906:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.655909:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513079.655910:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.655916:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.655921:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23188 00000800:00000200:22.0:1616513079.655926:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.655951:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661889810560 00010000:00000200:14.0:1616513079.655963:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7ec00 x1688661889810560/t0(0) o400->e4a79d03-1f15-027a-ed96-5ed59e6704eb@10.31.132.244@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.655977:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661889810560, offset 192 00000400:00000200:14.0:1616513079.655983:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.244@tcp1 00000400:00000200:14.0:1616513079.655991:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.244@tcp1 routed destination 00000400:00000200:14.0:1616513079.656005:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.244@tcp1(10.31.132.244@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.656012:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.656018:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.656020:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.656022:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.656027:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.656073:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.656084:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.656089:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.656090:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.656094:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513079.656095:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.656098:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756cc0 00000800:00000200:8.0:1616513079.656100:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:29.0:1616513079.658643:0:6495:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925648, portal 4 00000100:00000200:29.0:1616513079.658649:0:6495:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925648, offset 0 00000400:00000200:29.0:1616513079.658656:0:6495:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:29.0:1616513079.658662:0:6495:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:29.0:1616513079.658676:0:6495:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513079.658683:0:6495:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:29.0:1616513079.658690:0:6495:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513079.658693:0:6495:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:29.0:1616513079.658696:0:6495:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:29.0:1616513079.658701:0:6495:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513079.658717:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513079.658773:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513079.658780:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:24.0:1616513079.658783:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.658790:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:24.0:1616513079.658797:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1392400 x1693067725925648/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513079.658809:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2550 00000800:00000200:24.0:1616513079.658813:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513079.658816:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513079.658928:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513079.658983:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513079.658993:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:8.0:1616513079.658993:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513079.659002:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513079.659009:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626d10 00000400:00000200:28.0:1616513079.659016:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d5681 [1] + 192 00000400:00000200:28.0:1616513079.659022:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.659027:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1392400 x1693067725925648/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513079.659042:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513079.659045:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513079.659047:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:29.0:1616513079.659052:0:6495:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1392400 x1693067725925648/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:29.0:1616513079.659065:0:6495:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1392400 x1693067725925648/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:29.0:1616513079.659076:0:6495:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b25d8 00000800:00000200:7.2:1616513079.662870:0:49405:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.662927:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.662937:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513079.662938:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.662945:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.662946:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.66@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.662965:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.66@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.662968:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.66@tcp1 of length 224 into portal 26 MB=0x5f909c32c1140 00000400:00000200:30.0:1616513079.662973:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.66@tcp1 of length 224 into portal 12 MB=0x5f909c32c11c0 00000400:00000200:2.0:1616513079.662974:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.66@tcp1 of length 224/224 into md 0xf97c84d9 [1] + 0 00000400:00000200:2.0:1616513079.662979:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241dc0 00000400:00000200:2.0:1616513079.662981:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513079.662983:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.66@tcp1 of length 224/224 into md 0xf96450b9 [1] + 90616 00000400:00000200:30.0:1616513079.662987:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.662987:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.662990:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241dc0 00000100:00000200:30.0:1616513079.662992:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513079.662993:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513079.662996:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513079.662997:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513079.662997:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513079.663000:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513079.663001:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.663004:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.663052:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681195208020288 00010000:00000200:20.0:1616513079.663062:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1681195208020288/t0(0) o400->b9bf8d81-a08d-4a33-b659-ab54f58c3f65@10.31.131.66@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.663076:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681195208020288, offset 192 00000100:00000200:14.0:1616513079.663078:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681195208020416 00000400:00000200:20.0:1616513079.663081:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.66@tcp1 00000400:00000200:20.0:1616513079.663089:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.66@tcp1 routed destination 00010000:00000200:14.0:1616513079.663089:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a78480 x1681195208020416/t0(0) o400->3a88e120-5ed1-48c4-feb8-81211fe28b22@10.31.131.66@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.663102:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681195208020416, offset 192 00000400:00000200:20.0:1616513079.663103:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.66@tcp1(10.31.131.66@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.663109:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513079.663109:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.66@tcp1 00000800:00000200:20.0:1616513079.663115:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513079.663117:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.66@tcp1 routed destination 00000800:00000200:20.0:1616513079.663118:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.663121:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.663126:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513079.663130:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.66@tcp1(10.31.131.66@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:20.0:1616513079.663132:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.663132:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.663136:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.663141:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.663144:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.663146:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.663150:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513079.663187:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.663194:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513079.663194:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513079.663197:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.663197:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513079.663204:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.663209:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22a18 00000800:00000200:22.0:1616513079.663213:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513079.663250:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.663257:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.663260:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.663266:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.663270:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756880 00000800:00000200:24.0:1616513079.663274:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.663277:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.665957:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513079.666010:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.666019:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513079.666029:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.13@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.666038:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.13@tcp1 of length 224 into portal 26 MB=0x5ffd49b814c00 00000400:00000200:18.0:1616513079.666045:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.13@tcp1 of length 224/224 into md 0xf97c84f9 [1] + 0 00000400:00000200:18.0:1616513079.666049:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240d48 00000400:00000200:18.0:1616513079.666052:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.666055:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513079.666059:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240d48 00000800:00000200:18.0:1616513079.666062:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.666064:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.666066:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.666069:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.666078:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.666079:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.666086:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.13@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.666095:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.13@tcp1 of length 224 into portal 12 MB=0x5ffd49b814d00 00000400:00000200:2.0:1616513079.666100:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.13@tcp1 of length 224/224 into md 0xf96450b9 [1] + 90840 00000400:00000200:2.0:1616513079.666105:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.666108:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513079.666114:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.666117:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.666119:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.666120:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663490644992 00010000:00000200:20.0:1616513079.666130:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688663490644992/t0(0) o400->949d49c1-6574-81b1-2633-f3ba0cf01a74@10.31.138.13@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.666143:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663490644992, offset 192 00000400:00000200:20.0:1616513079.666149:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.13@tcp1 00000400:00000200:20.0:1616513079.666156:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.13@tcp1 routed destination 00000400:00000200:20.0:1616513079.666170:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.13@tcp1(10.31.138.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.666176:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:14.0:1616513079.666176:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663490645248 00000800:00000200:20.0:1616513079.666181:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.666184:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.666186:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:14.0:1616513079.666187:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21f500 x1688663490645248/t0(0) o400->6a7a7af5-6f2e-7071-3031-f1c2dd911786@10.31.138.13@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513079.666190:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.666195:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.666197:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:14.0:1616513079.666201:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663490645248, offset 192 00000400:00000200:14.0:1616513079.666207:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.13@tcp1 00000400:00000200:14.0:1616513079.666213:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.13@tcp1 routed destination 00000400:00000200:14.0:1616513079.666227:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.13@tcp1(10.31.138.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.666233:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.666238:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.666241:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.666243:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.666247:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513079.666252:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.666259:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.666262:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513079.666263:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.666268:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.666274:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23b18 00000800:00000200:22.0:1616513079.666277:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.666290:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.666301:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.666304:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.666305:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.666310:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:8.0:1616513079.666310:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.666312:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757870 00000800:00000200:8.0:1616513079.666316:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.666318:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.666320:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513079.666325:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.666327:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.666329:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.667964:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.668021:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.668030:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513079.668030:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513079.668039:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.166@tcp1 : PUT - for me 00000400:00000200:12.0:1616513079.668048:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.166@tcp1 of length 224 into portal 26 MB=0x603d00747e280 00000400:00000200:12.0:1616513079.668054:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.166@tcp1 of length 224/224 into md 0xf97c8509 [1] + 0 00000400:00000200:12.0:1616513079.668059:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241980 00000400:00000200:12.0:1616513079.668062:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513079.668064:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513079.668069:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241980 00000800:00000200:12.0:1616513079.668072:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.668075:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.668077:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.668093:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693041870496384 00010000:00000200:20.0:1616513079.668104:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1693041870496384/t0(0) o400->0aaff68f-3df7-4241-cd5e-c94766ee208b@10.31.129.166@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.668118:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693041870496384, offset 192 00000400:00000200:20.0:1616513079.668124:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.166@tcp1 00000400:00000200:20.0:1616513079.668131:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.166@tcp1 routed destination 00000400:00000200:20.0:1616513079.668146:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.166@tcp1(10.31.129.166@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.668153:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.668159:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.668161:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.668164:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.668168:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.668173:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.668175:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513079.668184:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.668188:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513079.668191:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513079.668196:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513079.668201:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22000 00000800:00000200:30.0:1616513079.668204:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.668216:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.668221:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513079.668227:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.166@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.668232:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.166@tcp1 of length 224 into portal 12 MB=0x603d00747e600 00000400:00000200:30.0:1616513079.668237:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.166@tcp1 of length 224/224 into md 0xf96450b9 [1] + 91064 00000400:00000200:30.0:1616513079.668240:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.668242:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513079.668245:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.668247:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513079.668249:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.668251:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.668309:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693041870497280 00010000:00000200:14.0:1616513079.668319:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7b600 x1693041870497280/t0(0) o400->36448a5b-5283-7979-b94f-af5f02b37daf@10.31.129.166@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.668332:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693041870497280, offset 192 00000400:00000200:14.0:1616513079.668337:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.166@tcp1 00000400:00000200:14.0:1616513079.668343:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.166@tcp1 routed destination 00000400:00000200:14.0:1616513079.668356:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.166@tcp1(10.31.129.166@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.668362:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.668367:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.668372:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.668374:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.668378:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.668383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.668395:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.668401:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.668404:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.668410:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.668413:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757650 00000800:00000200:28.0:1616513079.668417:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.668420:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.672598:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.672650:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.672659:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513079.672660:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.672667:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.44@tcp1 : PUT - for me 00000800:00000200:11.2:1616513079.672673:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513079.672676:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.44@tcp1 of length 224 into portal 12 MB=0x6001199608980 00000400:00000200:22.0:1616513079.672682:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.44@tcp1 of length 224/224 into md 0xf96450b9 [1] + 91288 00000400:00000200:22.0:1616513079.672687:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.672689:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513079.672695:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.672698:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.672700:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513079.672727:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.672734:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.672741:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.44@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.672749:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.44@tcp1 of length 224 into portal 26 MB=0x6001199608680 00000400:00000200:8.0:1616513079.672754:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.44@tcp1 of length 224/224 into md 0xf97c8519 [1] + 0 00000100:00000200:14.0:1616513079.672758:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688925447948672 00000400:00000200:8.0:1616513079.672761:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240880 00000400:00000200:8.0:1616513079.672764:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.672766:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00010000:00000200:14.0:1616513079.672768:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec200900 x1688925447948672/t0(0) o400->37cef1e3-df6b-de6f-1e60-604f5599b8ad@10.31.132.44@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:8.0:1616513079.672770:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240880 00000800:00000200:8.0:1616513079.672772:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.672774:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.672776:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.672778:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513079.672781:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688925447948672, offset 192 00000400:00000200:14.0:1616513079.672786:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.44@tcp1 00000400:00000200:14.0:1616513079.672793:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.44@tcp1 routed destination 00000400:00000200:14.0:1616513079.672805:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.44@tcp1(10.31.132.44@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.672811:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.672816:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.672818:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.672820:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.672824:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.672829:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.672831:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688925447947904 00010000:00000200:20.0:1616513079.672840:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688925447947904/t0(0) o400->fd05804e-30af-f8b2-16b8-bd5969423743@10.31.132.44@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.672854:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688925447947904, offset 192 00000400:00000200:20.0:1616513079.672859:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.44@tcp1 00000400:00000200:20.0:1616513079.672866:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.44@tcp1 routed destination 00000400:00000200:20.0:1616513079.672879:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.44@tcp1(10.31.132.44@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.672885:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.672890:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.672892:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513079.672893:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513079.672894:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.672898:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513079.672899:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.672902:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513079.672903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513079.672904:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513079.672908:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.672913:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756e58 00000800:00000200:28.0:1616513079.672915:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513079.672917:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.672920:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513079.672921:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.672924:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.672930:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.672935:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee223b8 00000800:00000200:28.0:1616513079.672938:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.672941:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.673359:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513079.673362:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513079.673367:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.673373:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:12.0:1616513079.673374:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513079.673376:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513079.673380:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.20@tcp1 : PUT - for me 00000800:00000200:12.0:1616513079.673383:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513079.673384:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513079.673387:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.20@tcp1 of length 224 into portal 26 MB=0x5ffd43da3dcc0 00000400:00000200:30.0:1616513079.673392:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.20@tcp1 of length 224/224 into md 0xf97c8539 [1] + 0 00000400:00000200:12.0:1616513079.673393:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.20@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.673396:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241650 00000400:00000200:30.0:1616513079.673398:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.673400:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513079.673402:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.20@tcp1 of length 224 into portal 12 MB=0x5ffd43da3dd40 00000400:00000200:30.0:1616513079.673404:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241650 00000800:00000200:30.0:1616513079.673406:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.673408:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513079.673408:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.20@tcp1 of length 224/224 into md 0x865bbec1 [1] + 23856 00000800:00000200:30.0:1616513079.673410:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513079.673413:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513079.673415:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661915851968 00000100:00000200:12.0:1616513079.673416:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:20.0:1616513079.673420:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688661915851968/t0(0) o400->048702fd-6f6e-777b-0edf-f3ecb37b1036@10.31.133.20@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513079.673422:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.673425:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.673427:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.673430:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661915851968, offset 192 00000400:00000200:20.0:1616513079.673445:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.20@tcp1 00000400:00000200:20.0:1616513079.673450:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.20@tcp1 routed destination 00000400:00000200:20.0:1616513079.673462:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.20@tcp1(10.31.133.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.673467:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.673471:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.673473:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.673474:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.673478:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513079.673482:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.673483:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513079.673487:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661915852096 00000800:00000200:18.0:1616513079.673492:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.673496:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.673497:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:0.0:1616513079.673499:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c665100 x1688661915852096/t0(0) o400->5519faf0-7f18-987b-6234-ebcca9606624@10.31.133.20@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513079.673501:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.673502:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.673506:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23760 00000800:00000200:18.0:1616513079.673508:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513079.673514:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661915852096, offset 192 00000400:00000200:0.0:1616513079.673520:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.20@tcp1 00000400:00000200:0.0:1616513079.673527:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.20@tcp1 routed destination 00000400:00000200:0.0:1616513079.673542:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.20@tcp1(10.31.133.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.673548:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.673554:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.673557:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.673559:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.673564:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.673569:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.673578:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.673581:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.673583:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.673587:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.673588:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.673591:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05d38 00000800:00000200:28.0:1616513079.673593:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.714764:0:49410:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.714821:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513079.714822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513079.714831:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513079.714834:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513079.714834:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513079.714841:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.84@tcp1 : PUT - for me 00000800:00000200:8.0:1616513079.714842:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513079.714851:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.84@tcp1 of length 224 into portal 26 MB=0x6036ded935f00 00000400:00000200:8.0:1616513079.714851:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.84@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.714857:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.11.84@tcp1 of length 224/224 into md 0xf97c8549 [1] + 0 00000400:00000200:8.0:1616513079.714859:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.84@tcp1 of length 224 into portal 12 MB=0x6036ded935f80 00000400:00000200:22.0:1616513079.714862:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782414b8 00000400:00000200:22.0:1616513079.714865:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.714865:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.11.84@tcp1 of length 224/224 into md 0x865bbec1 [1] + 24080 00000100:00000200:22.0:1616513079.714868:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513079.714870:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513079.714872:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782414b8 00000100:00000200:8.0:1616513079.714872:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513079.714878:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513079.714879:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513079.714881:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513079.714882:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513079.714883:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513079.714884:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.714886:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.714950:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692620532440832 00010000:00000200:20.0:1616513079.714961:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1692620532440832/t0(0) o400->8f515bce-5495-d864-04be-5bd4526958c5@10.31.11.84@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.714975:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692620532440832, offset 192 00000100:00000200:0.0:1616513079.714977:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692620532440960 00000400:00000200:20.0:1616513079.714980:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.84@tcp1 00000400:00000200:20.0:1616513079.714987:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.84@tcp1 routed destination 00010000:00000200:0.0:1616513079.714990:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0000 x1692620532440960/t0(0) o400->900f21fb-dcd6-3760-941f-098753b96011@10.31.11.84@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513079.715005:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.84@tcp1(10.31.11.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513079.715005:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692620532440960, offset 192 00000800:00000200:20.0:1616513079.715011:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513079.715011:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.84@tcp1 00000800:00000200:20.0:1616513079.715017:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:0.0:1616513079.715018:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.84@tcp1 routed destination 00000800:00000200:20.0:1616513079.715020:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.715021:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.715026:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.715031:0:49410:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513079.715032:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.84@tcp1(10.31.11.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:20.0:1616513079.715033:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513079.715039:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.715044:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.715047:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.715049:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.715054:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.715059:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513079.715085:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.715092:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.715094:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.715100:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.715106:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ee0 00000800:00000200:2.0:1616513079.715109:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.715112:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513079.715115:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.715123:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.715125:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.715128:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.715132:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.715136:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb046e8 00000800:00000200:24.0:1616513079.715140:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.715849:0:49410:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513079.715859:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.715867:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513079.715868:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.715875:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513079.715876:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.143@tcp1 : PUT - for me 00000800:00000200:2.0:1616513079.715877:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.715882:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.143@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.715885:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.143@tcp1 of length 224 into portal 26 MB=0x5ffd48ab7dc00 00000400:00000200:22.0:1616513079.715889:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.143@tcp1 of length 224 into portal 12 MB=0x5ffd48ab7dc80 00000400:00000200:30.0:1616513079.715892:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.143@tcp1 of length 224/224 into md 0xf97c8559 [1] + 0 00000400:00000200:30.0:1616513079.715897:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240330 00000400:00000200:30.0:1616513079.715899:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513079.715901:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.143@tcp1 of length 224/224 into md 0xf96450b9 [1] + 91512 00000400:00000200:22.0:1616513079.715905:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.715906:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513079.715910:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240330 00000800:00000200:30.0:1616513079.715913:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:22.0:1616513079.715913:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513079.715916:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513079.715918:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.715919:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.715921:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.715922:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.715923:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663209008128 00010000:00000200:20.0:1616513079.715929:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688663209008128/t0(0) o400->8e3bb7d5-3d5a-2d1e-65aa-b42f2f40df12@10.31.136.143@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.715940:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663209008128, offset 192 00000400:00000200:20.0:1616513079.715945:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.143@tcp1 00000400:00000200:20.0:1616513079.715951:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.143@tcp1 routed destination 00000400:00000200:20.0:1616513079.715964:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.143@tcp1(10.31.136.143@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.715970:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.715975:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.715977:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.715979:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.715982:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513079.715986:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663209008256 00000800:00000200:7.2:1616513079.715986:0:49410:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.715987:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513079.715998:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:14.0:1616513079.715999:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec205a00 x1688663209008256/t0(0) o400->d5b60367-6d4e-77ff-6f86-31613e86c44e@10.31.136.143@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513079.716005:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.716008:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.716008:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513079.716013:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663209008256, offset 192 00000400:00000200:18.0:1616513079.716015:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.716020:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22aa0 00000400:00000200:14.0:1616513079.716020:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.143@tcp1 00000400:00000200:14.0:1616513079.716027:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.143@tcp1 routed destination 00000800:00000200:18.0:1616513079.716033:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:14.0:1616513079.716042:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.143@tcp1(10.31.136.143@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.716049:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.716055:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.716057:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.716059:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.716064:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.716069:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.716137:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.716145:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.716147:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513079.716148:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513079.716154:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513079.716158:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757b18 00000800:00000200:12.0:1616513079.716162:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.729735:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513079.729747:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.729754:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513079.729762:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.74@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.729781:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.74@tcp1 of length 224 into portal 12 MB=0x5ffd4a9f0e640 00000400:00000200:30.0:1616513079.729786:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.74@tcp1 of length 224/224 into md 0xf96450b9 [1] + 91736 00000400:00000200:30.0:1616513079.729791:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.729793:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513079.729799:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.729801:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513079.729804:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.729806:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.729865:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663732840000 00010000:00000200:14.0:1616513079.729877:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7da00 x1688663732840000/t0(0) o400->19b1699a-6de9-096e-3781-5e7ed5bd7ba8@10.31.138.74@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.729902:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663732840000, offset 192 00000800:00000200:11.2:1616513079.729903:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513079.729909:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.74@tcp1 00000400:00000200:14.0:1616513079.729916:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.74@tcp1 routed destination 00000400:00000200:14.0:1616513079.729931:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.74@tcp1(10.31.138.74@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.729938:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.729944:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.729947:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.729949:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.729953:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.729958:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513079.729959:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.729968:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.729978:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.74@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.729986:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.74@tcp1 of length 224 into portal 26 MB=0x5ffd4a9f0e5c0 00000400:00000200:28.0:1616513079.729993:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.74@tcp1 of length 224/224 into md 0xf97c8569 [1] + 0 00000400:00000200:28.0:1616513079.729997:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241ed0 00000400:00000200:28.0:1616513079.730000:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.730003:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513079.730007:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241ed0 00000800:00000200:28.0:1616513079.730010:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513079.730012:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513079.730015:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.730017:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513079.730019:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513079.730020:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513079.730022:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.730028:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.730032:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47573a8 00000800:00000200:2.0:1616513079.730036:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.730039:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.730073:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663732839872 00010000:00000200:20.0:1616513079.730084:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688663732839872/t0(0) o400->12214e06-d234-90f8-d6bf-97209a23abf7@10.31.138.74@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.730098:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663732839872, offset 192 00000400:00000200:20.0:1616513079.730104:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.74@tcp1 00000400:00000200:20.0:1616513079.730112:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.74@tcp1 routed destination 00000400:00000200:20.0:1616513079.730127:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.74@tcp1(10.31.138.74@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.730133:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.730139:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.730143:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.730145:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.730150:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.730154:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513079.730157:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513079.730165:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.730170:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.730172:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.730178:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.730183:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22770 00000800:00000200:8.0:1616513079.730186:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.730188:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.738677:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.738732:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.738741:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.738750:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.247@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.738759:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.247@tcp1 of length 224 into portal 26 MB=0x60183d4d4e000 00000400:00000200:28.0:1616513079.738766:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.247@tcp1 of length 224/224 into md 0xf97c8579 [1] + 0 00000400:00000200:28.0:1616513079.738770:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782413a8 00000400:00000200:28.0:1616513079.738773:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.738776:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513079.738780:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782413a8 00000800:00000200:28.0:1616513079.738783:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.738784:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.738786:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.738791:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.738794:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.738795:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.738808:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.247@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.738817:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.247@tcp1 of length 224 into portal 12 MB=0x60183d4d4e180 00000400:00000200:24.0:1616513079.738823:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.247@tcp1 of length 224/224 into md 0x865bbec1 [1] + 24304 00000400:00000200:24.0:1616513079.738828:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.738831:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513079.738838:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.738841:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.738843:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.738845:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515583328256 00010000:00000200:20.0:1616513079.738856:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1690515583328256/t0(0) o400->ceda7c9f-d7ad-f5fe-267b-68a8de906fcc@10.31.141.247@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.738871:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690515583328256, offset 192 00000400:00000200:20.0:1616513079.738877:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.247@tcp1 00000400:00000200:20.0:1616513079.738884:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.247@tcp1 routed destination 00000400:00000200:20.0:1616513079.738899:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.247@tcp1(10.31.141.247@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513079.738903:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515583328640 00000800:00000200:20.0:1616513079.738906:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.738911:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.738914:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.738916:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513079.738916:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d4c80 x1690515583328640/t0(0) o400->54fc719f-ad21-639d-b4e2-9720184a9f2d@10.31.141.247@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513079.738921:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513079.738927:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.738928:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513079.738930:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690515583328640, offset 192 00000400:00000200:0.0:1616513079.738938:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.247@tcp1 00000800:00000200:30.0:1616513079.738939:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.738943:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:0.0:1616513079.738945:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.247@tcp1 routed destination 00000400:00000200:30.0:1616513079.738958:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513079.738963:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:0.0:1616513079.738967:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.247@tcp1(10.31.141.247@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513079.738968:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee235c8 00000800:00000200:30.0:1616513079.738971:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.738973:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513079.738974:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.738980:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.738983:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.738985:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.738990:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.738994:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.739048:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.739057:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513079.739059:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.739060:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513079.739066:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513079.739070:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:12.0:1616513079.739074:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513079.754656:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925664, portal 4 00000100:00000200:26.0:1616513079.754662:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925664, offset 0 00000400:00000200:26.0:1616513079.754669:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:26.0:1616513079.754675:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:26.0:1616513079.754688:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513079.754694:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:26.0:1616513079.754700:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513079.754705:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:26.0:1616513079.754708:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:26.0:1616513079.754712:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513079.754769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:11.2:1616513079.754789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.754798:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513079.754803:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513079.754806:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.754812:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513079.754818:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74dd218480 x1693067725925664/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513079.754839:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afed0 00000800:00000200:8.0:1616513079.754843:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513079.754845:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.754851:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:8.0:1616513079.754853:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.754854:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513079.754857:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513079.754863:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.754863:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.156@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.754864:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.156@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.754874:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.156@tcp1 of length 224 into portal 26 MB=0x5ffdb7d881680 00000400:00000200:8.0:1616513079.754874:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.156@tcp1 of length 224 into portal 12 MB=0x5ffdb7d8818c0 00000400:00000200:24.0:1616513079.754880:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.156@tcp1 of length 224/224 into md 0xf97c8599 [1] + 0 00000400:00000200:8.0:1616513079.754880:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.156@tcp1 of length 224/224 into md 0x865bbec1 [1] + 24528 00000400:00000200:8.0:1616513079.754884:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.754886:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240908 00000100:00000200:8.0:1616513079.754887:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513079.754888:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.754891:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513079.754893:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513079.754895:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240908 00000800:00000200:8.0:1616513079.754895:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.754897:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.754898:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.754900:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.754902:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:5.2:1616513079.754941:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513079.754953:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:0.0:1616513079.754958:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693052545216 00000800:00000200:28.0:1616513079.754960:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:12.0:1616513079.754963:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513079.754970:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00010000:00000200:0.0:1616513079.754971:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3f00 x1688693052545216/t0(0) o400->572124c0-f09c-987b-6dcb-a2021a5826ad@10.31.140.156@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513079.754973:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693052544640 00000400:00000200:28.0:1616513079.754978:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626d20 00000400:00000200:28.0:1616513079.754984:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d5751 [1] + 192 00010000:00000200:20.0:1616513079.754984:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688693052544640/t0(0) o400->159558c6-eb79-c978-3953-39a2f6c92c61@10.31.140.156@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513079.754989:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513079.754989:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693052545216, offset 192 00000100:00000200:28.0:1616513079.754994:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74dd218480 x1693067725925664/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513079.755008:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:0.0:1616513079.755010:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.156@tcp1 00000800:00000200:28.0:1616513079.755011:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513079.755013:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:20.0:1616513079.755013:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693052544640, offset 192 00000100:00000200:26.0:1616513079.755017:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74dd218480 x1693067725925664/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513079.755018:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.156@tcp1 routed destination 00000100:00000200:26.0:1616513079.755030:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74dd218480 x1693067725925664/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513079.755033:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.156@tcp1(10.31.140.156@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513079.755040:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa90 00000800:00000200:0.0:1616513079.755040:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513079.755045:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.156@tcp1 00000800:00000200:0.0:1616513079.755045:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.755048:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.755051:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513079.755052:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.156@tcp1 routed destination 00000800:00000200:0.0:1616513079.755056:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.755064:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513079.755066:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.156@tcp1(10.31.140.156@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.755072:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.755077:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513079.755078:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513079.755080:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.755082:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513079.755084:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513079.755086:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513079.755087:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513079.755092:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513079.755093:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513079.755094:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.755098:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb055c8 00000800:00000200:8.0:1616513079.755101:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513079.755102:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513079.755105:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.755106:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.755107:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513079.755109:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513079.755111:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.755115:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22880 00000800:00000200:8.0:1616513079.755117:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.769847:0:49416:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513079.769859:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.769866:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513079.769873:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.236@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.769880:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.236@tcp1 of length 224 into portal 12 MB=0x60466f9695480 00000400:00000200:30.0:1616513079.769886:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.236@tcp1 of length 224/224 into md 0xf96450b9 [1] + 91960 00000400:00000200:30.0:1616513079.769890:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.769893:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513079.769899:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.769902:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513079.769903:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.769905:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.769963:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693690177868928 00010000:00000200:14.0:1616513079.769976:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7d100 x1693690177868928/t0(0) o400->a1b14576-c6b1-997c-aecf-9cb3866bb41b@10.31.136.236@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.769989:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693690177868928, offset 192 00000400:00000200:14.0:1616513079.769994:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.236@tcp1 00000400:00000200:14.0:1616513079.770002:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.236@tcp1 routed destination 00000400:00000200:14.0:1616513079.770015:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.236@tcp1(10.31.136.236@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.770021:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513079.770027:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.770029:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513079.770031:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513079.770036:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.770041:0:49416:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513079.770062:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513079.770104:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.770111:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.770114:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513079.770120:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513079.770121:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.770124:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756ff0 00000800:00000200:2.0:1616513079.770128:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513079.770130:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513079.770131:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513079.770139:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.236@tcp1 : PUT - for me 00000400:00000200:12.0:1616513079.770148:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.236@tcp1 of length 224 into portal 26 MB=0x60466f9695400 00000400:00000200:12.0:1616513079.770155:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.236@tcp1 of length 224/224 into md 0xf97c85a9 [1] + 0 00000400:00000200:12.0:1616513079.770160:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240f68 00000400:00000200:12.0:1616513079.770163:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513079.770165:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513079.770170:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240f68 00000800:00000200:12.0:1616513079.770173:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.770176:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513079.770176:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513079.770181:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.770233:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693690177868800 00010000:00000200:20.0:1616513079.770243:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1693690177868800/t0(0) o400->b088678d-eea1-0fff-f5a7-7bdc2c6f01c8@10.31.136.236@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.770256:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693690177868800, offset 192 00000400:00000200:20.0:1616513079.770262:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.236@tcp1 00000400:00000200:20.0:1616513079.770269:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.236@tcp1 routed destination 00000400:00000200:20.0:1616513079.770283:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.236@tcp1(10.31.136.236@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.770289:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.770294:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.770297:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.770299:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.770303:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.770308:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513079.770309:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513079.770362:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.770369:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.770371:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513079.770372:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.770378:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.770384:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23c28 00000800:00000200:24.0:1616513079.770387:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.804821:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513079.804876:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.804885:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513079.804893:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.233@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.804902:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.233@tcp1 of length 224 into portal 26 MB=0x60182b48ce5c0 00000400:00000200:30.0:1616513079.804909:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.233@tcp1 of length 224/224 into md 0xf97c85b9 [1] + 0 00000400:00000200:30.0:1616513079.804913:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70440 00000400:00000200:30.0:1616513079.804916:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.804918:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513079.804922:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70440 00000800:00000200:30.0:1616513079.804925:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.804928:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513079.804929:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.804932:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.804987:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510746772928 00010000:00000200:20.0:1616513079.804999:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1690510746772928/t0(0) o400->4fba629a-4f41-49ec-45c5-8853453c92ce@10.31.132.233@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.805013:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510746772928, offset 192 00000400:00000200:20.0:1616513079.805020:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.233@tcp1 00000400:00000200:20.0:1616513079.805027:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.233@tcp1 routed destination 00000400:00000200:20.0:1616513079.805042:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.233@tcp1(10.31.132.233@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.805049:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.805055:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.805058:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.805060:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.805065:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.805070:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.805071:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513079.805124:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.805131:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.805134:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.805140:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.805146:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23c28 00000800:00000200:2.0:1616513079.805149:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.805152:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.805158:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513079.805161:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513079.805166:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.233@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.805174:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.233@tcp1 of length 224 into portal 12 MB=0x60182b48ce780 00000400:00000200:2.0:1616513079.805180:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.233@tcp1 of length 224/224 into md 0xf96450b9 [1] + 92184 00000400:00000200:2.0:1616513079.805186:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.805188:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513079.805194:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.805197:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.805199:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.805268:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510746773376 00010000:00000200:14.0:1616513079.805280:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21d100 x1690510746773376/t0(0) o400->83c8c562-9afc-0483-082c-d781f18d0250@10.31.132.233@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.805295:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510746773376, offset 192 00000400:00000200:14.0:1616513079.805302:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.233@tcp1 00000400:00000200:14.0:1616513079.805309:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.233@tcp1 routed destination 00000400:00000200:14.0:1616513079.805324:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.233@tcp1(10.31.132.233@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.805332:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.805338:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.805341:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.805343:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.805348:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.805395:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.805469:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.805475:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.805478:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.805486:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.805490:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756ff0 00000800:00000200:28.0:1616513079.805494:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.805497:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.822739:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.822794:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.822803:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513079.822805:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.822813:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.32@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.822823:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.32@tcp1 of length 224 into portal 26 MB=0x6041a0cfbd080 00000400:00000200:22.0:1616513079.822829:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.32@tcp1 of length 224/224 into md 0xf97c85d9 [1] + 0 00000400:00000200:22.0:1616513079.822834:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f706e8 00000400:00000200:22.0:1616513079.822837:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.822839:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.822844:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f706e8 00000800:00000200:22.0:1616513079.822846:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.822849:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.822851:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.822907:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359793754240 00010000:00000200:20.0:1616513079.822918:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1693359793754240/t0(0) o400->8e571246-de70-82fc-7a85-5abeebea7ceb@10.31.129.32@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.822932:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693359793754240, offset 192 00000400:00000200:20.0:1616513079.822940:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.32@tcp1 00000400:00000200:20.0:1616513079.822948:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.32@tcp1 routed destination 00000400:00000200:20.0:1616513079.822964:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.32@tcp1(10.31.129.32@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.822970:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:7.2:1616513079.822974:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.822976:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.822979:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.822981:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513079.822986:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:20.0:1616513079.822993:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513079.823027:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.823036:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513079.823045:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.32@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.823054:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.32@tcp1 of length 224 into portal 12 MB=0x6041a0cfbd400 00000400:00000200:18.0:1616513079.823061:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.32@tcp1 of length 224/224 into md 0xf96450b9 [1] + 92408 00000400:00000200:18.0:1616513079.823066:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.823069:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.823076:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513079.823079:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513079.823080:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:18.0:1616513079.823087:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513079.823088:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513079.823090:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513079.823091:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513079.823098:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.823103:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee235c8 00000800:00000200:2.0:1616513079.823107:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.823139:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359793755136 00010000:00000200:14.0:1616513079.823152:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfec00 x1693359793755136/t0(0) o400->fb202663-9142-9874-c584-cf777eefc15a@10.31.129.32@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.823166:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693359793755136, offset 192 00000400:00000200:14.0:1616513079.823173:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.32@tcp1 00000400:00000200:14.0:1616513079.823180:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.32@tcp1 routed destination 00000400:00000200:14.0:1616513079.823195:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.32@tcp1(10.31.129.32@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.823202:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.823207:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.823210:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.823212:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.823217:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.823263:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.823275:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.823281:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.823283:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.823289:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.823292:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47573a8 00000800:00000200:8.0:1616513079.823296:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.823299:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.828230:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.828283:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.828293:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.828293:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513079.828302:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.170@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.828311:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.170@tcp1 of length 224 into portal 26 MB=0x5ffe50e311280 00000400:00000200:22.0:1616513079.828317:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.170@tcp1 of length 224/224 into md 0xf97c85e9 [1] + 0 00000400:00000200:22.0:1616513079.828322:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71e48 00000400:00000200:22.0:1616513079.828326:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.828329:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.828333:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71e48 00000800:00000200:22.0:1616513079.828336:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.828339:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.828340:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.828397:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688734134243968 00000800:00000200:11.2:1616513079.828404:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:20.0:1616513079.828408:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688734134243968/t0(0) o400->b1b4b62c-82f3-15ae-4216-6169e4b60f73@10.31.141.170@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.828422:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688734134243968, offset 192 00000400:00000200:20.0:1616513079.828428:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.170@tcp1 00000400:00000200:20.0:1616513079.828447:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.170@tcp1 routed destination 00000800:00000200:28.0:1616513079.828459:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513079.828462:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.170@tcp1(10.31.141.170@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513079.828469:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513079.828469:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.828475:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513079.828478:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.170@tcp1 : PUT - for me 00000800:00000200:20.0:1616513079.828478:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.828480:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.828484:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513079.828486:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.170@tcp1 of length 224 into portal 12 MB=0x5ffe50e311440 00000800:00000200:7.2:1616513079.828489:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.828491:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513079.828493:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.170@tcp1 of length 224/224 into md 0x865bbec1 [1] + 24752 00000400:00000200:28.0:1616513079.828498:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.828500:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513079.828508:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.828511:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.828513:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.828516:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.828539:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688734134244416 00000800:00000200:18.0:1616513079.828545:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:0.0:1616513079.828551:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9f980 x1688734134244416/t0(0) o400->99e9cd1a-c620-0dd2-2c36-dfc573677dea@10.31.141.170@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513079.828552:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.828555:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.828561:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.828567:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22aa0 00000100:00000200:0.0:1616513079.828570:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688734134244416, offset 192 00000800:00000200:18.0:1616513079.828571:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.828574:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513079.828576:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.170@tcp1 00000400:00000200:0.0:1616513079.828583:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.170@tcp1 routed destination 00000400:00000200:0.0:1616513079.828598:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.170@tcp1(10.31.141.170@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.828605:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.828610:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.828615:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.828617:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.828622:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.828627:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.828635:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.828639:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.828641:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.828647:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.828650:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04dd0 00000800:00000200:8.0:1616513079.828653:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.828656:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.834792:0:49422:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513079.834804:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.834810:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513079.834818:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.97@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.834826:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.97@tcp1 of length 224 into portal 26 MB=0x5ffd51f863280 00000400:00000200:30.0:1616513079.834831:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.97@tcp1 of length 224/224 into md 0xf97c85f9 [1] + 0 00000400:00000200:30.0:1616513079.834835:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71100 00000400:00000200:30.0:1616513079.834838:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.834840:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513079.834843:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71100 00000800:00000200:30.0:1616513079.834846:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.834848:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513079.834849:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.834851:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.834855:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513079.834856:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513079.834861:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.97@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.834866:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.97@tcp1 of length 224 into portal 12 MB=0x5ffd51f8633c0 00000400:00000200:30.0:1616513079.834870:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.97@tcp1 of length 224/224 into md 0xf96450b9 [1] + 92632 00000400:00000200:30.0:1616513079.834874:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.834876:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513079.834881:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.834883:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.834884:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.834906:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665705558656 00010000:00000200:20.0:1616513079.834917:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688665705558656/t0(0) o400->cb769567-ddf3-1532-ca08-665e09759c1d@10.31.131.97@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.834931:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665705558656, offset 192 00000400:00000200:20.0:1616513079.834938:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.97@tcp1 00000400:00000200:20.0:1616513079.834945:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.97@tcp1 routed destination 00000100:00000200:14.0:1616513079.834948:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665705558976 00000400:00000200:20.0:1616513079.834960:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.97@tcp1(10.31.131.97@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:14.0:1616513079.834960:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf81b00 x1688665705558976/t0(0) o400->e2d5ec9f-8881-4802-d235-3e5d7c4a73c4@10.31.131.97@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513079.834967:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.834974:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:14.0:1616513079.834974:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665705558976, offset 192 00000800:00000200:20.0:1616513079.834977:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.834979:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:14.0:1616513079.834981:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.97@tcp1 00000800:00000200:20.0:1616513079.834984:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513079.834988:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.97@tcp1 routed destination 00000800:00000200:7.2:1616513079.834989:0:49422:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.834991:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513079.835003:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.97@tcp1(10.31.131.97@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.835009:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.835015:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.835018:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.835020:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.835025:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.835032:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513079.835042:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.835049:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.835052:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.835053:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513079.835058:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513079.835065:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23760 00000800:00000200:2.0:1616513079.835069:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513079.835088:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.835095:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513079.835098:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513079.835104:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513079.835108:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757b18 00000800:00000200:12.0:1616513079.835112:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.835115:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.835935:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.835944:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.835949:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.835956:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.9@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.835965:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.9@tcp1 of length 224 into portal 26 MB=0x5ffd4d2e5de00 00000400:00000200:8.0:1616513079.835971:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.9@tcp1 of length 224/224 into md 0xf97c8609 [1] + 0 00000400:00000200:8.0:1616513079.835975:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f718f8 00000400:00000200:8.0:1616513079.835977:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.835979:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513079.835983:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f718f8 00000800:00000200:8.0:1616513079.835985:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.835988:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.835989:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.835991:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.835996:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664419982848 00010000:00000200:20.0:1616513079.836001:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688664419982848/t0(0) o400->263bd3ca-4748-1ff2-fd02-25f6e0f85e71@10.31.138.9@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.836012:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664419982848, offset 192 00000400:00000200:20.0:1616513079.836017:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.9@tcp1 00000400:00000200:20.0:1616513079.836023:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.9@tcp1 routed destination 00000400:00000200:20.0:1616513079.836035:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.9@tcp1(10.31.138.9@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.836040:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.836044:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.836047:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.836048:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.836051:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513079.836055:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.836058:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.836068:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.836072:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513079.836074:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513079.836079:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513079.836083:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22000 00000800:00000200:30.0:1616513079.836085:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.836088:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513079.836152:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.836162:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.836168:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513079.836171:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513079.836176:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.9@tcp1 : PUT - for me 00000400:00000200:12.0:1616513079.836184:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.9@tcp1 of length 224 into portal 12 MB=0x5ffd4d2e5e100 00000400:00000200:12.0:1616513079.836190:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.9@tcp1 of length 224/224 into md 0x865bbec1 [1] + 24976 00000400:00000200:12.0:1616513079.836196:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513079.836198:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513079.836204:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.836208:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.836210:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.836286:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664419983616 00010000:00000200:0.0:1616513079.836298:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c667080 x1688664419983616/t0(0) o400->d2bbbd11-48b6-2b03-3e09-02d82d75c1a9@10.31.138.9@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513079.836313:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664419983616, offset 192 00000400:00000200:0.0:1616513079.836319:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.9@tcp1 00000400:00000200:0.0:1616513079.836326:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.9@tcp1 routed destination 00000400:00000200:0.0:1616513079.836342:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.9@tcp1(10.31.138.9@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.836348:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.836354:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.836357:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.836359:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.836364:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.836369:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.836380:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.836384:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.836386:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513079.836390:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513079.836390:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.836393:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:8.0:1616513079.836396:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:27.0:1616513079.866579:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925680, portal 4 00000100:00000200:27.0:1616513079.866585:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925680, offset 0 00000400:00000200:27.0:1616513079.866592:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:27.0:1616513079.866600:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:27.0:1616513079.866614:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513079.866621:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:27.0:1616513079.866629:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513079.866632:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:27.0:1616513079.866634:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:27.0:1616513079.866640:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513079.866687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513079.866764:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513079.866772:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513079.866775:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513079.866782:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513079.866789:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be43f00 x1693067725925680/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513079.866801:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248110 00000800:00000200:22.0:1616513079.866805:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513079.866809:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513079.866857:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513079.866909:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513079.866917:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513079.866923:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:30.0:1616513079.866927:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513079.866934:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626d30 00000400:00000200:30.0:1616513079.866940:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c8cf9 [1] + 192 00000400:00000200:30.0:1616513079.866945:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.866950:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be43f00 x1693067725925680/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513079.866963:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513079.866966:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:30.0:1616513079.866967:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:27.0:1616513079.866974:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be43f00 x1693067725925680/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513079.866987:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be43f00 x1693067725925680/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513079.866997:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248088 00000400:00000200:23.0:1616513079.870511:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:23.0:1616513079.870518:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513079.873286:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513079.873343:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.873352:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513079.873361:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.45@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.873372:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.45@tcp1 of length 224 into portal 26 MB=0x5ffd42cbb70c0 00000400:00000200:28.0:1616513079.873379:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.45@tcp1 of length 224/224 into md 0xf97c8629 [1] + 0 00000400:00000200:28.0:1616513079.873384:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240220 00000400:00000200:28.0:1616513079.873387:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.873390:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513079.873395:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240220 00000800:00000200:28.0:1616513079.873398:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.873399:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.873401:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.873406:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513079.873409:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.873410:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513079.873419:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.45@tcp1 : PUT - for me 00000400:00000200:12.0:1616513079.873428:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.45@tcp1 of length 224 into portal 12 MB=0x5ffd42cbb7140 00000400:00000200:12.0:1616513079.873452:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.45@tcp1 of length 224/224 into md 0x865bbec1 [1] + 25200 00000400:00000200:12.0:1616513079.873458:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513079.873460:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513079.873467:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.873471:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.873473:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.873475:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661632184512 00010000:00000200:20.0:1616513079.873487:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688661632184512/t0(0) o400->d039073a-6214-968a-66dd-d1536f525337@10.31.138.45@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.873503:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661632184512, offset 192 00000400:00000200:20.0:1616513079.873509:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.45@tcp1 00000400:00000200:20.0:1616513079.873517:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.45@tcp1 routed destination 00000400:00000200:20.0:1616513079.873533:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.45@tcp1(10.31.138.45@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513079.873534:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661632184640 00000800:00000200:20.0:1616513079.873539:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.873546:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513079.873548:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661f80 x1688661632184640/t0(0) o400->09b5278d-8bf3-fd01-5b83-ebee5b3d49b6@10.31.138.45@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513079.873549:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.873551:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.873557:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513079.873562:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661632184640, offset 192 00000800:00000200:7.2:1616513079.873563:0:49426:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.873565:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513079.873569:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.45@tcp1 00000400:00000200:0.0:1616513079.873576:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.45@tcp1 routed destination 00000400:00000200:0.0:1616513079.873591:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.45@tcp1(10.31.138.45@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.873598:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.873604:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.873606:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.873608:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513079.873613:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513079.873619:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513079.873619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513079.873625:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.873628:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513079.873630:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513079.873635:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:8.0:1616513079.873635:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.873638:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.873640:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513079.873641:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22a18 00000800:00000200:22.0:1616513079.873644:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513079.873645:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513079.873647:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513079.873648:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05210 00000800:00000200:8.0:1616513079.873652:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513079.882515:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925696, portal 4 00000100:00000200:23.0:1616513079.882520:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925696, offset 0 00000400:00000200:23.0:1616513079.882527:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:23.0:1616513079.882533:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:23.0:1616513079.882545:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513079.882551:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:23.0:1616513079.882558:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513079.882561:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:23.0:1616513079.882565:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:23.0:1616513079.882570:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513079.882616:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513079.882637:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513079.882643:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513079.882646:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513079.882653:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513079.882659:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63b600 x1693067725925696/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513079.882671:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f3a8 00000800:00000200:1.0:1616513079.882676:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513079.882679:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513079.882790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513079.882801:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513079.882807:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513079.882811:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513079.882817:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513079.882826:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626d40 00000400:00000200:1.0:1616513079.882832:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffb95 [1] + 192 00000400:00000200:1.0:1616513079.882837:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513079.882842:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63b600 x1693067725925696/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513079.882855:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513079.882858:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513079.882860:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:23.0:1616513079.882863:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63b600 x1693067725925696/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513079.882875:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63b600 x1693067725925696/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513079.882884:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eee0 00000100:00000200:7.0:1616513079.899155:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925712, portal 4 00000100:00000200:7.0:1616513079.899160:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925712, offset 0 00000400:00000200:7.0:1616513079.899167:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:7.0:1616513079.899173:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:7.0:1616513079.899186:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513079.899192:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:7.0:1616513079.899199:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513079.899202:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:7.0:1616513079.899204:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:7.0:1616513079.899209:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513079.899255:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513079.899312:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513079.899319:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:24.0:1616513079.899323:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513079.899329:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:24.0:1616513079.899336:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63a880 x1693067725925712/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513079.899350:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637fc28 00000800:00000200:24.0:1616513079.899354:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513079.899357:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513079.899445:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513079.899499:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513079.899509:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:8.0:1616513079.899511:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513079.899518:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513079.899528:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626d50 00000400:00000200:28.0:1616513079.899535:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d5821 [1] + 192 00000400:00000200:28.0:1616513079.899540:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.899545:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63a880 x1693067725925712/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513079.899561:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513079.899564:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513079.899566:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:7.0:1616513079.899568:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63a880 x1693067725925712/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513079.899581:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63a880 x1693067725925712/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513079.899592:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f7e8 00000800:00000200:11.2:1616513079.902972:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513079.903026:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.903035:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513079.903044:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.14@tcp1 : PUT - for me 00000400:00000200:24.0:1616513079.903054:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.14@tcp1 of length 224 into portal 26 MB=0x5ffd860172500 00000400:00000200:24.0:1616513079.903060:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.14@tcp1 of length 224/224 into md 0xf97c8649 [1] + 0 00000400:00000200:24.0:1616513079.903065:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240bb0 00000400:00000200:24.0:1616513079.903068:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513079.903070:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513079.903075:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240bb0 00000800:00000200:24.0:1616513079.903078:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.903079:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513079.903081:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513079.903084:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513079.903140:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688679673701632 00000800:00000200:7.2:1616513079.903149:0:49429:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:20.0:1616513079.903153:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688679673701632/t0(0) o400->55f9dbe6-dba3-a573-ad07-1af085de0c8d@10.31.133.14@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.903168:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688679673701632, offset 192 00000400:00000200:20.0:1616513079.903174:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.14@tcp1 00000400:00000200:20.0:1616513079.903182:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.14@tcp1 routed destination 00000400:00000200:20.0:1616513079.903198:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.14@tcp1(10.31.133.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.903204:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513079.903207:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.903211:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.903213:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513079.903216:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513079.903216:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513079.903219:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:20.0:1616513079.903221:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:7.2:1616513079.903226:0:49429:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:20.0:1616513079.903227:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513079.903228:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.14@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.903237:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.14@tcp1 of length 224 into portal 12 MB=0x5ffd860172580 00000400:00000200:18.0:1616513079.903243:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.14@tcp1 of length 224/224 into md 0xf96450b9 [1] + 92856 00000400:00000200:18.0:1616513079.903249:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.903251:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.903259:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513079.903262:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513079.903264:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.903267:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.903271:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.903273:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.903279:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513079.903280:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.903283:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22110 00000800:00000200:18.0:1616513079.903286:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.903354:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688679673701760 00010000:00000200:14.0:1616513079.903378:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfc380 x1688679673701760/t0(0) o400->82069d9b-66b0-16a2-8498-124835ba5421@10.31.133.14@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.903392:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688679673701760, offset 192 00000400:00000200:14.0:1616513079.903397:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.14@tcp1 00000400:00000200:14.0:1616513079.903404:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.14@tcp1 routed destination 00000400:00000200:14.0:1616513079.903419:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.14@tcp1(10.31.133.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.903425:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.903430:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.903433:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.903451:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.903456:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.903462:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.903476:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.903481:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.903483:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.903489:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.903493:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756e58 00000800:00000200:8.0:1616513079.903496:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.903499:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.916682:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513079.916694:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.916702:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513079.916711:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.186@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.916719:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.186@tcp1 of length 224 into portal 26 MB=0x5ffd532a3f500 00000400:00000200:30.0:1616513079.916725:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.186@tcp1 of length 224/224 into md 0xf97c8689 [1] + 0 00000400:00000200:30.0:1616513079.916730:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241760 00000400:00000200:30.0:1616513079.916733:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.916735:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513079.916739:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241760 00000800:00000200:30.0:1616513079.916742:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.916744:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513079.916746:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.916748:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.916803:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666026276096 00010000:00000200:20.0:1616513079.916814:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688666026276096/t0(0) o400->5bc81df9-0844-e27e-bdfa-ada5ab823d9e@10.31.136.186@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.916828:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666026276096, offset 192 00000400:00000200:20.0:1616513079.916835:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.186@tcp1 00000400:00000200:20.0:1616513079.916842:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.186@tcp1 routed destination 00000400:00000200:20.0:1616513079.916858:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.186@tcp1(10.31.136.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.916864:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.916870:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.916873:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.916875:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.916880:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.916885:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.916887:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513079.916940:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.916947:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.916950:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513079.916956:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.916962:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23078 00000800:00000200:18.0:1616513079.916966:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.916969:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.917068:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513079.917076:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.917081:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513079.917084:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513079.917088:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.186@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.917094:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.186@tcp1 of length 224 into portal 12 MB=0x5ffd532a3f640 00000400:00000200:30.0:1616513079.917098:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.186@tcp1 of length 224/224 into md 0xf96450b9 [1] + 93080 00000400:00000200:30.0:1616513079.917102:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.917104:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513079.917109:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.917111:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.917113:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.917175:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666026276416 00010000:00000200:14.0:1616513079.917188:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf80480 x1688666026276416/t0(0) o400->e35ce482-dfed-6878-4303-7dafe00e56e3@10.31.136.186@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.917202:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666026276416, offset 192 00000400:00000200:14.0:1616513079.917208:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.186@tcp1 00000400:00000200:14.0:1616513079.917215:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.186@tcp1 routed destination 00000400:00000200:14.0:1616513079.917231:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.186@tcp1(10.31.136.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.917237:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.917243:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.917246:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.917248:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.917253:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.917260:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.917320:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.917326:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513079.917329:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513079.917335:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513079.917339:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757650 00000800:00000200:12.0:1616513079.917344:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.917347:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.919797:0:49431:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513079.919852:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.919861:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513079.919870:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.235@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.919879:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.235@tcp1 of length 224 into portal 26 MB=0x5ffd43b7b9100 00000400:00000200:18.0:1616513079.919886:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.235@tcp1 of length 224/224 into md 0xf97c8699 [1] + 0 00000400:00000200:18.0:1616513079.919891:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241078 00000400:00000200:18.0:1616513079.919893:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.919896:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513079.919900:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241078 00000800:00000200:18.0:1616513079.919903:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.919906:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.919908:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.919913:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.919916:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.919917:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.919925:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.235@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.919935:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.235@tcp1 of length 224 into portal 12 MB=0x5ffd43b7b9180 00000400:00000200:2.0:1616513079.919941:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.235@tcp1 of length 224/224 into md 0xf96450b9 [1] + 93304 00000400:00000200:2.0:1616513079.919946:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.919949:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513079.919955:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.919958:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.919960:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.919963:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661879656704 00010000:00000200:20.0:1616513079.919974:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688661879656704/t0(0) o400->f2ceb410-89b3-ce36-47d0-5e5be2b6f18e@10.31.132.235@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.919988:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661879656704, offset 192 00000400:00000200:20.0:1616513079.919996:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.235@tcp1 00000400:00000200:20.0:1616513079.920003:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.235@tcp1 routed destination 00000400:00000200:20.0:1616513079.920018:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.235@tcp1(10.31.132.235@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:14.0:1616513079.920019:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661879656832 00000800:00000200:20.0:1616513079.920025:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.920031:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:14.0:1616513079.920031:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21ad00 x1688661879656832/t0(0) o400->be6076d0-7dcf-823e-e996-9aa514d3ea44@10.31.132.235@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513079.920034:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.920036:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.920041:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.920045:0:49431:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:14.0:1616513079.920046:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661879656832, offset 192 00000100:00000200:20.0:1616513079.920047:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513079.920052:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.235@tcp1 00000800:00000200:30.0:1616513079.920054:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.920058:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513079.920059:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.235@tcp1 routed destination 00000400:00000200:30.0:1616513079.920060:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513079.920063:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513079.920066:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513079.920070:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ff0 00000800:00000200:30.0:1616513079.920073:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:14.0:1616513079.920076:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.235@tcp1(10.31.132.235@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.920083:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.920088:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.920091:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.920093:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.920098:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.920104:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.920113:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.920117:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.920119:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.920125:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.920128:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757870 00000800:00000200:8.0:1616513079.920130:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.920133:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.927067:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513079.927123:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513079.927132:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513079.927141:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.21@tcp1 : PUT - for me 00000400:00000200:2.0:1616513079.927150:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.21@tcp1 of length 224 into portal 26 MB=0x5ffd4505ea500 00000400:00000200:2.0:1616513079.927156:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.21@tcp1 of length 224/224 into md 0xf97c86a9 [1] + 0 00000400:00000200:2.0:1616513079.927161:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241a08 00000400:00000200:2.0:1616513079.927164:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513079.927166:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513079.927171:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241a08 00000800:00000200:2.0:1616513079.927174:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.927175:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513079.927177:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513079.927180:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.927234:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662230082816 00010000:00000200:20.0:1616513079.927245:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688662230082816/t0(0) o400->529e589f-6000-5afa-1de2-80e257b5b8c3@10.31.133.21@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.927259:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662230082816, offset 192 00000400:00000200:20.0:1616513079.927265:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.21@tcp1 00000400:00000200:20.0:1616513079.927272:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.21@tcp1 routed destination 00000400:00000200:20.0:1616513079.927287:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.21@tcp1(10.31.133.21@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.927294:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.927300:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.927303:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.927305:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.927309:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.927315:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.927316:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513079.927369:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.927376:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.927379:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513079.927380:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.927385:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.927391:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22f68 00000800:00000200:18.0:1616513079.927394:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.927462:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.927473:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.927481:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513079.927482:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513079.927490:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.21@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.927499:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.21@tcp1 of length 224 into portal 12 MB=0x5ffd4505ea640 00000400:00000200:22.0:1616513079.927505:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.21@tcp1 of length 224/224 into md 0xf96450b9 [1] + 93528 00000400:00000200:22.0:1616513079.927510:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.927515:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513079.927522:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.927525:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.927526:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.927584:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662230083136 00010000:00000200:14.0:1616513079.927595:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec207500 x1688662230083136/t0(0) o400->27e7ca02-0666-15ad-c6c3-2ad33a190387@10.31.133.21@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.927608:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662230083136, offset 192 00000400:00000200:14.0:1616513079.927614:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.21@tcp1 00000400:00000200:14.0:1616513079.927621:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.21@tcp1 routed destination 00000400:00000200:14.0:1616513079.927635:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.21@tcp1(10.31.133.21@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.927641:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.927646:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.927649:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.927651:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.927655:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.927663:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.927718:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513079.927724:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513079.927727:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513079.927733:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513079.927737:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756880 00000800:00000200:12.0:1616513079.927741:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.927744:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.927750:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513079.927752:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.927752:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.927754:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513079.927756:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513079.946596:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925728, portal 4 00000100:00000200:30.0:1616513079.946600:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925728, offset 0 00000400:00000200:30.0:1616513079.946606:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513079.946612:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513079.946623:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513079.946630:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513079.946635:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513079.946637:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513079.946639:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513079.946644:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513079.946689:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513079.946747:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513079.946754:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513079.946757:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513079.946758:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513079.946764:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513079.946770:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74dd21ec00 x1693067725925728/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513079.946782:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee770 00000800:00000200:28.0:1616513079.946786:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513079.946872:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:11.2:1616513079.946918:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.946927:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513079.946937:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:8.0:1616513079.946938:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.946944:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513079.946946:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:28.0:1616513079.946947:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513079.946950:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513079.946952:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.2@tcp1 : PUT - for me 00000400:00000200:12.0:1616513079.946956:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626d60 00000400:00000200:8.0:1616513079.946961:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.2@tcp1 of length 224 into portal 26 MB=0x5ffd6c789cac0 00000400:00000200:12.0:1616513079.946963:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d5871 [1] + 192 00000400:00000200:8.0:1616513079.946967:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.2@tcp1 of length 224/224 into md 0xf97c86c9 [1] + 0 00000400:00000200:12.0:1616513079.946968:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.946971:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782405d8 00000100:00000200:12.0:1616513079.946973:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74dd21ec00 x1693067725925728/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513079.946973:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.946976:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513079.946980:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782405d8 00000800:00000200:8.0:1616513079.946983:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.946985:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.946987:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513079.946988:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513079.946991:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513079.946992:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513079.946994:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74dd21ec00 x1693067725925728/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513079.947006:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74dd21ec00 x1693067725925728/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513086 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513079.947015:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000100:00000200:20.0:1616513079.947042:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672819333824 00010000:00000200:20.0:1616513079.947052:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688672819333824/t0(0) o400->67a22219-923e-b71f-6494-38bd9a27fb96@10.31.130.2@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.947065:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688672819333824, offset 192 00000400:00000200:20.0:1616513079.947071:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.2@tcp1 00000400:00000200:20.0:1616513079.947079:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.2@tcp1 routed destination 00000400:00000200:20.0:1616513079.947092:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.2@tcp1(10.31.130.2@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.947098:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.947106:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.947108:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.947110:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.947115:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513079.947121:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513079.947122:0:49434:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.947132:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.947142:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513079.947144:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513079.947146:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:30.0:1616513079.947149:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513079.947152:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22c38 00000800:00000200:30.0:1616513079.947155:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513079.947155:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513079.947157:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513079.947161:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513079.947165:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513079.947169:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.2@tcp1 : PUT - for me 00000400:00000200:28.0:1616513079.947177:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.2@tcp1 of length 224 into portal 12 MB=0x5ffd6c789cc80 00000800:00000200:7.2:1616513079.947181:0:49434:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513079.947183:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.2@tcp1 of length 224/224 into md 0x865bbec1 [1] + 25424 00000400:00000200:28.0:1616513079.947188:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513079.947191:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.947192:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513079.947198:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.947200:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.947202:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513079.947202:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513079.947211:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.105@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.947220:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.105@tcp1 of length 224 into portal 26 MB=0x5ffd404359900 00000400:00000200:18.0:1616513079.947226:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.105@tcp1 of length 224/224 into md 0xf97c86d9 [1] + 0 00000400:00000200:18.0:1616513079.947231:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241188 00000400:00000200:18.0:1616513079.947233:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.947236:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513079.947240:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241188 00000800:00000200:18.0:1616513079.947243:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.947245:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513079.947247:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.947250:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.947252:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660952324352 00010000:00000200:20.0:1616513079.947258:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688660952324352/t0(0) o400->7fabb374-c64d-0008-4fb4-bf2baf859786@10.31.128.105@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513079.947262:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672819334272 00000100:00000200:20.0:1616513079.947268:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660952324352, offset 192 00000400:00000200:20.0:1616513079.947273:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.105@tcp1 00010000:00000200:0.0:1616513079.947275:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9d100 x1688672819334272/t0(0) o400->e88ee8c6-23d0-3091-0a29-285eb0e65a3a@10.31.130.2@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513079.947279:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.105@tcp1 routed destination 00000400:00000200:20.0:1616513079.947291:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.105@tcp1(10.31.128.105@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.947296:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:0.0:1616513079.947300:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688672819334272, offset 192 00000800:00000200:20.0:1616513079.947301:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.947303:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513079.947305:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513079.947306:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.2@tcp1 00000800:00000200:20.0:1616513079.947308:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513079.947312:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513079.947313:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513079.947314:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.2@tcp1 routed destination 00000800:00000200:8.0:1616513079.947320:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.947324:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.947326:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513079.947328:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513079.947329:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.2@tcp1(10.31.130.2@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513079.947331:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.947335:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22220 00000800:00000200:0.0:1616513079.947335:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513079.947337:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513079.947341:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.947344:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.947346:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513079.947351:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.947355:0:49434:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.947363:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.947367:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513079.947368:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513079.947372:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513079.947374:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513079.947377:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05870 00000800:00000200:30.0:1616513079.947379:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.947531:0:49434:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513079.947540:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.947546:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513079.947550:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.947552:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.105@tcp1 : PUT - for me 00000400:00000200:18.0:1616513079.947558:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.105@tcp1 of length 224 into portal 12 MB=0x5ffd404359d80 00000400:00000200:18.0:1616513079.947564:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.105@tcp1 of length 224/224 into md 0xf96450b9 [1] + 93752 00000400:00000200:18.0:1616513079.947567:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513079.947569:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.947574:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513079.947576:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513079.947578:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.947638:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660952325504 00010000:00000200:14.0:1616513079.947649:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfcc80 x1688660952325504/t0(0) o400->de217ff8-9e7a-5457-49de-5c9005641af7@10.31.128.105@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.947663:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660952325504, offset 192 00000400:00000200:14.0:1616513079.947669:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.105@tcp1 00000400:00000200:14.0:1616513079.947676:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.105@tcp1 routed destination 00000400:00000200:14.0:1616513079.947690:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.105@tcp1(10.31.128.105@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.947696:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.947701:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.947703:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.947707:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.947712:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.947716:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513079.947726:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513079.947731:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.947733:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.947735:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.947740:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.947744:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756cc0 00000800:00000200:24.0:1616513079.947748:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.957332:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513079.957386:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513079.957396:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513079.957397:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.957404:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513079.957405:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.83@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.957411:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.83@tcp1 : PUT - for me 00000400:00000200:12.0:1616513079.957414:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.83@tcp1 of length 224 into portal 26 MB=0x60183827f3cc0 00000400:00000200:8.0:1616513079.957419:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.83@tcp1 of length 224 into portal 12 MB=0x60183827f3dc0 00000400:00000200:12.0:1616513079.957421:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.83@tcp1 of length 224/224 into md 0xf97c86e9 [1] + 0 00000400:00000200:8.0:1616513079.957424:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.83@tcp1 of length 224/224 into md 0x865bbec1 [1] + 25648 00000400:00000200:12.0:1616513079.957426:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241b18 00000400:00000200:12.0:1616513079.957428:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.957429:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513079.957431:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:8.0:1616513079.957432:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513079.957445:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:12.0:1616513079.957448:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241b18 00000800:00000200:8.0:1616513079.957448:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513079.957450:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513079.957452:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513079.957454:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513079.957455:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513079.957457:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513079.957503:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514201984448 00010000:00000200:0.0:1616513079.957516:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d2d00 x1690514201984448/t0(0) o400->6dd2d385-f8a5-f131-1d22-3e685e6d4f09@10.31.131.83@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513079.957519:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514201984192 00010000:00000200:20.0:1616513079.957530:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1690514201984192/t0(0) o400->639e3650-15d7-9ddd-6614-c69c4a81b6a8@10.31.131.83@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513079.957530:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514201984448, offset 192 00000400:00000200:0.0:1616513079.957536:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.83@tcp1 00000100:00000200:20.0:1616513079.957544:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514201984192, offset 192 00000400:00000200:0.0:1616513079.957544:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.83@tcp1 routed destination 00000400:00000200:20.0:1616513079.957551:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.83@tcp1 00000400:00000200:20.0:1616513079.957558:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.83@tcp1 routed destination 00000400:00000200:0.0:1616513079.957559:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.83@tcp1(10.31.131.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513079.957566:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513079.957573:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.83@tcp1(10.31.131.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.957580:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513079.957581:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513079.957583:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513079.957585:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.957586:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.957588:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513079.957590:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513079.957591:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513079.957595:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513079.957597:0:49435:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513079.957601:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513079.957604:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513079.957653:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513079.957660:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513079.957663:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513079.957665:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513079.957670:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513079.957672:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513079.957674:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04660 00000800:00000200:2.0:1616513079.957678:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513079.957679:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513079.957682:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513079.957683:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513079.957688:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513079.957694:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23a08 00000800:00000200:24.0:1616513079.957697:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.980813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513079.980869:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513079.980878:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513079.980888:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.38@tcp1 : PUT - for me 00000400:00000200:22.0:1616513079.980897:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.38@tcp1 of length 224 into portal 26 MB=0x5ffd40ef53300 00000400:00000200:22.0:1616513079.980903:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.38@tcp1 of length 224/224 into md 0xf97c8709 [1] + 0 00000400:00000200:22.0:1616513079.980908:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782416d8 00000400:00000200:22.0:1616513079.980911:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513079.980913:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513079.980918:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782416d8 00000800:00000200:22.0:1616513079.980921:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.980922:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.980924:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.980926:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.980982:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661132653312 00000800:00000200:7.2:1616513079.980984:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:20.0:1616513079.980993:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688661132653312/t0(0) o400->ee435dc0-aa4c-1421-d1aa-705d91e036a6@10.31.138.38@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:30.0:1616513079.980995:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.981004:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:20.0:1616513079.981007:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661132653312, offset 192 00000400:00000200:30.0:1616513079.981012:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.38@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.981014:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.38@tcp1 00000400:00000200:30.0:1616513079.981020:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.38@tcp1 of length 224 into portal 12 MB=0x5ffd40ef53400 00000400:00000200:20.0:1616513079.981021:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.38@tcp1 routed destination 00000400:00000200:30.0:1616513079.981027:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.38@tcp1 of length 224/224 into md 0xf96450b9 [1] + 93976 00000400:00000200:30.0:1616513079.981032:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513079.981036:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.38@tcp1(10.31.138.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:30.0:1616513079.981050:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513079.981050:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.981056:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513079.981057:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.981059:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513079.981059:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513079.981061:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513079.981061:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513079.981063:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513079.981066:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.981071:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.981073:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513079.981082:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513079.981088:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513079.981090:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.981092:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.981097:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.981102:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23430 00000800:00000200:18.0:1616513079.981106:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.981122:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661132653568 00010000:00000200:14.0:1616513079.981134:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf87080 x1688661132653568/t0(0) o400->f8ae9f70-b2ec-a257-9cc6-fbdad6a17236@10.31.138.38@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.981148:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661132653568, offset 192 00000400:00000200:14.0:1616513079.981154:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.38@tcp1 00000400:00000200:14.0:1616513079.981161:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.38@tcp1 routed destination 00000400:00000200:14.0:1616513079.981176:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.38@tcp1(10.31.138.38@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.981183:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.981188:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.981191:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.981193:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.981198:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.981205:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.981216:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.981221:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513079.981223:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513079.981228:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513079.981232:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757f58 00000800:00000200:8.0:1616513079.981235:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.981237:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.982833:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513079.982841:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.982846:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513079.982852:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.36@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.982859:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.36@tcp1 of length 224 into portal 26 MB=0x6041a0e529e00 00000400:00000200:30.0:1616513079.982864:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.36@tcp1 of length 224/224 into md 0xf97c8729 [1] + 0 00000400:00000200:30.0:1616513079.982868:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241ba0 00000400:00000200:30.0:1616513079.982869:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.982871:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513079.982875:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241ba0 00000800:00000200:30.0:1616513079.982877:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513079.982879:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513079.982880:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.982882:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513079.982940:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359816220160 00010000:00000200:20.0:1616513079.982950:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1693359816220160/t0(0) o400->c54ee747-68dd-98be-a307-a6abe1a16571@10.31.129.36@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.982965:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693359816220160, offset 192 00000400:00000200:20.0:1616513079.982971:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.36@tcp1 00000400:00000200:20.0:1616513079.982979:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.36@tcp1 routed destination 00000400:00000200:20.0:1616513079.982994:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.36@tcp1(10.31.129.36@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.983001:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513079.983006:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513079.983009:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513079.983011:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513079.983015:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513079.983021:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513079.983022:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513079.983074:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513079.983081:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513079.983084:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513079.983090:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513079.983096:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22220 00000800:00000200:22.0:1616513079.983100:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513079.983102:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513079.983113:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513079.983122:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.983127:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513079.983128:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513079.983134:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.36@tcp1 : PUT - for me 00000400:00000200:30.0:1616513079.983139:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.36@tcp1 of length 224 into portal 12 MB=0x6041a0e52a280 00000400:00000200:30.0:1616513079.983143:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.36@tcp1 of length 224/224 into md 0xf96450b9 [1] + 94200 00000400:00000200:30.0:1616513079.983147:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513079.983149:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513079.983152:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513079.983154:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513079.983155:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513079.983216:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359816221312 00010000:00000200:14.0:1616513079.983228:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf82880 x1693359816221312/t0(0) o400->30898665-5bfe-c0d2-cd6c-a9d94415a7be@10.31.129.36@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513079.983242:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693359816221312, offset 192 00000400:00000200:14.0:1616513079.983249:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.36@tcp1 00000400:00000200:14.0:1616513079.983257:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.36@tcp1 routed destination 00000400:00000200:14.0:1616513079.983271:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.36@tcp1(10.31.129.36@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513079.983278:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513079.983284:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513079.983287:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513079.983289:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513079.983293:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.983298:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.983358:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.983366:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.983369:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.983375:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.983379:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756660 00000800:00000200:28.0:1616513079.983382:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.983385:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513079.992290:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513079.992303:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513079.992314:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513079.992322:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.201@tcp1 : PUT - for me 00000400:00000200:8.0:1616513079.992330:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.201@tcp1 of length 224 into portal 26 MB=0x6001be264d080 00000400:00000200:8.0:1616513079.992336:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.201@tcp1 of length 224/224 into md 0xf97c8739 [1] + 0 00000400:00000200:8.0:1616513079.992340:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240dd0 00000400:00000200:8.0:1616513079.992342:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513079.992345:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513079.992349:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240dd0 00000800:00000200:8.0:1616513079.992351:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513079.992354:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513079.992355:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513079.992358:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513079.992397:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513079.992412:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969622638720 00010000:00000200:20.0:1616513079.992423:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688969622638720/t0(0) o400->3e1f5ce0-b596-9b79-5026-623743cf424f@10.31.128.201@tcp1:36/0 lens 224/224 e 0 to 0 dl 1616513171 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513079.992451:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688969622638720, offset 192 00000800:00000200:22.0:1616513079.992453:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513079.992459:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.201@tcp1 00000800:00000200:22.0:1616513079.992462:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513079.992467:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.201@tcp1 routed destination 00000400:00000200:22.0:1616513079.992471:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.201@tcp1 : PUT - for me 00000400:00000200:20.0:1616513079.992482:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.201@tcp1(10.31.128.201@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513079.992489:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:22.0:1616513079.992492:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.201@tcp1 of length 224 into portal 12 MB=0x6001be264d4c0 00000800:00000200:20.0:1616513079.992495:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513079.992498:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.201@tcp1 of length 224/224 into md 0xf96450b9 [1] + 94424 00000800:00000200:20.0:1616513079.992498:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513079.992500:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:22.0:1616513079.992504:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513079.992505:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:22.0:1616513079.992506:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513079.992510:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000100:00000200:20.0:1616513079.992511:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513079.992514:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:18.0:1616513079.992518:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513079.992520:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000400:00000200:18.0:1616513079.992521:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513079.992522:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513079.992525:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513079.992528:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513079.992533:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22f68 00000800:00000200:18.0:1616513079.992537:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513079.992579:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969622639808 00010000:00000200:26.0:1616513079.992591:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec200d80 x1688969622639808/t0(0) o400->0ba0e701-c369-0ecd-5d35-6b2fbb26dca7@10.31.128.201@tcp1:705/0 lens 224/224 e 0 to 0 dl 1616513085 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513079.992605:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688969622639808, offset 192 00000400:00000200:26.0:1616513079.992612:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.201@tcp1 00000400:00000200:26.0:1616513079.992620:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.201@tcp1 routed destination 00000400:00000200:26.0:1616513079.992635:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.201@tcp1(10.31.128.201@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513079.992641:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513079.992647:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513079.992650:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513079.992652:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513079.992663:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513079.992668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513079.992723:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513079.992729:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513079.992732:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513079.992739:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513079.992743:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa90 00000800:00000200:28.0:1616513079.992747:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513079.992750:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.009357:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.009378:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.009386:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.009394:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.117@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.009404:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.117@tcp1 of length 224 into portal 26 MB=0x5ffd509692800 00000400:00000200:30.0:1616513080.009410:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.117@tcp1 of length 224/224 into md 0xf97c8759 [1] + 0 00000400:00000200:30.0:1616513080.009414:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d81782417e8 00000400:00000200:30.0:1616513080.009417:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.009419:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.009423:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81782417e8 00000800:00000200:30.0:1616513080.009425:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.009427:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.009429:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.009431:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.009487:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665334556672 00010000:00000200:20.0:1616513080.009498:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688665334556672/t0(0) o400->61659242-5880-df6d-ada7-d7018d5521e5@10.31.136.117@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.009528:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665334556672, offset 192 00000400:00000200:20.0:1616513080.009534:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.117@tcp1 00000400:00000200:20.0:1616513080.009541:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.117@tcp1 routed destination 00000400:00000200:20.0:1616513080.009555:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.117@tcp1(10.31.136.117@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.009561:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.009566:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.009569:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.009571:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.009576:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.009581:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.009582:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513080.009632:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.009638:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.009641:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.009647:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.009652:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23078 00000800:00000200:18.0:1616513080.009655:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.009658:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.009664:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.009665:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513080.009671:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.117@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.009678:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.117@tcp1 of length 224 into portal 12 MB=0x5ffd509692940 00000400:00000200:18.0:1616513080.009684:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.117@tcp1 of length 224/224 into md 0xf96450b9 [1] + 94648 00000400:00000200:18.0:1616513080.009689:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.009691:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513080.009697:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.009699:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.009701:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.009758:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665334556992 00010000:00000200:26.0:1616513080.009770:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfba80 x1688665334556992/t0(0) o400->9d6f474a-00a5-401f-895b-94fb4fe2140c@10.31.136.117@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.009783:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665334556992, offset 192 00000400:00000200:26.0:1616513080.009789:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.117@tcp1 00000400:00000200:26.0:1616513080.009796:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.117@tcp1 routed destination 00000400:00000200:26.0:1616513080.009810:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.117@tcp1(10.31.136.117@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.009816:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.009821:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.009823:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.009827:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.009832:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.009881:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.009937:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.009945:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.009947:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.009948:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.009954:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.009958:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afba0 00000800:00000200:12.0:1616513080.009962:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513080.026560:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925744, portal 4 00000100:00000200:10.0:1616513080.026566:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925744, offset 0 00000400:00000200:10.0:1616513080.026572:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:10.0:1616513080.026577:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:10.0:1616513080.026588:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513080.026594:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:10.0:1616513080.026600:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513080.026602:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:10.0:1616513080.026604:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:10.0:1616513080.026609:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513080.026657:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513080.026677:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513080.026683:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513080.026686:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513080.026693:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513080.026699:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdc57f980 x1693067725925744/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513080.026712:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82d48 00000800:00000200:27.0:1616513080.026717:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513080.026720:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513080.026838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513080.026848:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513080.026854:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513080.026862:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513080.026870:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626d70 00000400:00000200:27.0:1616513080.026879:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166b8d [1] + 192 00000400:00000200:27.0:1616513080.026884:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513080.026889:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdc57f980 x1693067725925744/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513080.026900:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513080.026904:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513080.026907:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513080.026908:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:10.0:1616513080.026910:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdc57f980 x1693067725925744/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513080.026921:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdc57f980 x1693067725925744/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513080.026930:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83078 00000800:00000200:11.2:1616513080.041159:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513080.041198:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513080.041215:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.041225:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.041234:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.137@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.041244:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.137@tcp1 of length 224 into portal 26 MB=0x5ffd49cf64a80 00000400:00000200:28.0:1616513080.041251:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.137@tcp1 of length 224/224 into md 0xf97c8769 [1] + 0 00000800:00000200:22.0:1616513080.041254:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513080.041256:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240b28 00000400:00000200:28.0:1616513080.041258:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.041261:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513080.041263:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513080.041269:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240b28 00000800:00000200:28.0:1616513080.041272:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.041272:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513080.041273:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.137@tcp1 : PUT - for me 00000800:00000200:30.0:1616513080.041274:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513080.041275:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513080.041277:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513080.041282:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.137@tcp1 of length 224 into portal 12 MB=0x5ffd49cf64e00 00000400:00000200:22.0:1616513080.041288:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.137@tcp1 of length 224/224 into md 0xf96450b9 [1] + 94872 00000400:00000200:22.0:1616513080.041293:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.041296:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513080.041303:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.041306:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.041307:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.041335:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663515089536 00010000:00000200:20.0:1616513080.041346:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688663515089536/t0(0) o400->6bb70b63-589b-4ae3-6756-2448f9ecbe8f@10.31.129.137@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.041361:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663515089536, offset 192 00000400:00000200:20.0:1616513080.041367:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.137@tcp1 00000100:00000200:26.0:1616513080.041368:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663515090432 00000400:00000200:20.0:1616513080.041374:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.137@tcp1 routed destination 00010000:00000200:26.0:1616513080.041381:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec203600 x1688663515090432/t0(0) o400->3df31a0d-e3ca-ecf1-01dd-50a22e33d4b2@10.31.129.137@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513080.041390:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.137@tcp1(10.31.129.137@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513080.041395:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663515090432, offset 192 00000800:00000200:20.0:1616513080.041401:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513080.041402:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.137@tcp1 00000800:00000200:20.0:1616513080.041407:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.041409:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:26.0:1616513080.041410:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.137@tcp1 routed destination 00000800:00000200:20.0:1616513080.041412:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.041417:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.041421:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.041423:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513080.041425:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.137@tcp1(10.31.129.137@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.041431:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.041450:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.041453:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.041455:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.041460:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.041464:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.041475:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513080.041476:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513080.041481:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.041484:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513080.041484:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.041487:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513080.041490:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.041493:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aea18 00000400:00000200:2.0:1616513080.041494:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:8.0:1616513080.041497:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.041500:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513080.041500:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22a18 00000800:00000200:2.0:1616513080.041503:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.041506:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.045614:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.045670:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.045679:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.045688:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.78@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.045699:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.78@tcp1 of length 224 into portal 26 MB=0x5ffd42be1ba00 00000400:00000200:24.0:1616513080.045707:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.78@tcp1 of length 224/224 into md 0xf97c8779 [1] + 0 00000400:00000200:24.0:1616513080.045712:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241320 00000400:00000200:24.0:1616513080.045715:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.045718:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.045722:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241320 00000800:00000200:28.0:1616513080.045723:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.045725:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.045728:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.045730:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.045786:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661617916416 00010000:00000200:20.0:1616513080.045797:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688661617916416/t0(0) o400->f811440c-f1ac-3a68-42e0-3d645ba8d138@10.31.138.78@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.045811:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661617916416, offset 192 00000400:00000200:20.0:1616513080.045817:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.78@tcp1 00000400:00000200:20.0:1616513080.045825:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.78@tcp1 routed destination 00000400:00000200:20.0:1616513080.045840:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.78@tcp1(10.31.138.78@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.045854:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.045860:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.045863:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.045865:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.045870:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.045876:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.045876:0:49444:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.045886:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.045890:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.045892:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.045898:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.045903:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23760 00000800:00000200:30.0:1616513080.045906:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.045908:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.046024:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.046032:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.046037:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.046046:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.78@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.046054:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.78@tcp1 of length 224 into portal 12 MB=0x5ffd42be1bb40 00000400:00000200:8.0:1616513080.046059:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.78@tcp1 of length 224/224 into md 0x865bbec1 [1] + 25872 00000400:00000200:8.0:1616513080.046064:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.046066:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513080.046073:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.046075:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.046077:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.046079:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.046137:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661617916736 00010000:00000200:0.0:1616513080.046149:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6300 x1688661617916736/t0(0) o400->50e6441e-c218-68df-8036-f55e4a8455c5@10.31.138.78@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.046163:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661617916736, offset 192 00000400:00000200:0.0:1616513080.046169:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.78@tcp1 00000400:00000200:0.0:1616513080.046176:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.78@tcp1 routed destination 00000400:00000200:0.0:1616513080.046190:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.78@tcp1(10.31.138.78@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.046198:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.046203:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.046206:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.046208:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.046212:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.046217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.046228:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.046233:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.046236:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.046238:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.046242:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.046246:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04660 00000800:00000200:28.0:1616513080.046250:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.054918:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513080.054976:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.054985:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513080.054994:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.53@tcp1 : PUT - for me 00000400:00000200:2.0:1616513080.055004:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.53@tcp1 of length 224 into portal 26 MB=0x5f908788b8180 00000400:00000200:2.0:1616513080.055010:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.53@tcp1 of length 224/224 into md 0xf97c8789 [1] + 0 00000400:00000200:2.0:1616513080.055015:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240aa0 00000400:00000200:2.0:1616513080.055018:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513080.055021:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513080.055025:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240aa0 00000800:00000200:2.0:1616513080.055028:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.055031:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513080.055031:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513080.055038:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513080.055041:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.055042:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.055051:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.53@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.055059:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.53@tcp1 of length 224 into portal 12 MB=0x5f908788b8200 00000400:00000200:18.0:1616513080.055068:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.53@tcp1 of length 224/224 into md 0xf96450b9 [1] + 95096 00000400:00000200:18.0:1616513080.055073:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.055076:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513080.055083:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.055086:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.055088:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681189661016448 00000800:00000200:18.0:1616513080.055088:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:20.0:1616513080.055099:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1681189661016448/t0(0) o400->4aae66d7-9265-af6a-1f89-926fa38b116e@10.31.131.53@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.055113:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681189661016448, offset 192 00000400:00000200:20.0:1616513080.055119:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.53@tcp1 00000400:00000200:20.0:1616513080.055127:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.53@tcp1 routed destination 00000400:00000200:20.0:1616513080.055142:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.53@tcp1(10.31.131.53@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513080.055148:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681189661016576 00000800:00000200:20.0:1616513080.055148:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.055154:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.055157:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:26.0:1616513080.055160:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf9200 x1681189661016576/t0(0) o400->503b0f43-2c85-fb9f-69cb-f9ae7e81dec4@10.31.131.53@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.055161:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.055166:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.055172:0:49445:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.055173:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513080.055174:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681189661016576, offset 192 00000400:00000200:26.0:1616513080.055181:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.53@tcp1 00000400:00000200:26.0:1616513080.055188:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.53@tcp1 routed destination 00000400:00000200:26.0:1616513080.055203:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.53@tcp1(10.31.131.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.055209:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.055215:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.055218:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.055220:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.055225:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513080.055227:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513080.055232:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513080.055234:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.055237:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.055237:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.055244:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.055249:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee235c8 00000800:00000200:8.0:1616513080.055250:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513080.055253:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513080.055255:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.055257:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513080.055263:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.055266:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af650 00000800:00000200:8.0:1616513080.055269:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.055271:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.057460:0:49445:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513080.057514:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.057524:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513080.057532:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.135@tcp1 : PUT - for me 00000400:00000200:2.0:1616513080.057541:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.135@tcp1 of length 224 into portal 12 MB=0x5ffd4580e3080 00000400:00000200:2.0:1616513080.057548:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.135@tcp1 of length 224/224 into md 0xf96450b9 [1] + 95320 00000400:00000200:2.0:1616513080.057553:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513080.057555:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513080.057562:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.057565:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513080.057566:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.057569:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.057625:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662359027840 00010000:00000200:26.0:1616513080.057638:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21cc80 x1688662359027840/t0(0) o400->315a2029-8787-f09b-fbc8-2a992637acde@10.31.136.135@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.057652:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662359027840, offset 192 00000400:00000200:26.0:1616513080.057660:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.135@tcp1 00000400:00000200:26.0:1616513080.057668:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.135@tcp1 routed destination 00000400:00000200:26.0:1616513080.057682:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.135@tcp1(10.31.136.135@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.057689:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513080.057694:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.057697:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513080.057699:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513080.057704:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.057709:0:49445:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.057719:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.057722:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.057724:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.057730:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.057733:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae330 00000800:00000200:30.0:1616513080.057735:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.057737:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.057811:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.057865:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.057875:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.057884:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.135@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.057892:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.135@tcp1 of length 224 into portal 26 MB=0x5ffd4580e3000 00000400:00000200:24.0:1616513080.057899:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.135@tcp1 of length 224/224 into md 0xf97c87a9 [1] + 0 00000400:00000200:24.0:1616513080.057904:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241430 00000400:00000200:24.0:1616513080.057906:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.057909:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.057913:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241430 00000800:00000200:24.0:1616513080.057916:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.057919:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513080.057921:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.057925:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.057975:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662359027712 00010000:00000200:20.0:1616513080.057986:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688662359027712/t0(0) o400->10178bef-6ccd-cc66-0931-2b79c0f91052@10.31.136.135@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.057999:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662359027712, offset 192 00000400:00000200:20.0:1616513080.058005:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.135@tcp1 00000400:00000200:20.0:1616513080.058012:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.135@tcp1 routed destination 00000400:00000200:20.0:1616513080.058028:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.135@tcp1(10.31.136.135@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.058034:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.058039:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.058042:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.058044:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.058048:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513080.058054:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513080.058054:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.058061:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.058064:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.058066:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513080.058071:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.058074:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee235c8 00000800:00000200:8.0:1616513080.058077:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.058079:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:2.0:1616513080.058498:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925760, portal 4 00000100:00000200:2.0:1616513080.058502:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925760, offset 0 00000400:00000200:2.0:1616513080.058507:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:2.0:1616513080.058512:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:2.0:1616513080.058526:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513080.058533:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:2.0:1616513080.058538:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513080.058541:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513080.058543:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513080.058547:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513080.058593:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513080.058607:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513080.058611:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513080.058613:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513080.058616:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513080.058619:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513080.058623:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dec202400 x1693067725925760/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513080.058635:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed320 00000800:00000200:2.0:1616513080.058638:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513080.058690:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513080.058702:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513080.058708:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513080.058712:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:30.0:1616513080.058715:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513080.058722:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626d80 00000400:00000200:30.0:1616513080.058728:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c8ee9 [1] + 192 00000400:00000200:30.0:1616513080.058733:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.058737:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dec202400 x1693067725925760/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513080.058750:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513080.058752:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:30.0:1616513080.058754:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:2.0:1616513080.058756:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dec202400 x1693067725925760/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513080.058768:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dec202400 x1693067725925760/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513080.058777:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eeded0 00000800:00000200:11.2:1616513080.063953:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.064009:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.064019:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.064029:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.33@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.064038:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.33@tcp1 of length 224 into portal 26 MB=0x5ffd47d0257c0 00000400:00000200:28.0:1616513080.064045:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.33@tcp1 of length 224/224 into md 0xf97c87b9 [1] + 0 00000400:00000200:28.0:1616513080.064049:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240ff0 00000400:00000200:28.0:1616513080.064052:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.064055:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513080.064059:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ff0 00000800:00000200:28.0:1616513080.064062:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.064062:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.064065:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513080.064069:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513080.064072:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.064073:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.064081:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.33@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.064089:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.33@tcp1 of length 224 into portal 12 MB=0x5ffd47d0258c0 00000400:00000200:24.0:1616513080.064095:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.33@tcp1 of length 224/224 into md 0x865bbec1 [1] + 26096 00000400:00000200:24.0:1616513080.064100:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.064102:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.064109:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.064112:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.064113:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.064119:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662979008448 00010000:00000200:20.0:1616513080.064130:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688662979008448/t0(0) o400->e4829543-020b-40df-3b06-b51b4371a123@10.31.138.33@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.064143:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662979008448, offset 192 00000400:00000200:20.0:1616513080.064149:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.33@tcp1 00000400:00000200:20.0:1616513080.064156:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.33@tcp1 routed destination 00000400:00000200:20.0:1616513080.064170:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.33@tcp1(10.31.138.33@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513080.064174:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662979008704 00000800:00000200:20.0:1616513080.064175:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.064181:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.064183:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.064185:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513080.064187:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d6c00 x1688662979008704/t0(0) o400->c24954c5-51ab-fc3b-10ed-fcbed759f9f7@10.31.138.33@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.064190:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.064196:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.064196:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513080.064201:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662979008704, offset 192 00000400:00000200:0.0:1616513080.064207:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.33@tcp1 00000400:00000200:0.0:1616513080.064215:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.33@tcp1 routed destination 00000400:00000200:0.0:1616513080.064230:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.33@tcp1(10.31.138.33@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.064236:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.064242:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.064245:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.064247:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513080.064251:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513080.064251:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.064256:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513080.064258:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.064261:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.064267:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513080.064268:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513080.064273:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22a18 00000800:00000200:12.0:1616513080.064275:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513080.064276:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513080.064278:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.064278:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513080.064279:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513080.064284:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.064288:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05870 00000800:00000200:12.0:1616513080.064294:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.068085:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.068142:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.068152:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.068153:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.068161:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.239@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.068170:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.239@tcp1 of length 224 into portal 12 MB=0x5ffd57fce6ec0 00000400:00000200:22.0:1616513080.068176:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.239@tcp1 of length 224/224 into md 0xf96450b9 [1] + 95544 00000400:00000200:22.0:1616513080.068181:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.068184:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513080.068190:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.068193:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.068195:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.068233:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.068249:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513080.068252:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667320905408 00000800:00000200:28.0:1616513080.068258:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00010000:00000200:26.0:1616513080.068263:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec205e80 x1688667320905408/t0(0) o400->5f300abf-1cd6-1675-a4b2-83d623f41a75@10.31.131.239@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513080.068267:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.239@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.068275:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.239@tcp1 of length 224 into portal 26 MB=0x5ffd57fce6e00 00000100:00000200:26.0:1616513080.068276:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667320905408, offset 192 00000400:00000200:26.0:1616513080.068282:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.239@tcp1 00000400:00000200:28.0:1616513080.068287:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.239@tcp1 of length 224/224 into md 0xf97c87f9 [1] + 0 00000400:00000200:26.0:1616513080.068289:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.239@tcp1 routed destination 00000400:00000200:28.0:1616513080.068291:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240088 00000400:00000200:28.0:1616513080.068294:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.068297:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513080.068301:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240088 00000800:00000200:24.0:1616513080.068301:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513080.068302:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.239@tcp1(10.31.131.239@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513080.068304:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.068307:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513080.068308:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513080.068309:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513080.068313:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.068316:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513080.068318:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513080.068322:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.068328:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.068343:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.068350:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.068353:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.068359:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:20.0:1616513080.068362:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667320905216 00000400:00000200:18.0:1616513080.068363:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeff0 00000800:00000200:18.0:1616513080.068367:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.068369:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:20.0:1616513080.068373:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688667320905216/t0(0) o400->8fa47592-e095-0030-d4b6-6a4f3cf0ce06@10.31.131.239@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.068387:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667320905216, offset 192 00000400:00000200:20.0:1616513080.068393:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.239@tcp1 00000400:00000200:20.0:1616513080.068400:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.239@tcp1 routed destination 00000400:00000200:20.0:1616513080.068414:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.239@tcp1(10.31.131.239@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.068420:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.068425:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.068428:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.068430:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.068447:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.068453:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.068454:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513080.068462:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.068466:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.068468:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513080.068473:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.068477:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22f68 00000800:00000200:8.0:1616513080.068481:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.068483:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.083034:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.083089:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.083097:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.083106:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.64@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.083116:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.64@tcp1 of length 224 into portal 12 MB=0x5ff4966a6b000 00000400:00000200:24.0:1616513080.083122:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.64@tcp1 of length 224/224 into md 0x865bbec1 [1] + 26320 00000400:00000200:24.0:1616513080.083127:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.083129:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.083135:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.083138:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.083140:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.083142:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.083206:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688065603448832 00010000:00000200:0.0:1616513080.083219:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d6780 x1688065603448832/t0(0) o400->a5a8f027-6f9a-b736-6458-55949fc8aaec@10.31.132.64@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513080.083233:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513080.083234:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688065603448832, offset 192 00000400:00000200:0.0:1616513080.083240:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.64@tcp1 00000800:00000200:30.0:1616513080.083245:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513080.083248:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.64@tcp1 routed destination 00000800:00000200:30.0:1616513080.083252:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513080.083263:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.64@tcp1(10.31.132.64@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.083269:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.083271:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.64@tcp1 : PUT - for me 00000800:00000200:0.0:1616513080.083275:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.083278:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:30.0:1616513080.083279:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.64@tcp1 of length 224 into portal 26 MB=0x5ff4966a6af80 00000800:00000200:0.0:1616513080.083280:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:30.0:1616513080.083284:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.64@tcp1 of length 224/224 into md 0xf97c8829 [1] + 0 00000800:00000200:0.0:1616513080.083285:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:30.0:1616513080.083288:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241870 00000400:00000200:30.0:1616513080.083290:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.083292:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.083296:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241870 00000800:00000200:22.0:1616513080.083296:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513080.083299:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513080.083301:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:30.0:1616513080.083302:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513080.083303:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513080.083305:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.083308:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.083314:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.083317:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05210 00000800:00000200:22.0:1616513080.083321:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.083356:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688065603448704 00010000:00000200:20.0:1616513080.083366:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688065603448704/t0(0) o400->3396f7d6-2f12-0e7e-8842-589e84920c9b@10.31.132.64@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.083378:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688065603448704, offset 192 00000400:00000200:20.0:1616513080.083384:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.64@tcp1 00000400:00000200:20.0:1616513080.083390:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.64@tcp1 routed destination 00000400:00000200:20.0:1616513080.083403:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.64@tcp1(10.31.132.64@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.083409:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.083414:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.083416:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.083418:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.083422:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.083427:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.083428:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513080.083442:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.083446:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.083448:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.083452:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513080.083454:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.083457:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23430 00000800:00000200:8.0:1616513080.083460:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.096798:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.096854:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.096863:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513080.096870:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.096872:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.249@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.096881:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.249@tcp1 of length 224 into portal 26 MB=0x604a46c52b280 00000400:00000200:24.0:1616513080.096888:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.249@tcp1 of length 224/224 into md 0xf97c8839 [1] + 0 00000400:00000200:24.0:1616513080.096893:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241540 00000400:00000200:24.0:1616513080.096896:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.096898:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.096903:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241540 00000800:00000200:24.0:1616513080.096906:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.096909:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.096911:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.096973:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954098770560 00010000:00000200:20.0:1616513080.096984:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1693954098770560/t0(0) o400->50fa7f58-af70-42f9-a954-599050d8b06a@10.31.140.249@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.096998:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954098770560, offset 192 00000400:00000200:20.0:1616513080.097004:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.249@tcp1 00000400:00000200:20.0:1616513080.097012:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.249@tcp1 routed destination 00000400:00000200:20.0:1616513080.097028:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.249@tcp1(10.31.140.249@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.097035:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.097041:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.097043:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.097046:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513080.097047:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.097050:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.097057:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.097058:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513080.097100:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.097109:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513080.097111:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.097118:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.249@tcp1 : PUT - for me 00000800:00000200:2.0:1616513080.097121:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513080.097126:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.249@tcp1 of length 224 into portal 12 MB=0x604a46c52b440 00000800:00000200:2.0:1616513080.097127:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.097130:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513080.097131:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513080.097133:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.249@tcp1 of length 224/224 into md 0x865bbec1 [1] + 26544 00000400:00000200:2.0:1616513080.097136:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.097138:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513080.097141:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22a18 00000100:00000200:12.0:1616513080.097144:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513080.097145:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513080.097151:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.097154:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.097156:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.097216:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954098771008 00010000:00000200:0.0:1616513080.097228:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c664380 x1693954098771008/t0(0) o400->52fbe9ab-eda3-cf3a-3982-a2108fd72afe@10.31.140.249@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.097243:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954098771008, offset 192 00000400:00000200:0.0:1616513080.097249:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.249@tcp1 00000400:00000200:0.0:1616513080.097256:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.249@tcp1 routed destination 00000400:00000200:0.0:1616513080.097271:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.249@tcp1(10.31.140.249@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.097278:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.097283:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.097286:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.097288:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.097293:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.097302:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.097313:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.097318:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.097321:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.097323:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.097327:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.097331:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:28.0:1616513080.097335:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.099626:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.099636:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.099643:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.099651:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.92@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.099660:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.92@tcp1 of length 224 into portal 26 MB=0x5ffdcfe6f0840 00000400:00000200:8.0:1616513080.099666:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.92@tcp1 of length 224/224 into md 0xf97c8859 [1] + 0 00000400:00000200:8.0:1616513080.099671:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240000 00000400:00000200:8.0:1616513080.099674:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.099676:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.099681:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240000 00000800:00000200:8.0:1616513080.099684:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.099686:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.099687:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513080.099694:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.099697:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.099698:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.099706:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.92@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.099715:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.92@tcp1 of length 224 into portal 12 MB=0x5ffdcfe6f08c0 00000400:00000200:12.0:1616513080.099721:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.92@tcp1 of length 224/224 into md 0x865bbec1 [1] + 26768 00000400:00000200:12.0:1616513080.099727:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.099729:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.099736:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.099740:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.099741:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.099744:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699510130752 00010000:00000200:20.0:1616513080.099755:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688699510130752/t0(0) o400->2156d1a0-8f19-faf3-6ebf-f1dbe086c0b7@10.31.138.92@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.099769:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699510130752, offset 192 00000400:00000200:20.0:1616513080.099776:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.92@tcp1 00000400:00000200:20.0:1616513080.099783:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.92@tcp1 routed destination 00000400:00000200:20.0:1616513080.099798:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.92@tcp1(10.31.138.92@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513080.099800:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699510130880 00000800:00000200:20.0:1616513080.099805:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.099810:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513080.099812:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c667500 x1688699510130880/t0(0) o400->a176630d-0056-33b4-91a5-21ce7f6f85ec@10.31.138.92@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.099813:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.099815:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.099820:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.099825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513080.099826:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699510130880, offset 192 00000100:00000200:20.0:1616513080.099827:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513080.099832:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.92@tcp1 00000400:00000200:0.0:1616513080.099840:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.92@tcp1 routed destination 00000400:00000200:0.0:1616513080.099855:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.92@tcp1(10.31.138.92@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.099861:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.099867:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.099870:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.099872:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.099876:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513080.099877:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513080.099882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513080.099884:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.099887:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.099894:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.099899:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22a18 00000800:00000200:22.0:1616513080.099902:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.099905:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513080.099954:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.099962:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.099964:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.099966:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513080.099973:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.099977:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04dd0 00000800:00000200:28.0:1616513080.099981:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.119840:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.119895:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.119904:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.119913:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.206@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.119922:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.206@tcp1 of length 224 into portal 26 MB=0x60183aa6ff000 00000400:00000200:12.0:1616513080.119928:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.206@tcp1 of length 224/224 into md 0xf97c8869 [1] + 0 00000400:00000200:12.0:1616513080.119933:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240ee0 00000400:00000200:12.0:1616513080.119936:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.119938:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.119943:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240ee0 00000800:00000200:12.0:1616513080.119946:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.119949:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.119951:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.119953:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.120004:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514872070144 00010000:00000200:20.0:1616513080.120014:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1690514872070144/t0(0) o400->8c5f40fd-878a-7f55-7641-4ddfbf9ae8a9@10.31.136.206@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.120028:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514872070144, offset 192 00000400:00000200:20.0:1616513080.120034:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.206@tcp1 00000400:00000200:20.0:1616513080.120041:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.206@tcp1 routed destination 00000400:00000200:20.0:1616513080.120055:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.206@tcp1(10.31.136.206@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.120061:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.120066:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.120071:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.120073:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.120077:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.120083:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.120089:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.120102:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.120115:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.120117:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.120123:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.120128:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22a18 00000800:00000200:30.0:1616513080.120131:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.120134:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.120137:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513080.120165:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.120177:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513080.120180:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.120182:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513080.120184:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.120186:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.120213:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.120224:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.120232:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.120240:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.141@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.120248:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.141@tcp1 of length 224 into portal 26 MB=0x5ffdf54556800 00000400:00000200:8.0:1616513080.120254:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.141@tcp1 of length 224/224 into md 0xf97c8899 [1] + 0 00000400:00000200:8.0:1616513080.120259:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240660 00000400:00000200:8.0:1616513080.120261:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.120264:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.120268:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240660 00000800:00000200:8.0:1616513080.120271:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.120274:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.120276:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.120278:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.120280:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709541226496 00010000:00000200:20.0:1616513080.120285:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688709541226496/t0(0) o400->6708f814-7892-3729-cb37-a71d0a73fd02@10.31.129.141@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.120296:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688709541226496, offset 192 00000400:00000200:20.0:1616513080.120301:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.141@tcp1 00000400:00000200:20.0:1616513080.120307:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.141@tcp1 routed destination 00000400:00000200:20.0:1616513080.120319:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.141@tcp1(10.31.129.141@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.120324:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.120328:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.120330:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.120332:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.120335:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513080.120339:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513080.120340:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.120351:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.120356:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.120359:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.120360:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.120365:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.120371:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e33a8 00000800:00000200:24.0:1616513080.120374:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.120621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.120629:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.120634:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513080.120637:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513080.120641:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.206@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.120648:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.206@tcp1 of length 224 into portal 12 MB=0x60183aa6ff140 00000400:00000200:8.0:1616513080.120652:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.206@tcp1 of length 224/224 into md 0x865bbec1 [1] + 26992 00000400:00000200:8.0:1616513080.120656:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.120658:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513080.120663:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.120665:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.120667:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.120727:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514872070464 00010000:00000200:0.0:1616513080.120739:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4800 x1690514872070464/t0(0) o400->45bdc5d5-cd92-8c72-30cf-37fa265d5647@10.31.136.206@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.120752:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514872070464, offset 192 00000400:00000200:0.0:1616513080.120758:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.206@tcp1 00000400:00000200:0.0:1616513080.120765:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.206@tcp1 routed destination 00000400:00000200:0.0:1616513080.120780:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.206@tcp1(10.31.136.206@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.120786:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513080.120792:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.120794:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513080.120796:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513080.120801:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.120806:0:49452:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.120815:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.120819:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.120821:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513080.120824:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513080.120825:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.120828:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb055c8 00000800:00000200:30.0:1616513080.120830:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.120888:0:49452:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.120899:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.120908:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513080.120911:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.120918:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.141@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.120926:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.141@tcp1 of length 224 into portal 12 MB=0x5ffdf54556cc0 00000400:00000200:22.0:1616513080.120933:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.141@tcp1 of length 224/224 into md 0xf96450b9 [1] + 95768 00000400:00000200:22.0:1616513080.120938:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.120941:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513080.120948:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.120951:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.120953:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.121010:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709541227712 00010000:00000200:26.0:1616513080.121022:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec206c00 x1688709541227712/t0(0) o400->b45d9598-c349-ceab-dde9-7e5ae02bc403@10.31.129.141@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.121036:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688709541227712, offset 192 00000400:00000200:26.0:1616513080.121042:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.141@tcp1 00000400:00000200:26.0:1616513080.121048:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.141@tcp1 routed destination 00000400:00000200:26.0:1616513080.121062:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.141@tcp1(10.31.129.141@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.121068:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.121074:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.121078:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.121080:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.121084:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.121089:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.121097:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.121101:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.121103:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.121106:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.121109:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.121112:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af100 00000800:00000200:12.0:1616513080.121115:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:2.0:1616513080.122568:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925776, portal 4 00000100:00000200:2.0:1616513080.122576:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925776, offset 0 00000400:00000200:2.0:1616513080.122582:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:2.0:1616513080.122588:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:2.0:1616513080.122600:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513080.122607:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:2.0:1616513080.122613:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513080.122616:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513080.122618:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513080.122623:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513080.122669:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513080.122684:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513080.122689:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513080.122691:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513080.122692:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513080.122697:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513080.122702:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdc57a880 x1693067725925776/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513080.122714:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed320 00000800:00000200:2.0:1616513080.122717:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513080.122760:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513080.122813:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513080.122822:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513080.122831:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513080.122839:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626d90 00000400:00000200:18.0:1616513080.122845:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c8f99 [1] + 192 00000400:00000200:18.0:1616513080.122851:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.122856:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdc57a880 x1693067725925776/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513080.122866:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513080.122871:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513080.122874:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513080.122877:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:2.0:1616513080.122877:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdc57a880 x1693067725925776/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513080.122888:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdc57a880 x1693067725925776/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513080.122897:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eeded0 00000800:00000200:7.2:1616513080.150371:0:49455:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513080.150375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513080.150383:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.150390:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.150398:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.67@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.150407:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.67@tcp1 of length 224 into portal 26 MB=0x6018362ccb580 00000400:00000200:30.0:1616513080.150413:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.67@tcp1 of length 224/224 into md 0xf97c88a9 [1] + 0 00000400:00000200:30.0:1616513080.150418:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178241d38 00000400:00000200:30.0:1616513080.150420:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.150423:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.150426:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178241d38 00000800:00000200:30.0:1616513080.150429:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.150432:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.150433:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.150444:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513080.150459:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.150469:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513080.150471:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.150478:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.67@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.150487:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.67@tcp1 of length 224 into portal 12 MB=0x6018362ccb640 00000400:00000200:28.0:1616513080.150493:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.67@tcp1 of length 224/224 into md 0x865bbec1 [1] + 27216 00000100:00000200:20.0:1616513080.150494:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513670190464 00000400:00000200:28.0:1616513080.150499:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.150501:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:20.0:1616513080.150506:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1690513670190464/t0(0) o400->df82a241-c7d5-f412-25d9-54cff39ab765@10.31.131.67@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513080.150509:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.150512:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.150514:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.150520:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513670190464, offset 192 00000400:00000200:20.0:1616513080.150526:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.67@tcp1 00000400:00000200:20.0:1616513080.150533:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.67@tcp1 routed destination 00000400:00000200:20.0:1616513080.150549:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.67@tcp1(10.31.131.67@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.150555:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.150561:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.150564:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.150566:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.150571:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.150576:0:49455:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.150578:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513080.150578:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513670190656 00010000:00000200:0.0:1616513080.150590:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9a400 x1690513670190656/t0(0) o400->3e1a2b1c-7150-6c4b-246f-dc5dc7f0a0a0@10.31.131.67@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.150605:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513670190656, offset 192 00000400:00000200:0.0:1616513080.150611:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.67@tcp1 00000400:00000200:0.0:1616513080.150618:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.67@tcp1 routed destination 00000400:00000200:0.0:1616513080.150643:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.67@tcp1(10.31.131.67@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513080.150647:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513080.150649:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513080.150653:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513080.150654:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513080.150656:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513080.150657:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.150659:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513080.150663:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513080.150663:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513080.150668:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e36d8 00000800:00000200:11.2:1616513080.150668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513080.150672:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.150675:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513080.150727:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.150737:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.150739:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.150742:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.150746:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.150750:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:24.0:1616513080.150754:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.159443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.159455:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.159462:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.159471:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.210@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.159480:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.210@tcp1 of length 224 into portal 26 MB=0x601838c380000 00000400:00000200:8.0:1616513080.159486:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.210@tcp1 of length 224/224 into md 0xf97c88b9 [1] + 0 00000400:00000200:8.0:1616513080.159491:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8178240770 00000400:00000200:8.0:1616513080.159493:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.159496:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.159500:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178240770 00000800:00000200:8.0:1616513080.159502:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.159505:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.159507:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.159509:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.159572:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514365087744 00010000:00000200:20.0:1616513080.159583:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1690514365087744/t0(0) o400->a7944de4-8f38-ee9e-ccf5-ac9cda3c25ff@10.31.132.210@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.159597:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514365087744, offset 192 00000400:00000200:20.0:1616513080.159603:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.210@tcp1 00000400:00000200:20.0:1616513080.159611:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.210@tcp1 routed destination 00000800:00000200:7.2:1616513080.159620:0:49456:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513080.159626:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.210@tcp1(10.31.132.210@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513080.159632:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.159633:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.159639:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.159639:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.159641:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.159644:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:30.0:1616513080.159647:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.210@tcp1 : PUT - for me 00000800:00000200:20.0:1616513080.159648:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:30.0:1616513080.159655:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.210@tcp1 of length 224 into portal 12 MB=0x601838c380080 00000100:00000200:20.0:1616513080.159655:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:30.0:1616513080.159662:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.210@tcp1 of length 224/224 into md 0xf96450b9 [1] + 95992 00000400:00000200:30.0:1616513080.159667:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.159670:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513080.159676:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513080.159678:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513080.159680:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.159682:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.159685:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.159687:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513080.159687:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.159693:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.159697:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3dc0 00000800:00000200:30.0:1616513080.159699:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.159743:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514365087872 00010000:00000200:26.0:1616513080.159756:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf83a80 x1690514365087872/t0(0) o400->39d24256-cca5-2862-4c8e-5f9e9b821d25@10.31.132.210@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.159771:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514365087872, offset 192 00000400:00000200:26.0:1616513080.159777:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.210@tcp1 00000400:00000200:26.0:1616513080.159784:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.210@tcp1 routed destination 00000400:00000200:26.0:1616513080.159800:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.210@tcp1(10.31.132.210@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.159806:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.159812:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.159815:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.159817:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.159822:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.159827:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.159898:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.159905:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.159907:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.159914:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.159918:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af100 00000800:00000200:12.0:1616513080.159922:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.159925:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.160392:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.160400:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.160406:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513080.160410:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513080.160413:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.234@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.160420:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.234@tcp1 of length 224 into portal 26 MB=0x5f907f72c4080 00000400:00000200:8.0:1616513080.160425:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.234@tcp1 of length 224/224 into md 0xf97c88d9 [1] + 0 00000400:00000200:8.0:1616513080.160429:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777eee0 00000400:00000200:8.0:1616513080.160430:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.160433:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.160442:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eee0 00000800:00000200:8.0:1616513080.160444:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.160446:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.160449:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.160451:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681187490513024 00010000:00000200:20.0:1616513080.160457:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1681187490513024/t0(0) o400->3799d411-e0a0-e6b4-f29f-21330b85980f@10.31.128.234@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.160469:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681187490513024, offset 192 00000400:00000200:20.0:1616513080.160474:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.234@tcp1 00000400:00000200:20.0:1616513080.160480:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.234@tcp1 routed destination 00000400:00000200:20.0:1616513080.160493:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.234@tcp1(10.31.128.234@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.160499:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.160504:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.160506:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.160508:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.160511:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.160515:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.160517:0:49456:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513080.160521:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.160530:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.160537:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513080.160540:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.160544:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.234@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.160553:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.234@tcp1 of length 224 into portal 12 MB=0x5f907f72c4100 00000400:00000200:12.0:1616513080.160559:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.234@tcp1 of length 224/224 into md 0x865bbec1 [1] + 27440 00000400:00000200:12.0:1616513080.160564:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.160567:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513080.160573:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513080.160573:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.160575:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.160577:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513080.160580:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.160583:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513080.160584:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513080.160590:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.160595:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2aa0 00000800:00000200:18.0:1616513080.160599:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513080.160637:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681187490513152 00010000:00000200:0.0:1616513080.160650:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c664c80 x1681187490513152/t0(0) o400->48e34e0b-521a-adde-fa29-69ce2adf8082@10.31.128.234@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.160664:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681187490513152, offset 192 00000400:00000200:0.0:1616513080.160670:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.234@tcp1 00000400:00000200:0.0:1616513080.160678:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.234@tcp1 routed destination 00000400:00000200:0.0:1616513080.160693:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.234@tcp1(10.31.128.234@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.160700:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.160705:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.160708:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.160712:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.160717:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.160721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.160732:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.160737:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.160740:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.160740:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.160746:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.160750:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb046e8 00000800:00000200:28.0:1616513080.160754:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:7.0:1616513080.170475:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925792, portal 4 00000100:00000200:7.0:1616513080.170481:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925792, offset 0 00000400:00000200:7.0:1616513080.170487:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:7.0:1616513080.170494:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:7.0:1616513080.170508:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513080.170516:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:7.0:1616513080.170523:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513080.170527:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:7.0:1616513080.170530:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:7.0:1616513080.170536:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513080.170583:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513080.170650:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513080.170657:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:24.0:1616513080.170660:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.170667:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:24.0:1616513080.170674:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63da00 x1693067725925792/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513080.170686:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637fdc0 00000800:00000200:24.0:1616513080.170690:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513080.170693:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513080.170767:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513080.170779:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513080.170787:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:28.0:1616513080.170794:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:8.0:1616513080.170795:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513080.170802:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626da0 00000400:00000200:8.0:1616513080.170808:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d5a71 [1] + 192 00000400:00000200:8.0:1616513080.170814:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.170819:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63da00 x1693067725925792/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513080.170833:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513080.170836:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513080.170838:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:7.0:1616513080.170851:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63da00 x1693067725925792/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513080.170861:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63da00 x1693067725925792/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513080.170869:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f7e8 00000100:00000200:3.0:1616513080.170889:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925808, portal 4 00000100:00000200:3.0:1616513080.170894:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925808, offset 0 00000400:00000200:3.0:1616513080.170901:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:3.0:1616513080.170907:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:3.0:1616513080.170920:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513080.170927:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:3.0:1616513080.170933:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513080.170936:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:3.0:1616513080.170938:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:3.0:1616513080.170943:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513080.170989:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513080.171008:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513080.171013:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513080.171015:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513080.171017:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513080.171022:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513080.171028:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc566780 x1693067725925808/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513080.171040:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2e58 00000800:00000200:27.0:1616513080.171045:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513080.171177:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513080.171186:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513080.171192:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513080.171196:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513080.171200:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513080.171208:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626db0 00000400:00000200:27.0:1616513080.171214:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166bad [1] + 192 00000400:00000200:27.0:1616513080.171219:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513080.171224:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc566780 x1693067725925808/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513080.171237:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513080.171240:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513080.171242:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:3.0:1616513080.171244:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc566780 x1693067725925808/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513080.171257:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc566780 x1693067725925808/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513080.171267:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:7.2:1616513080.179660:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.179672:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.179689:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.179697:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.66@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.179706:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.66@tcp1 of length 224 into portal 26 MB=0x5ffd8331f28c0 00000400:00000200:30.0:1616513080.179728:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.66@tcp1 of length 224/224 into md 0xf97c88e9 [1] + 0 00000400:00000200:30.0:1616513080.179732:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fd38 00000400:00000200:30.0:1616513080.179734:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.179737:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.179740:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fd38 00000800:00000200:30.0:1616513080.179742:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.179744:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.179745:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.179749:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.179805:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678919252160 00010000:00000200:20.0:1616513080.179816:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688678919252160/t0(0) o400->79f3fbe4-432f-c54f-635b-5d2fd52bd112@10.31.133.66@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.179830:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688678919252160, offset 192 00000400:00000200:20.0:1616513080.179838:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.66@tcp1 00000400:00000200:20.0:1616513080.179846:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.66@tcp1 routed destination 00000400:00000200:20.0:1616513080.179861:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.66@tcp1(10.31.133.66@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.179868:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.179874:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.179877:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.179879:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.179884:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.179889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.179890:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513080.179943:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.179950:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.179953:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513080.179959:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.179959:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.179965:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e25d8 00000800:00000200:2.0:1616513080.179969:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.180479:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.180491:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.180499:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.180499:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.180508:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.66@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.180517:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.66@tcp1 of length 224 into portal 12 MB=0x5ffd8331f2940 00000400:00000200:22.0:1616513080.180525:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.66@tcp1 of length 224/224 into md 0xf96450b9 [1] + 96216 00000400:00000200:22.0:1616513080.180530:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d84cb37c000 00000400:00000200:22.0:1616513080.180533:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.180536:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:22.0:1616513080.180540:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84cb37c000 00000800:00000200:22.0:1616513080.180543:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.180546:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.180548:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.180605:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678919252288 00010000:00000200:26.0:1616513080.180618:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74d9abe850 x1688678919252288/t0(0) o400->fba67d29-bca5-0998-1b54-8efffb168d1d@10.31.133.66@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.180632:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688678919252288, offset 192 00000400:00000200:26.0:1616513080.180639:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.66@tcp1 00000400:00000200:26.0:1616513080.180646:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.66@tcp1 routed destination 00000400:00000200:26.0:1616513080.180661:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.66@tcp1(10.31.133.66@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.180667:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.180673:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.180676:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.180678:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.180682:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.180690:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.180746:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.180753:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.180755:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.180762:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.180767:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeff0 00000800:00000200:24.0:1616513080.180771:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.180774:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513080.181087:0:184144:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 12 00000800:00000200:11.2:1616513080.181620:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.181674:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.181684:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.181693:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.241@tcp1 : PUT - for me 00000800:00000200:8.0:1616513080.181695:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.181702:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.241@tcp1 of length 224 into portal 26 MB=0x5ffd3d5156f40 00000400:00000200:28.0:1616513080.181709:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.241@tcp1 of length 224/224 into md 0xf97c88f9 [1] + 0 00000400:00000200:28.0:1616513080.181714:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f4b8 00000400:00000200:28.0:1616513080.181717:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.181719:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513080.181724:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f4b8 00000800:00000200:28.0:1616513080.181729:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.181732:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.181733:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.181784:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660161687360 00010000:00000200:20.0:1616513080.181794:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688660161687360/t0(0) o400->8898cb19-1ede-bef1-3e9f-3b6b7c43c8cd@10.31.128.241@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.181808:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660161687360, offset 192 00000400:00000200:20.0:1616513080.181813:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.241@tcp1 00000800:00000200:11.2:1616513080.181815:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:20.0:1616513080.181820:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.241@tcp1 routed destination 00000800:00000200:24.0:1616513080.181825:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.181831:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513080.181834:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.241@tcp1(10.31.128.241@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513080.181835:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.181839:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.241@tcp1 : PUT - for me 00000800:00000200:20.0:1616513080.181841:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.181847:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513080.181848:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.241@tcp1 of length 224 into portal 12 MB=0x5ffd3d5157240 00000800:00000200:20.0:1616513080.181850:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.181852:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513080.181853:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.241@tcp1 of length 224/224 into md 0x865bbec1 [1] + 27664 00000800:00000200:20.0:1616513080.181856:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513080.181859:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.181861:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513080.181861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.181862:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513080.181867:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.181870:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.181872:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513080.181913:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.181920:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.181923:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.181929:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.181935:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2f68 00000800:00000200:18.0:1616513080.181938:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.181941:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513080.181947:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660161688128 00010000:00000200:0.0:1616513080.181959:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3600 x1688660161688128/t0(0) o400->2fce07dc-872c-238c-bd5a-310c26086465@10.31.128.241@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.181972:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660161688128, offset 192 00000400:00000200:0.0:1616513080.181978:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.241@tcp1 00000400:00000200:0.0:1616513080.181985:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.241@tcp1 routed destination 00000400:00000200:0.0:1616513080.181999:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.241@tcp1(10.31.128.241@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.182005:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.182010:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.182012:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.182014:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.182019:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.182023:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.182032:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.182036:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.182038:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513080.182041:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513080.182044:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.182048:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05d38 00000800:00000200:8.0:1616513080.182052:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513080.186678:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925824, portal 4 00000100:00000200:22.0:1616513080.186684:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925824, offset 0 00000400:00000200:22.0:1616513080.186690:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513080.186696:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:22.0:1616513080.186708:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513080.186715:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:22.0:1616513080.186721:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513080.186723:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513080.186726:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513080.186730:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513080.186777:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513080.186789:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:30.0:1616513080.186793:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:30.0:1616513080.186795:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513080.186798:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:30.0:1616513080.186801:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:30.0:1616513080.186806:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74ddba0000 x1693067725925824/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513080.186818:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4af68 00000800:00000200:30.0:1616513080.186821:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513080.186946:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513080.186999:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513080.187008:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513080.187017:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513080.187025:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626dc0 00000400:00000200:18.0:1616513080.187032:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c9039 [1] + 192 00000400:00000200:18.0:1616513080.187037:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.187042:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74ddba0000 x1693067725925824/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513080.187053:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513080.187057:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513080.187059:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513080.187061:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:22.0:1616513080.187063:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74ddba0000 x1693067725925824/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513080.187076:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74ddba0000 x1693067725925824/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513080.187086:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b430 00000800:00000200:11.2:1616513080.187853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513080.187889:0:49459:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.187902:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513080.187908:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513080.187909:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.187910:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.187917:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.140@tcp1 : PUT - for me 00000800:00000200:12.0:1616513080.187917:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513080.187925:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.140@tcp1 of length 224 into portal 12 MB=0x603fcf44510c0 00000400:00000200:12.0:1616513080.187927:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.140@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.187940:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.140@tcp1 of length 224/224 into md 0xf964d519 [1] + 0 00000400:00000200:12.0:1616513080.187942:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.140@tcp1 of length 224 into portal 26 MB=0x603fcf4451000 00000400:00000200:22.0:1616513080.187945:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.187948:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.140@tcp1 of length 224/224 into md 0xf97c8909 [1] + 0 00000400:00000200:12.0:1616513080.187959:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee236d8 00000100:00000200:22.0:1616513080.187961:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513080.187962:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.187968:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513080.187969:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.187972:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.187973:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513080.187973:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee236d8 00000800:00000200:12.0:1616513080.187976:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.187979:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.187980:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.187984:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513080.188031:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234825072832 00010000:00000200:26.0:1616513080.188042:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec206780 x1693234825072832/t0(0) o400->30b9fd78-9a1d-793d-f30d-e38b183ce6d4@10.31.138.140@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513080.188051:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234825072640 00000100:00000200:26.0:1616513080.188057:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234825072832, offset 192 00010000:00000200:20.0:1616513080.188061:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1693234825072640/t0(0) o400->f923f112-f7f0-354b-bf0f-84047b031438@10.31.138.140@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513080.188063:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.140@tcp1 00000400:00000200:26.0:1616513080.188070:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.140@tcp1 routed destination 00000100:00000200:20.0:1616513080.188074:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234825072640, offset 192 00000400:00000200:20.0:1616513080.188080:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.140@tcp1 00000400:00000200:26.0:1616513080.188084:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.140@tcp1(10.31.138.140@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513080.188087:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.140@tcp1 routed destination 00000800:00000200:26.0:1616513080.188090:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513080.188095:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.188098:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513080.188100:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513080.188101:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.140@tcp1(10.31.138.140@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.188104:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.188109:0:49459:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.188111:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.188117:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.188119:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513080.188120:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.188121:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.188126:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513080.188126:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.188130:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513080.188130:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513080.188130:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:20.0:1616513080.188132:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513080.188137:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.188141:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af650 00000800:00000200:2.0:1616513080.188145:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513080.188183:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.188190:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.188193:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.188193:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.188199:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.188205:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e22a8 00000800:00000200:28.0:1616513080.188208:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.197576:0:49460:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.197589:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.197601:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.197609:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.14@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.197618:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.14@tcp1 of length 224 into portal 26 MB=0x5f93813974e80 00000400:00000200:30.0:1616513080.197624:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.14@tcp1 of length 224/224 into md 0xf97c8929 [1] + 0 00000400:00000200:30.0:1616513080.197628:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee224c8 00000400:00000200:30.0:1616513080.197630:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.197634:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.197637:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee224c8 00000800:00000200:30.0:1616513080.197640:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.197642:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.197644:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.197646:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.197651:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.197657:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.14@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.197663:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.14@tcp1 of length 224 into portal 12 MB=0x5f93813974f00 00000400:00000200:30.0:1616513080.197668:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.14@tcp1 of length 224/224 into md 0xf964d519 [1] + 224 00000400:00000200:30.0:1616513080.197672:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.197674:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513080.197679:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.197681:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.197682:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.197685:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.197701:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681394125721216 00010000:00000200:20.0:1616513080.197712:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1681394125721216/t0(0) o400->993ed52a-d4a9-f31f-762b-869343dc8e57@10.31.130.14@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.197726:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681394125721216, offset 192 00000400:00000200:20.0:1616513080.197733:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.14@tcp1 00000400:00000200:20.0:1616513080.197740:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.14@tcp1 routed destination 00000400:00000200:20.0:1616513080.197767:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.14@tcp1(10.31.130.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.197773:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:26.0:1616513080.197778:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681394125721344 00000800:00000200:20.0:1616513080.197778:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.197781:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.197783:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.197787:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:26.0:1616513080.197790:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf83f00 x1681394125721344/t0(0) o400->66cb43dc-9d67-83bf-63d3-7f5acf368bd9@10.31.130.14@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513080.197792:0:49460:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.197794:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513080.197803:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681394125721344, offset 192 00000400:00000200:26.0:1616513080.197809:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.14@tcp1 00000400:00000200:26.0:1616513080.197816:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.14@tcp1 routed destination 00000400:00000200:26.0:1616513080.197829:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.14@tcp1(10.31.130.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.197835:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.197841:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.197843:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.197845:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513080.197846:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513080.197849:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513080.197853:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513080.197856:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513080.197856:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513080.197857:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.197862:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.197868:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2a18 00000800:00000200:2.0:1616513080.197871:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513080.197911:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.197919:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.197922:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.197928:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.197932:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aea18 00000800:00000200:24.0:1616513080.197936:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.197939:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.200063:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da655200] (20)++ 00000800:00000200:30.0:1616513080.200075:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da655200] (21)++ 00000800:00000200:30.0:1616513080.200081:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.224@o2ib1 00000400:00000200:30.0:1616513080.200088:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.224@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513080.200094:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.224@o2ib1 of length 224 into portal 26 MB=0x5ffc7cd40d040 00000400:00000200:30.0:1616513080.200099:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.224@o2ib1 of length 224/224 into md 0xf97c8949 [1] + 0 00000400:00000200:30.0:1616513080.200103:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23870 00000400:00000200:30.0:1616513080.200105:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.200107:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.200110:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23870 00000800:00000200:30.0:1616513080.200112:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da655200] (22)++ 00000800:00000200:30.0:1616513080.200114:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da655200] (23)-- 00000800:00000200:30.0:1616513080.200116:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655200] (22)-- 00000800:00000200:30.0:1616513080.200118:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da655200] (21)++ 00000800:00000200:30.0:1616513080.200123:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.224@o2ib1 00000800:00000200:18.0:1616513080.200126:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655200] (22)-- 00000400:00000200:30.0:1616513080.200129:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.224@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513080.200134:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.224@o2ib1 of length 224 into portal 12 MB=0x5ffc7cd40d0c0 00000400:00000200:30.0:1616513080.200138:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.224@o2ib1 of length 224/224 into md 0xf964d519 [1] + 448 00000400:00000200:30.0:1616513080.200143:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.200144:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513080.200148:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da655200] (21)++ 00000800:00000200:30.0:1616513080.200150:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da655200] (22)-- 00000800:00000200:30.0:1616513080.200152:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655200] (21)-- 00000100:00000200:20.0:1616513080.200174:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688608490704960 00010000:00000200:20.0:1616513080.200185:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688608490704960/t0(0) o400->1aa6386a-c7a7-6e3c-f583-9b6bbd6c5e95@10.242.46.224@o2ib1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.200199:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688608490704960, offset 192 00000400:00000200:20.0:1616513080.200205:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.224@o2ib1 00000400:00000200:20.0:1616513080.200212:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.224@o2ib1 local destination 00000100:00000200:26.0:1616513080.200215:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688608490705088 00000400:00000200:20.0:1616513080.200226:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.224@o2ib1(10.242.46.224@o2ib1:10.242.46.224@o2ib1) : PUT try# 0 00010000:00000200:26.0:1616513080.200228:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf83180 x1688608490705088/t0(0) o400->07fd3ddc-1cb9-93c3-cbdc-ffd72d2250a7@10.242.46.224@o2ib1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.200232:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.224@o2ib1 00000800:00000200:20.0:1616513080.200238:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9e80] -> 10.242.46.224@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.200241:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da655200] (20)++ 00000100:00000200:26.0:1616513080.200242:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688608490705088, offset 192 00000800:00000200:20.0:1616513080.200245:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da655200] (21)++ 00000400:00000200:26.0:1616513080.200248:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.224@o2ib1 00000800:00000200:20.0:1616513080.200250:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da655200] (22)-- 00000400:00000200:26.0:1616513080.200255:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.224@o2ib1 local destination 00000800:00000200:7.2:1616513080.200255:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da655200] (21)++ 00000100:00000200:20.0:1616513080.200257:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513080.200269:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.224@o2ib1(10.242.46.224@o2ib1:10.242.46.224@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.200277:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.224@o2ib1 00000800:00000200:26.0:1616513080.200283:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9e80] -> 10.242.46.224@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.200286:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da655200] (22)++ 00000800:00000200:26.0:1616513080.200288:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da655200] (23)++ 00000800:00000200:26.0:1616513080.200293:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da655200] (24)-- 00000800:00000200:2.0:1616513080.200308:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da655200] (23)++ 00000800:00000200:2.0:1616513080.200315:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da655200] (24)-- 00000400:00000200:2.0:1616513080.200318:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513080.200319:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da655200] (23)++ 00000400:00000200:2.0:1616513080.200325:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.224@o2ib1: PUT: OK 00000800:00000200:18.0:1616513080.200326:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da655200] (24)-- 00000800:00000200:30.0:1616513080.200328:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655200] (23)-- 00000400:00000200:18.0:1616513080.200328:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513080.200330:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2110 00000800:00000200:2.0:1616513080.200334:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655200] (22)-- 00000400:00000200:18.0:1616513080.200335:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.224@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.200339:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afba0 00000800:00000200:18.0:1616513080.200343:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655200] (21)-- 00000800:00000200:11.2:1616513080.216042:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.216054:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.216062:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.216070:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.6@tcp1 : PUT - for me 00000800:00000200:7.2:1616513080.216071:0:49462:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513080.216080:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.6@tcp1 of length 224 into portal 26 MB=0x5ffd455d271c0 00000400:00000200:8.0:1616513080.216087:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.6@tcp1 of length 224/224 into md 0xf97c8969 [1] + 0 00000400:00000200:8.0:1616513080.216091:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee233a8 00000400:00000200:8.0:1616513080.216094:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.216097:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.216101:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee233a8 00000800:00000200:8.0:1616513080.216103:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.216106:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.216106:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.216110:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513080.216130:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.216140:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.216141:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.216149:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.6@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.216158:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.6@tcp1 of length 224 into portal 12 MB=0x5ffd455d27280 00000400:00000200:22.0:1616513080.216164:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.6@tcp1 of length 224/224 into md 0xf964d519 [1] + 672 00000100:00000200:20.0:1616513080.216166:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662321557952 00000400:00000200:22.0:1616513080.216169:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.216172:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:20.0:1616513080.216177:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688662321557952/t0(0) o400->ae211026-e845-534f-cdf2-ca12bde6892f@10.31.130.6@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513080.216179:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.216182:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.216183:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.216191:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662321557952, offset 192 00000400:00000200:20.0:1616513080.216198:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.6@tcp1 00000400:00000200:20.0:1616513080.216205:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.6@tcp1 routed destination 00000400:00000200:20.0:1616513080.216220:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.6@tcp1(10.31.130.6@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.216227:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.216233:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.216235:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.216238:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.216242:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.216247:0:49462:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:26.0:1616513080.216248:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662321558144 00000100:00000200:20.0:1616513080.216249:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:26.0:1616513080.216261:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec204380 x1688662321558144/t0(0) o400->83f9d45c-e8dc-4555-8639-616ebb81bae5@10.31.130.6@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.216275:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662321558144, offset 192 00000400:00000200:26.0:1616513080.216281:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.6@tcp1 00000400:00000200:26.0:1616513080.216289:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.6@tcp1 routed destination 00000800:00000200:2.0:1616513080.216301:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513080.216304:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.6@tcp1(10.31.130.6@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513080.216308:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513080.216311:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513080.216311:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513080.216316:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513080.216318:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:26.0:1616513080.216319:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.216321:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513080.216324:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3650 00000800:00000200:26.0:1616513080.216326:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513080.216327:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.216330:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.216331:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.216388:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.216395:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.216398:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.216404:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.216409:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa90 00000800:00000200:24.0:1616513080.216413:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.216416:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.217946:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.218000:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.218010:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513080.218011:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.218017:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.218019:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.241@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.218023:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.241@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.218028:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.241@tcp1 of length 224 into portal 26 MB=0x5ffd43b1df3c0 00000400:00000200:8.0:1616513080.218031:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.241@tcp1 of length 224 into portal 12 MB=0x5ffd43b1df440 00000400:00000200:28.0:1616513080.218035:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.241@tcp1 of length 224/224 into md 0xf97c8979 [1] + 0 00000400:00000200:8.0:1616513080.218036:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.241@tcp1 of length 224/224 into md 0x865bbec1 [1] + 27888 00000400:00000200:28.0:1616513080.218040:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23ed0 00000400:00000200:8.0:1616513080.218040:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513080.218042:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.218042:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:28.0:1616513080.218045:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513080.218048:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:28.0:1616513080.218049:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23ed0 00000800:00000200:8.0:1616513080.218050:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513080.218051:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513080.218052:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513080.218054:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.218055:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.218057:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.218113:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661873521728 00010000:00000200:0.0:1616513080.218126:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3180 x1688661873521728/t0(0) o400->e6ad66b2-2b26-7ff0-16d3-87405440b0e1@10.31.132.241@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513080.218129:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661873521600 00010000:00000200:20.0:1616513080.218140:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688661873521600/t0(0) o400->aa22b866-982d-43bc-0ca2-2f6ad35d2886@10.31.132.241@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513080.218140:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661873521728, offset 192 00000400:00000200:0.0:1616513080.218146:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.241@tcp1 00000400:00000200:0.0:1616513080.218153:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.241@tcp1 routed destination 00000100:00000200:20.0:1616513080.218154:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661873521600, offset 192 00000400:00000200:20.0:1616513080.218161:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.241@tcp1 00000400:00000200:20.0:1616513080.218168:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.241@tcp1 routed destination 00000400:00000200:0.0:1616513080.218169:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.241@tcp1(10.31.132.241@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.218175:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513080.218183:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.241@tcp1(10.31.132.241@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.218190:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.218190:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.218193:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.218195:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.218195:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.218198:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.218200:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.218200:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513080.218205:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513080.218208:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513080.218210:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.218211:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.218221:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513080.218221:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513080.218226:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.218228:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.218228:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.218231:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.218231:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513080.218233:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.218236:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000400:00000200:12.0:1616513080.218238:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:30.0:1616513080.218240:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.218242:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513080.218244:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2198 00000800:00000200:12.0:1616513080.218247:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513080.218654:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925840, portal 4 00000100:00000200:22.0:1616513080.218660:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925840, offset 0 00000400:00000200:22.0:1616513080.218666:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:22.0:1616513080.218672:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:22.0:1616513080.218686:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513080.218693:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:22.0:1616513080.218699:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513080.218702:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:22.0:1616513080.218705:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:22.0:1616513080.218709:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513080.218756:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.218775:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513080.218782:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513080.218785:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513080.218792:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513080.218800:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74dd21a880 x1693067725925840/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513080.218812:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bdc0 00000800:00000200:1.0:1616513080.218817:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513080.218820:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513080.218940:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513080.218951:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.218958:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513080.218961:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513080.218966:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513080.218974:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626dd0 00000400:00000200:1.0:1616513080.218981:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffba5 [1] + 192 00000400:00000200:1.0:1616513080.218986:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513080.218991:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74dd21a880 x1693067725925840/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513080.219005:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.219008:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513080.219010:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:22.0:1616513080.219015:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74dd21a880 x1693067725925840/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513080.219028:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74dd21a880 x1693067725925840/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513080.219039:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b430 00000800:00000200:11.2:1616513080.250890:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.250946:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.250955:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.250965:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.158@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.250975:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.158@tcp1 of length 224 into portal 12 MB=0x5ffd69b715fc0 00000400:00000200:28.0:1616513080.250981:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.158@tcp1 of length 224/224 into md 0x865bbec1 [1] + 28112 00000400:00000200:28.0:1616513080.250987:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.250989:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513080.250997:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.251000:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.251000:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513080.251004:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.251063:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672079536064 00010000:00000200:0.0:1616513080.251076:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b99200 x1688672079536064/t0(0) o400->9a657e8c-80f1-ecc8-feba-4e5544063df0@10.31.129.158@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.251091:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688672079536064, offset 192 00000400:00000200:0.0:1616513080.251099:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.158@tcp1 00000400:00000200:0.0:1616513080.251107:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.158@tcp1 routed destination 00000400:00000200:0.0:1616513080.251122:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.158@tcp1(10.31.129.158@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.251128:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513080.251135:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.251137:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513080.251140:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513080.251145:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.251152:0:49466:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.251208:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.251215:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.251218:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513080.251225:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.251229:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04440 00000800:00000200:2.0:1616513080.251233:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.251236:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.251242:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.251245:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.251250:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.158@tcp1 : PUT - for me 00000400:00000200:2.0:1616513080.251258:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.158@tcp1 of length 224 into portal 26 MB=0x5ffd69b715c40 00000400:00000200:2.0:1616513080.251264:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.158@tcp1 of length 224/224 into md 0xf97c89a9 [1] + 0 00000400:00000200:2.0:1616513080.251270:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23210 00000400:00000200:2.0:1616513080.251273:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513080.251275:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513080.251279:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23210 00000800:00000200:2.0:1616513080.251282:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.251284:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.251286:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.251345:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672079535168 00010000:00000200:20.0:1616513080.251356:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688672079535168/t0(0) o400->46d01f8d-a39d-2a2f-acff-d7f662a798fd@10.31.129.158@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.251372:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688672079535168, offset 192 00000400:00000200:20.0:1616513080.251378:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.158@tcp1 00000400:00000200:20.0:1616513080.251385:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.158@tcp1 routed destination 00000400:00000200:20.0:1616513080.251401:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.158@tcp1(10.31.129.158@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.251408:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.251414:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.251416:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.251418:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.251423:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.251428:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.251430:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513080.251443:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.251448:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.251450:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.251453:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513080.251456:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.251461:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3100 00000800:00000200:8.0:1616513080.251464:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.255969:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.256024:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.256034:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.256035:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.256043:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.184@tcp1 : PUT - for me 00000800:00000200:11.2:1616513080.256046:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513080.256055:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.184@tcp1 of length 224 into portal 26 MB=0x60027960aa000 00000400:00000200:22.0:1616513080.256061:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.184@tcp1 of length 224/224 into md 0xf97c89b9 [1] + 0 00000400:00000200:22.0:1616513080.256066:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee238f8 00000400:00000200:22.0:1616513080.256069:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.256071:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513080.256075:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee238f8 00000800:00000200:22.0:1616513080.256078:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.256081:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.256083:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513080.256101:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.256111:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.256119:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.184@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.256129:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.184@tcp1 of length 224 into portal 12 MB=0x60027960aa4c0 00000400:00000200:24.0:1616513080.256135:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.184@tcp1 of length 224/224 into md 0x865bbec1 [1] + 28336 00000400:00000200:24.0:1616513080.256140:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513080.256141:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689019881267200 00000100:00000200:24.0:1616513080.256143:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.256149:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.256152:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:20.0:1616513080.256152:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1689019881267200/t0(0) o400->b01da683-ac37-867e-e39b-d139f9116c4b@10.31.128.184@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513080.256154:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.256157:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.256166:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689019881267200, offset 192 00000400:00000200:20.0:1616513080.256172:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.184@tcp1 00000400:00000200:20.0:1616513080.256180:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.184@tcp1 routed destination 00000400:00000200:20.0:1616513080.256195:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.184@tcp1(10.31.128.184@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.256201:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.256207:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.256210:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.256212:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.256216:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.256221:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.256223:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513080.256224:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689019881268416 00010000:00000200:0.0:1616513080.256237:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d4380 x1689019881268416/t0(0) o400->de62c24c-3cf8-36de-6b90-28bd6d7b2038@10.31.128.184@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.256251:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689019881268416, offset 192 00000400:00000200:0.0:1616513080.256258:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.184@tcp1 00000400:00000200:0.0:1616513080.256266:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.184@tcp1 routed destination 00000400:00000200:0.0:1616513080.256281:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.184@tcp1(10.31.128.184@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.256288:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.256293:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513080.256295:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513080.256296:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.256298:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513080.256302:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513080.256303:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513080.256305:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513080.256308:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513080.256312:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.256317:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2cc0 00000800:00000200:12.0:1616513080.256319:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513080.256321:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.256324:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513080.256326:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.256329:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.256331:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.256336:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.256340:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04908 00000800:00000200:12.0:1616513080.256344:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.257178:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.257191:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.257200:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513080.257202:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.257207:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513080.257209:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.257210:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.167@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.257214:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.167@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.257220:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.167@tcp1 of length 224 into portal 26 MB=0x5ffdde200cf40 00000400:00000200:24.0:1616513080.257221:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.167@tcp1 of length 224 into portal 12 MB=0x5ffdde200cfc0 00000400:00000200:28.0:1616513080.257226:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.167@tcp1 of length 224/224 into md 0xf97c89e9 [1] + 0 00000400:00000200:24.0:1616513080.257226:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.167@tcp1 of length 224/224 into md 0x865bbec1 [1] + 28560 00000400:00000200:24.0:1616513080.257230:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513080.257231:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23ba0 00000100:00000200:24.0:1616513080.257232:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513080.257234:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.257237:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513080.257237:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.257241:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.257242:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23ba0 00000800:00000200:24.0:1616513080.257243:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.257245:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.257248:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513080.257249:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703328112576 00000800:00000200:28.0:1616513080.257250:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.257255:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703328112448 00010000:00000200:0.0:1616513080.257256:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5100 x1688703328112576/t0(0) o400->8ebdd553-b1ec-79e7-5a84-10085107d971@10.31.132.167@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:20.0:1616513080.257260:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688703328112448/t0(0) o400->eba4fd46-d711-6050-56c8-a980cc37f923@10.31.132.167@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513080.257268:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703328112576, offset 192 00000400:00000200:0.0:1616513080.257274:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.167@tcp1 00000100:00000200:20.0:1616513080.257276:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703328112448, offset 192 00000400:00000200:0.0:1616513080.257280:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.167@tcp1 routed destination 00000400:00000200:20.0:1616513080.257281:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.167@tcp1 00000400:00000200:20.0:1616513080.257287:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.167@tcp1 routed destination 00000400:00000200:0.0:1616513080.257293:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.167@tcp1(10.31.132.167@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.257299:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513080.257302:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.167@tcp1(10.31.132.167@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.257308:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.257309:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.257313:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.257313:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513080.257315:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.257316:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.257317:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.257318:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513080.257321:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513080.257323:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.257326:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513080.257326:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513080.257332:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513080.257335:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513080.257336:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.257338:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.257338:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.257340:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.257341:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513080.257345:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.257346:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.257348:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04cc0 00000800:00000200:30.0:1616513080.257352:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513080.257352:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2c38 00000800:00000200:30.0:1616513080.257354:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513080.257354:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.264692:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.264748:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.264758:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.264767:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.68@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.264776:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.68@tcp1 of length 224 into portal 26 MB=0x5ffd48a376cc0 00000400:00000200:24.0:1616513080.264783:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.68@tcp1 of length 224/224 into md 0xf97c8a09 [1] + 0 00000400:00000200:24.0:1616513080.264788:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22440 00000400:00000200:24.0:1616513080.264790:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.264793:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.264797:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22440 00000800:00000200:24.0:1616513080.264800:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.264802:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.264803:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.264809:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.264812:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.264813:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.264821:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.68@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.264830:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.68@tcp1 of length 224 into portal 12 MB=0x5ffd48a376d40 00000400:00000200:28.0:1616513080.264836:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.68@tcp1 of length 224/224 into md 0x865bbec1 [1] + 28784 00000400:00000200:28.0:1616513080.264842:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.264844:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513080.264851:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.264855:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.264857:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.264864:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663200591040 00010000:00000200:20.0:1616513080.264874:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688663200591040/t0(0) o400->32731613-c4b4-daf5-ff52-dec48afc0afa@10.31.136.68@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.264887:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663200591040, offset 192 00000400:00000200:20.0:1616513080.264893:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.68@tcp1 00000400:00000200:20.0:1616513080.264900:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.68@tcp1 routed destination 00000400:00000200:20.0:1616513080.264914:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.68@tcp1(10.31.136.68@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513080.264916:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663200591168 00000800:00000200:20.0:1616513080.264920:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.264926:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.264928:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:0.0:1616513080.264929:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9de80 x1688663200591168/t0(0) o400->79d51d5e-25d6-81ef-c74b-12391277af1b@10.31.136.68@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.264930:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.264935:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.264940:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.264941:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513080.264943:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663200591168, offset 192 00000400:00000200:0.0:1616513080.264949:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.68@tcp1 00000400:00000200:0.0:1616513080.264957:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.68@tcp1 routed destination 00000400:00000200:0.0:1616513080.264972:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.68@tcp1(10.31.136.68@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.264978:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.264984:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.264986:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.264989:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513080.264993:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513080.264993:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.264999:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513080.265000:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.265003:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.265010:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513080.265010:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513080.265015:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3ba0 00000800:00000200:12.0:1616513080.265017:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513080.265019:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513080.265019:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.265020:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513080.265022:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513080.265026:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.265030:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:12.0:1616513080.265034:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.265187:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.265196:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.265201:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.265208:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.245@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.265217:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.245@tcp1 of length 224 into portal 26 MB=0x5ffd362760900 00000400:00000200:30.0:1616513080.265223:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.245@tcp1 of length 224/224 into md 0xf97c8a19 [1] + 0 00000400:00000200:30.0:1616513080.265226:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22bb0 00000400:00000200:30.0:1616513080.265229:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.265231:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.265235:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22bb0 00000800:00000200:30.0:1616513080.265237:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.265239:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.265242:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.265244:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.265247:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658238638336 00000800:00000200:30.0:1616513080.265249:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513080.265249:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:20.0:1616513080.265252:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688658238638336/t0(0) o400->290db5df-4329-6a31-70a2-c79890522916@10.31.132.245@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.265263:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658238638336, offset 192 00000400:00000200:30.0:1616513080.265266:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.245@tcp1 : PUT - for me 00000400:00000200:20.0:1616513080.265267:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.245@tcp1 00000400:00000200:30.0:1616513080.265271:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.245@tcp1 of length 224 into portal 12 MB=0x5ffd362760980 00000400:00000200:20.0:1616513080.265273:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.245@tcp1 routed destination 00000400:00000200:30.0:1616513080.265276:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.245@tcp1 of length 224/224 into md 0xf964d519 [1] + 896 00000400:00000200:30.0:1616513080.265280:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.265282:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513080.265286:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.245@tcp1(10.31.132.245@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513080.265289:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.265291:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513080.265291:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.265293:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513080.265295:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.265299:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.265300:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.265303:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.265307:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.265308:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.265321:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.265328:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.265331:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513080.265332:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.265337:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.265348:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3430 00000800:00000200:2.0:1616513080.265352:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.265354:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658238638464 00010000:00000200:26.0:1616513080.265366:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf84800 x1688658238638464/t0(0) o400->2451f854-7b07-674d-a0aa-35a766f6d615@10.31.132.245@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.265380:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658238638464, offset 192 00000400:00000200:26.0:1616513080.265387:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.245@tcp1 00000400:00000200:26.0:1616513080.265395:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.245@tcp1 routed destination 00000400:00000200:26.0:1616513080.265410:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.245@tcp1(10.31.132.245@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.265417:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.265422:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.265425:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.265427:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.265432:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.265440:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.265450:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.265453:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.265456:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513080.265458:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.265461:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.265464:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afed0 00000800:00000200:24.0:1616513080.265467:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513080.266489:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925872, portal 4 00000100:00000200:28.0:1616513080.266493:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925888, portal 4 00000100:00000200:26.0:1616513080.266493:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925872, offset 0 00000100:00000200:28.0:1616513080.266498:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925888, offset 0 00000400:00000200:26.0:1616513080.266498:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:28.0:1616513080.266503:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:26.0:1616513080.266503:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:28.0:1616513080.266508:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:26.0:1616513080.266515:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.266521:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000100:00000200:3.0:1616513080.266524:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925856, portal 4 00000800:00000200:26.0:1616513080.266526:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513080.266527:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.266529:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000100:00000200:3.0:1616513080.266530:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925856, offset 0 00000800:00000200:26.0:1616513080.266531:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:28.0:1616513080.266533:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:26.0:1616513080.266534:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:3.0:1616513080.266537:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513080.266539:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513080.266541:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:3.0:1616513080.266542:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:28.0:1616513080.266545:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:28.0:1616513080.266549:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:3.0:1616513080.266556:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513080.266563:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513080.266569:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513080.266573:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:3.0:1616513080.266575:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:3.0:1616513080.266580:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:1.2:1616513080.266582:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:5.2:1616513080.266587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513080.266600:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:1.0:1616513080.266602:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000800:00000200:8.0:1616513080.266605:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513080.266608:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513080.266608:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (24)-- 00000800:00000200:5.0:1616513080.266611:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000400:00000200:1.0:1616513080.266611:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513080.266614:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:5.0:1616513080.266617:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (24)-- 00000400:00000200:5.0:1616513080.266620:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.266621:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd639680 x1693067725925856/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513080.266621:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:5.0:1616513080.266627:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513080.266628:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dec201b00 x1693067725925872/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513080.266632:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2330 00000100:00000200:5.0:1616513080.266634:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b2d7080 x1693067725925888/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513080.266636:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513080.266638:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000400:00000200:1.0:1616513080.266640:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae880 00000800:00000200:1.0:1616513080.266645:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:5.0:1616513080.266646:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd188 00000800:00000200:1.0:1616513080.266648:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:5.0:1616513080.266650:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513080.266758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513080.266767:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.266774:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513080.266775:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:5.0:1616513080.266780:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513080.266782:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:9.0:1616513080.266784:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:5.0:1616513080.266788:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513080.266790:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626e00 00000400:00000200:5.0:1616513080.266796:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626df0 00000400:00000200:1.0:1616513080.266797:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffbc5 [1] + 192 00000400:00000200:1.0:1616513080.266802:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:5.0:1616513080.266804:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffbb5 [1] + 192 00000400:00000200:5.0:1616513080.266810:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513080.266812:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b2d7080 x1693067725925888/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.2:1616513080.266819:0:6468:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:1.0:1616513080.266826:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000100:00000200:5.0:1616513080.266827:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dec201b00 x1693067725925872/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513080.266829:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513080.266830:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:5.0:1616513080.266841:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:28.0:1616513080.266844:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b2d7080 x1693067725925888/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513080.266845:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:5.0:1616513080.266847:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:28.0:1616513080.266856:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b2d7080 x1693067725925888/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513080.266866:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc000 00000100:00000200:26.0:1616513080.266871:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dec201b00 x1693067725925872/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513080.266883:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dec201b00 x1693067725925872/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513080.266885:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513080.266891:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:26.0:1616513080.266893:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af6d8 00000800:00000200:24.0:1616513080.266894:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513080.266898:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513080.266904:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626de0 00000400:00000200:28.0:1616513080.266910:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d5bb1 [1] + 192 00000400:00000200:28.0:1616513080.266914:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.266918:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd639680 x1693067725925856/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513080.266930:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513080.266932:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513080.266934:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:3.0:1616513080.266941:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd639680 x1693067725925856/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513080.266970:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd639680 x1693067725925856/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513080.266980:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:7.2:1616513080.286925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513080.286942:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.286959:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.286966:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.286974:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.130@tcp1 : PUT - for me 00000800:00000200:18.0:1616513080.286982:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513080.286983:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.130@tcp1 of length 224 into portal 26 MB=0x601832ccfaf00 00000400:00000200:8.0:1616513080.286990:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.130@tcp1 of length 224/224 into md 0xf97c8a39 [1] + 0 00000800:00000200:18.0:1616513080.286992:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.287010:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513080.287010:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22dd0 00000400:00000200:8.0:1616513080.287013:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.287014:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.130@tcp1 : PUT - for me 00000100:00000200:8.0:1616513080.287015:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.287019:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22dd0 00000800:00000200:8.0:1616513080.287022:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513080.287023:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.130@tcp1 of length 224 into portal 12 MB=0x601832ccfaf80 00000800:00000200:8.0:1616513080.287025:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.287026:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513080.287030:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.130@tcp1 of length 224/224 into md 0xf964d519 [1] + 1120 00000800:00000200:8.0:1616513080.287030:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513080.287035:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.287037:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513080.287044:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.287047:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.287049:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.287079:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512764415744 00010000:00000200:20.0:1616513080.287089:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1690512764415744/t0(0) o400->cbb10c5d-2201-55ba-1ad4-80c454d980a6@10.31.136.130@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.287101:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512764415744, offset 192 00000400:00000200:20.0:1616513080.287107:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.130@tcp1 00000400:00000200:20.0:1616513080.287113:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.130@tcp1 routed destination 00000100:00000200:26.0:1616513080.287121:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512764415872 00000400:00000200:20.0:1616513080.287127:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.130@tcp1(10.31.136.130@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.287133:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:26.0:1616513080.287134:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf9680 x1690512764415872/t0(0) o400->370dc0a9-046d-1d92-aecb-88706731ba61@10.31.136.130@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.287138:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.287140:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.287142:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.287146:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:26.0:1616513080.287148:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512764415872, offset 192 00000800:00000200:7.2:1616513080.287151:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.287152:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513080.287154:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.130@tcp1 00000400:00000200:26.0:1616513080.287162:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.130@tcp1 routed destination 00000400:00000200:26.0:1616513080.287177:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.130@tcp1(10.31.136.130@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.287183:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.287189:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.287192:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.287194:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.287198:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513080.287204:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513080.287204:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513080.287211:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.287214:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.287220:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.287225:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e38f8 00000800:00000200:22.0:1616513080.287228:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.287231:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513080.287256:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.287263:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.287266:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.287272:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.287276:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af6d8 00000800:00000200:24.0:1616513080.287279:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.287282:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.300109:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.300164:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.300173:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513080.300175:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513080.300175:0:49471:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513080.300184:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.86@tcp1 : PUT - for me 00000800:00000200:30.0:1616513080.300187:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513080.300193:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.86@tcp1 of length 224 into portal 12 MB=0x60374b8317380 00000800:00000200:30.0:1616513080.300195:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513080.300199:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.11.86@tcp1 of length 224/224 into md 0x865bbec1 [1] + 29008 00000400:00000200:30.0:1616513080.300204:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.86@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.300205:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.300207:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:30.0:1616513080.300212:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.86@tcp1 of length 224 into portal 26 MB=0x60374b8317000 00000800:00000200:12.0:1616513080.300214:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.300216:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513080.300218:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.11.86@tcp1 of length 224/224 into md 0xf97c8a49 [1] + 0 00000800:00000200:12.0:1616513080.300218:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513080.300223:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23e48 00000400:00000200:30.0:1616513080.300225:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.300228:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.300232:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23e48 00000800:00000200:30.0:1616513080.300234:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.300237:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.300238:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.300241:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513080.300282:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692649701602176 00010000:00000200:0.0:1616513080.300295:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c665580 x1692649701602176/t0(0) o400->cf1e8c8b-4032-1c19-b2c1-3d07aaf93370@10.31.11.86@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513080.300300:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692649701601280 00000100:00000200:0.0:1616513080.300309:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692649701602176, offset 192 00010000:00000200:20.0:1616513080.300311:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1692649701601280/t0(0) o400->d0c59bf5-09ce-7369-5320-8e5620a2f0b7@10.31.11.86@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513080.300315:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.86@tcp1 00000400:00000200:0.0:1616513080.300323:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.86@tcp1 routed destination 00000100:00000200:20.0:1616513080.300325:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692649701601280, offset 192 00000400:00000200:20.0:1616513080.300331:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.86@tcp1 00000400:00000200:0.0:1616513080.300338:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.86@tcp1(10.31.11.86@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513080.300339:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.86@tcp1 routed destination 00000800:00000200:0.0:1616513080.300345:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513080.300354:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.86@tcp1(10.31.11.86@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.300361:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.300361:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.300364:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.300366:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.300366:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.300369:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.300371:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.300371:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.300375:0:49471:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.300376:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.300381:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.300383:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513080.300429:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.300447:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513080.300450:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513080.300450:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.300457:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513080.300457:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.300459:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.300461:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:22.0:1616513080.300465:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513080.300466:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513080.300468:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513080.300471:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3870 00000800:00000200:24.0:1616513080.300476:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.300479:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.305709:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.305729:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.305737:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.305745:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.226@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.305754:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.226@tcp1 of length 224 into portal 26 MB=0x60182f4ce2540 00000400:00000200:8.0:1616513080.305760:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.226@tcp1 of length 224/224 into md 0xf97c8a59 [1] + 0 00000400:00000200:8.0:1616513080.305764:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee226e8 00000400:00000200:8.0:1616513080.305767:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.305769:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.305773:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee226e8 00000800:00000200:8.0:1616513080.305776:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.305779:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.305780:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.305782:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.305793:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.305802:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.226@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.305811:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.226@tcp1 of length 224 into portal 12 MB=0x60182f4ce25c0 00000400:00000200:12.0:1616513080.305817:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.226@tcp1 of length 224/224 into md 0x865bbec1 [1] + 29232 00000400:00000200:12.0:1616513080.305822:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.305825:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.305831:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.305834:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:20.0:1616513080.305836:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511824790848 00000800:00000200:12.0:1616513080.305836:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.305839:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:20.0:1616513080.305848:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1690511824790848/t0(0) o400->df4b4c81-f3c2-4d8c-8053-89a80981a20b@10.31.132.226@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.305862:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511824790848, offset 192 00000400:00000200:20.0:1616513080.305868:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.226@tcp1 00000400:00000200:20.0:1616513080.305876:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.226@tcp1 routed destination 00000400:00000200:20.0:1616513080.305890:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.226@tcp1(10.31.132.226@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513080.305896:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511824790976 00000800:00000200:20.0:1616513080.305897:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.305902:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.305905:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.305907:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513080.305908:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c662400 x1690511824790976/t0(0) o400->19387b05-4e26-7908-dbce-9061586905c9@10.31.132.226@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.305912:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.305918:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.305919:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.305927:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.305931:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.305933:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1616513080.305933:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511824790976, offset 192 00000400:00000200:30.0:1616513080.305938:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:0.0:1616513080.305940:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.226@tcp1 00000400:00000200:30.0:1616513080.305944:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e34b8 00000800:00000200:30.0:1616513080.305946:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513080.305947:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.226@tcp1 routed destination 00000800:00000200:30.0:1616513080.305948:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513080.305962:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.226@tcp1(10.31.132.226@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.305969:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.305974:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.305977:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.305979:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.305984:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.305989:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.306043:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.306050:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.306052:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.306052:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.306059:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.306063:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04aa0 00000800:00000200:28.0:1616513080.306067:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.314606:0:6484:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925904, portal 4 00000100:00000200:20.0:1616513080.314611:0:6484:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925904, offset 0 00000400:00000200:20.0:1616513080.314618:0:6484:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:20.0:1616513080.314624:0:6484:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:20.0:1616513080.314637:0:6484:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.314644:0:6484:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:20.0:1616513080.314650:0:6484:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.314653:0:6484:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:20.0:1616513080.314655:0:6484:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:20.0:1616513080.314660:0:6484:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513080.314708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.314727:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513080.314733:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513080.314736:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513080.314743:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513080.314751:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cc6b98900 x1693067725925904/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513080.314763:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3a90 00000800:00000200:1.0:1616513080.314769:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513080.314772:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513080.314858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513080.314867:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.314873:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513080.314881:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513080.314889:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626e10 00000400:00000200:1.0:1616513080.314896:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffbd5 [1] + 192 00000400:00000200:1.0:1616513080.314901:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513080.314908:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cc6b98900 x1693067725925904/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513080.314918:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513080.314922:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.314925:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513080.314927:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:20.0:1616513080.314933:0:6484:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cc6b98900 x1693067725925904/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513080.314946:0:6484:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cc6b98900 x1693067725925904/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513080.314956:0:6484:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3188 00000800:00000200:7.2:1616513080.327468:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.327524:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.327534:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.327543:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.241@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.327553:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.241@tcp1 of length 224 into portal 26 MB=0x604a465d48500 00000400:00000200:22.0:1616513080.327560:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.241@tcp1 of length 224/224 into md 0xf97c8a79 [1] + 0 00000400:00000200:22.0:1616513080.327564:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22cc0 00000400:00000200:22.0:1616513080.327567:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.327570:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513080.327577:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22cc0 00000800:00000200:22.0:1616513080.327580:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.327580:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.327583:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.327585:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.327643:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693953989838080 00010000:00000200:20.0:1616513080.327654:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1693953989838080/t0(0) o400->0a031849-6149-8844-f4a7-c1d44537e6b0@10.31.140.241@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513080.327672:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513080.327679:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693953989838080, offset 192 00000800:00000200:30.0:1616513080.327682:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513080.327685:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.241@tcp1 00000800:00000200:30.0:1616513080.327689:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513080.327692:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.241@tcp1 routed destination 00000400:00000200:30.0:1616513080.327697:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.241@tcp1 : PUT - for me 00000400:00000200:20.0:1616513080.327708:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.241@tcp1(10.31.140.241@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.327714:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.327716:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.241@tcp1 of length 224 into portal 12 MB=0x604a465d48640 00000800:00000200:20.0:1616513080.327720:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513080.327722:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.241@tcp1 of length 224/224 into md 0xf964d519 [1] + 1344 00000800:00000200:20.0:1616513080.327723:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.327725:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:30.0:1616513080.327727:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.327730:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513080.327730:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513080.327735:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513080.327737:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000100:00000200:20.0:1616513080.327737:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.327740:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:30.0:1616513080.327742:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513080.327744:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513080.327745:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.327746:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.327753:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.327759:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3188 00000800:00000200:18.0:1616513080.327762:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.327815:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693953989838400 00010000:00000200:26.0:1616513080.327827:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf86780 x1693953989838400/t0(0) o400->d0b7ff4f-e9cb-e830-f8e7-8579950d2009@10.31.140.241@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.327850:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693953989838400, offset 192 00000400:00000200:26.0:1616513080.327856:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.241@tcp1 00000400:00000200:26.0:1616513080.327863:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.241@tcp1 routed destination 00000400:00000200:26.0:1616513080.327877:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.241@tcp1(10.31.140.241@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.327883:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.327888:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.327891:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.327893:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.327897:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.327904:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.327961:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.327968:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.327971:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.327977:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.327981:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af3a8 00000800:00000200:24.0:1616513080.327985:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.327988:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.330250:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.330304:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.330313:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.330313:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.330322:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.202@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.330331:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.202@tcp1 of length 224 into portal 26 MB=0x5ffd4b0a50340 00000400:00000200:22.0:1616513080.330337:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.202@tcp1 of length 224/224 into md 0xf97c8a99 [1] + 0 00000400:00000200:22.0:1616513080.330342:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23f58 00000400:00000200:22.0:1616513080.330345:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.330347:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513080.330351:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23f58 00000800:00000200:22.0:1616513080.330354:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.330357:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.330359:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.330415:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663845307200 00010000:00000200:20.0:1616513080.330426:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688663845307200/t0(0) o400->2da8f3d9-67e2-1590-d118-95e92f9c5ecd@10.31.136.202@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513080.330444:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513080.330450:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663845307200, offset 192 00000800:00000200:8.0:1616513080.330456:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513080.330457:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.202@tcp1 00000800:00000200:8.0:1616513080.330463:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513080.330465:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.202@tcp1 routed destination 00000400:00000200:8.0:1616513080.330474:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.202@tcp1 : PUT - for me 00000400:00000200:20.0:1616513080.330480:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.202@tcp1(10.31.136.202@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513080.330483:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.202@tcp1 of length 224 into portal 12 MB=0x5ffd4b0a50440 00000800:00000200:20.0:1616513080.330487:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513080.330488:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.202@tcp1 of length 224/224 into md 0x865bbec1 [1] + 29456 00000800:00000200:20.0:1616513080.330492:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:8.0:1616513080.330493:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513080.330495:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:8.0:1616513080.330496:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513080.330497:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.330502:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513080.330502:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.330504:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.330507:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513080.330507:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513080.330509:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.330509:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513080.330567:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663845307456 00000800:00000200:2.0:1616513080.330571:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.330578:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:0.0:1616513080.330579:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d2880 x1688663845307456/t0(0) o400->4cbf22b4-fc56-706e-80d8-a5a9c258e95f@10.31.136.202@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513080.330581:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513080.330587:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.330593:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e37e8 00000100:00000200:0.0:1616513080.330596:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663845307456, offset 192 00000800:00000200:2.0:1616513080.330597:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.330599:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513080.330602:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.202@tcp1 00000400:00000200:0.0:1616513080.330609:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.202@tcp1 routed destination 00000400:00000200:0.0:1616513080.330624:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.202@tcp1(10.31.136.202@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.330630:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.330635:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.330637:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.330639:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.330644:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.330650:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.330705:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.330712:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.330715:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.330717:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.330721:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.330725:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05540 00000800:00000200:24.0:1616513080.330729:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.332750:0:49474:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.332760:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.332765:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.332772:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.35@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.332780:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.35@tcp1 of length 224 into portal 26 MB=0x6041a0b1bf100 00000400:00000200:30.0:1616513080.332785:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.35@tcp1 of length 224/224 into md 0xf97c8aa9 [1] + 0 00000400:00000200:30.0:1616513080.332789:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22d48 00000400:00000200:30.0:1616513080.332791:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.332793:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.332796:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22d48 00000800:00000200:30.0:1616513080.332798:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.332800:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.332802:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.332804:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.332859:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359762305280 00010000:00000200:20.0:1616513080.332870:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1693359762305280/t0(0) o400->adbba6ca-f17a-c21c-7b4e-8009e0049648@10.31.129.35@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.332885:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693359762305280, offset 192 00000400:00000200:20.0:1616513080.332891:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.35@tcp1 00000400:00000200:20.0:1616513080.332899:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.35@tcp1 routed destination 00000400:00000200:20.0:1616513080.332916:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.35@tcp1(10.31.129.35@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.332923:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.332928:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.332931:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.332933:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.332938:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.332942:0:49474:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.332945:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513080.332995:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.333002:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.333005:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513080.333012:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.333017:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3078 00000800:00000200:2.0:1616513080.333021:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.333024:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.333155:0:49474:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.333163:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.333168:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513080.333172:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513080.333176:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.35@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.333181:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.35@tcp1 of length 224 into portal 12 MB=0x6041a0b1bf580 00000400:00000200:30.0:1616513080.333186:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.35@tcp1 of length 224/224 into md 0xf964d519 [1] + 1568 00000400:00000200:30.0:1616513080.333189:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.333191:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513080.333195:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.333197:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.333199:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.333267:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359762306432 00010000:00000200:26.0:1616513080.333279:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf82d00 x1693359762306432/t0(0) o400->d4dfd110-cd16-b404-0792-e2221ad487fe@10.31.129.35@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.333294:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693359762306432, offset 192 00000400:00000200:26.0:1616513080.333300:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.35@tcp1 00000400:00000200:26.0:1616513080.333308:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.35@tcp1 routed destination 00000400:00000200:26.0:1616513080.333323:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.35@tcp1(10.31.129.35@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.333329:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.333335:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.333337:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.333339:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.333344:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.333349:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.333357:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.333361:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.333363:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513080.333368:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.333371:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae088 00000800:00000200:8.0:1616513080.333373:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.333376:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.343851:0:49475:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513080.343911:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.343929:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513080.343938:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.238@tcp1 : PUT - for me 00000400:00000200:2.0:1616513080.343948:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.238@tcp1 of length 224 into portal 26 MB=0x603e10072ea00 00000400:00000200:2.0:1616513080.343954:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.238@tcp1 of length 224/224 into md 0xf97c8af9 [1] + 0 00000400:00000200:2.0:1616513080.343959:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23cb0 00000400:00000200:2.0:1616513080.343965:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513080.343967:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513080.343972:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23cb0 00000800:00000200:2.0:1616513080.343975:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.343976:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.343978:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513080.343983:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513080.343986:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.343987:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.343995:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.238@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.344007:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.238@tcp1 of length 224 into portal 12 MB=0x603e10072ea80 00000400:00000200:18.0:1616513080.344013:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.238@tcp1 of length 224/224 into md 0xf964d519 [1] + 1792 00000400:00000200:18.0:1616513080.344018:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.344021:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513080.344028:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.344031:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.344033:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.344035:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114770319872 00010000:00000200:20.0:1616513080.344046:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1693114770319872/t0(0) o400->35eb4353-54e4-ae4d-0429-dbb40fd340e6@10.31.136.238@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.344060:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114770319872, offset 192 00000400:00000200:20.0:1616513080.344067:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.238@tcp1 00000400:00000200:20.0:1616513080.344074:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.238@tcp1 routed destination 00000400:00000200:20.0:1616513080.344089:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.238@tcp1(10.31.136.238@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513080.344091:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114770320000 00000800:00000200:20.0:1616513080.344096:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.344102:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:26.0:1616513080.344103:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bff980 x1693114770320000/t0(0) o400->7011ae42-688f-3c00-4ef6-55d2b995a358@10.31.136.238@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.344105:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.344107:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.344112:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.344116:0:49475:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:26.0:1616513080.344118:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114770320000, offset 192 00000100:00000200:20.0:1616513080.344118:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513080.344124:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.238@tcp1 00000800:00000200:30.0:1616513080.344125:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.344129:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.344131:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513080.344131:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.238@tcp1 routed destination 00000800:00000200:2.0:1616513080.344134:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513080.344136:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.344140:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e27f8 00000800:00000200:30.0:1616513080.344142:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:26.0:1616513080.344146:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.238@tcp1(10.31.136.238@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.344155:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.344161:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.344163:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.344166:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.344170:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.344178:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.344234:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.344240:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.344243:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.344250:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.344254:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af870 00000800:00000200:12.0:1616513080.344258:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.344261:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.373381:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.373395:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.373404:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.373413:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.144@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.373421:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.144@tcp1 of length 224 into portal 26 MB=0x5ffd4aa55f8c0 00000400:00000200:8.0:1616513080.373428:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.144@tcp1 of length 224/224 into md 0xf97c8b09 [1] + 0 00000400:00000200:8.0:1616513080.373433:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23298 00000400:00000200:8.0:1616513080.373446:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:28.0:1616513080.373448:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513080.373449:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.373453:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23298 00000800:00000200:8.0:1616513080.373456:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.373459:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.373461:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.373523:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663739463872 00010000:00000200:20.0:1616513080.373534:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688663739463872/t0(0) o400->b81ec530-d374-5db2-738e-2220733b54cf@10.31.136.144@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.373548:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663739463872, offset 192 00000400:00000200:20.0:1616513080.373554:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.144@tcp1 00000400:00000200:20.0:1616513080.373562:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.144@tcp1 routed destination 00000400:00000200:20.0:1616513080.373577:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.144@tcp1(10.31.136.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.373583:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.373589:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.373594:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.373596:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.373601:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.373607:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.373608:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.373671:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.373678:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.373681:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.373688:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.373693:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2b28 00000800:00000200:18.0:1616513080.373697:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.373700:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.373788:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.373842:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.373852:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.373860:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.144@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.373868:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.144@tcp1 of length 224 into portal 12 MB=0x5ffd4aa55f940 00000400:00000200:12.0:1616513080.373875:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.144@tcp1 of length 224/224 into md 0x865bbec1 [1] + 29680 00000400:00000200:12.0:1616513080.373882:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.373884:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.373891:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.373894:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.373896:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.373901:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.373917:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663739464000 00010000:00000200:0.0:1616513080.373930:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663600 x1688663739464000/t0(0) o400->9d54a4ff-e601-932d-52f1-f5e65c79cb4b@10.31.136.144@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.373944:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663739464000, offset 192 00000400:00000200:0.0:1616513080.373951:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.144@tcp1 00000400:00000200:0.0:1616513080.373958:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.144@tcp1 routed destination 00000400:00000200:0.0:1616513080.373973:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.144@tcp1(10.31.136.144@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.373982:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.373987:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.373990:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.373992:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.373997:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.374002:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.374013:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.374019:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.374021:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.374022:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.374028:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.374032:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05cb0 00000800:00000200:28.0:1616513080.374036:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.391639:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.391696:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.391705:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.391714:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.213@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.391724:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.213@tcp1 of length 224 into portal 26 MB=0x5ffd4b9575640 00000400:00000200:24.0:1616513080.391731:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.213@tcp1 of length 224/224 into md 0xf97c8b19 [1] + 0 00000400:00000200:24.0:1616513080.391735:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee222a8 00000400:00000200:24.0:1616513080.391738:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.391741:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.391746:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee222a8 00000800:00000200:24.0:1616513080.391749:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.391749:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.391751:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.391753:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.391809:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663991211584 00010000:00000200:20.0:1616513080.391820:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688663991211584/t0(0) o400->701f9e6a-96ee-3ddc-7383-4020499fcd79@10.31.132.213@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.391834:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663991211584, offset 192 00000400:00000200:20.0:1616513080.391841:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.213@tcp1 00000400:00000200:20.0:1616513080.391849:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.213@tcp1 routed destination 00000400:00000200:20.0:1616513080.391864:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.213@tcp1(10.31.132.213@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.391870:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.391876:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.391879:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.391881:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.391886:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.391892:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.391896:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.391952:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.391959:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.391962:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513080.391963:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.391968:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.391973:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2660 00000800:00000200:2.0:1616513080.391977:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.392368:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.392377:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.392384:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.392391:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.213@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.392399:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.213@tcp1 of length 224 into portal 12 MB=0x5ffd4b95757c0 00000400:00000200:8.0:1616513080.392404:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.213@tcp1 of length 224/224 into md 0x865bbec1 [1] + 29904 00000400:00000200:8.0:1616513080.392408:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.392410:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513080.392416:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.392419:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.392420:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.392422:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.392481:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663991211968 00010000:00000200:0.0:1616513080.392493:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6780 x1688663991211968/t0(0) o400->6c255da8-6aba-d725-1130-51fad454318f@10.31.132.213@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.392509:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663991211968, offset 192 00000400:00000200:0.0:1616513080.392515:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.213@tcp1 00000400:00000200:0.0:1616513080.392523:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.213@tcp1 routed destination 00000400:00000200:0.0:1616513080.392538:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.213@tcp1(10.31.132.213@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.392544:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.392550:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.392553:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.392555:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.392559:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.392565:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.392575:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.392581:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.392583:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.392586:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.392590:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.392594:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb058f8 00000800:00000200:12.0:1616513080.392598:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.398396:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.398409:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.398416:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.398423:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.65@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.398431:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.65@tcp1 of length 224 into portal 12 MB=0x5ffd3f01a61c0 00000400:00000200:8.0:1616513080.398442:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.65@tcp1 of length 224/224 into md 0x865bbec1 [1] + 30128 00000400:00000200:8.0:1616513080.398448:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.398450:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513080.398455:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.398457:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.398459:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.398461:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.398519:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660614996416 00010000:00000200:0.0:1616513080.398531:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0d80 x1688660614996416/t0(0) o400->6803481d-d70d-f77e-f278-3c364993ce72@10.31.128.65@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.398546:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660614996416, offset 192 00000400:00000200:0.0:1616513080.398552:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.65@tcp1 00000800:00000200:7.2:1616513080.398553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513080.398559:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.65@tcp1 routed destination 00000400:00000200:0.0:1616513080.398574:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.65@tcp1(10.31.128.65@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.398581:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513080.398586:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.398590:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513080.398592:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513080.398597:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.398607:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.398617:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.398626:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.65@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.398635:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.65@tcp1 of length 224 into portal 26 MB=0x5ffd3f01a6100 00000400:00000200:18.0:1616513080.398641:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.65@tcp1 of length 224/224 into md 0xf97c8b29 [1] + 0 00000400:00000200:18.0:1616513080.398646:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22908 00000400:00000200:18.0:1616513080.398648:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.398651:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513080.398655:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22908 00000800:00000200:18.0:1616513080.398658:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513080.398661:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:18.0:1616513080.398661:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513080.398668:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513080.398669:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513080.398671:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.398672:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.398679:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.398683:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb044c8 00000800:00000200:22.0:1616513080.398687:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.398719:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660614996224 00010000:00000200:20.0:1616513080.398730:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688660614996224/t0(0) o400->df02d3c9-ee5b-1073-250e-fd39b2f83805@10.31.128.65@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.398744:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660614996224, offset 192 00000400:00000200:20.0:1616513080.398751:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.65@tcp1 00000400:00000200:20.0:1616513080.398758:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.65@tcp1 routed destination 00000400:00000200:20.0:1616513080.398773:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.65@tcp1(10.31.128.65@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.398779:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.398785:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.398788:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.398790:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.398795:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.398800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.398801:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513080.398854:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.398861:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.398864:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.398866:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.398871:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.398877:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2000 00000800:00000200:24.0:1616513080.398882:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.406970:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513080.407027:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.407037:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.407038:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.407046:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.115@tcp1 : PUT - for me 00000400:00000200:2.0:1616513080.407055:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.115@tcp1 of length 224 into portal 26 MB=0x601953fc6df00 00000400:00000200:2.0:1616513080.407062:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.115@tcp1 of length 224/224 into md 0xf97c8b59 [1] + 0 00000400:00000200:2.0:1616513080.407067:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23540 00000400:00000200:2.0:1616513080.407069:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513080.407072:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513080.407076:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23540 00000800:00000200:2.0:1616513080.407080:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.407082:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.407084:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.407103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513080.407140:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590392016640 00010000:00000200:20.0:1616513080.407151:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1690590392016640/t0(0) o400->5d6bdcfb-39ca-4f67-a0d6-7e040a46ffbd@10.31.140.115@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513080.407156:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.407165:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690590392016640, offset 192 00000800:00000200:18.0:1616513080.407165:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513080.407172:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.115@tcp1 00000400:00000200:18.0:1616513080.407173:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.115@tcp1 : PUT - for me 00000400:00000200:20.0:1616513080.407180:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.115@tcp1 routed destination 00000400:00000200:18.0:1616513080.407181:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.115@tcp1 of length 224 into portal 12 MB=0x601953fc6e140 00000400:00000200:18.0:1616513080.407188:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.115@tcp1 of length 224/224 into md 0xf964d519 [1] + 2016 00000400:00000200:20.0:1616513080.407195:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.115@tcp1(10.31.140.115@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513080.407195:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.407197:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513080.407202:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513080.407204:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.407207:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513080.407208:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513080.407209:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513080.407210:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.407212:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513080.407213:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.407217:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.407223:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.407224:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.407231:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.407235:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.407237:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513080.407240:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513080.407243:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.407247:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2770 00000800:00000200:30.0:1616513080.407250:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.407273:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590392017216 00010000:00000200:26.0:1616513080.407284:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf8d80 x1690590392017216/t0(0) o400->51cd9923-fde3-b68c-1424-662d41f06ac4@10.31.140.115@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.407297:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690590392017216, offset 192 00000400:00000200:26.0:1616513080.407303:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.115@tcp1 00000400:00000200:26.0:1616513080.407310:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.115@tcp1 routed destination 00000400:00000200:26.0:1616513080.407324:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.115@tcp1(10.31.140.115@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.407330:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.407335:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.407338:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.407340:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.407344:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.407390:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.407401:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.407405:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.407407:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513080.407412:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.407415:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afdc0 00000800:00000200:8.0:1616513080.407418:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.407420:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.420184:0:49483:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.420241:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.420251:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.420260:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.217@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.420269:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.217@tcp1 of length 224 into portal 26 MB=0x600aeb3903f00 00000400:00000200:22.0:1616513080.420275:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.217@tcp1 of length 224/224 into md 0xf97c8b69 [1] + 0 00000400:00000200:22.0:1616513080.420280:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23d38 00000400:00000200:22.0:1616513080.420283:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.420286:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513080.420290:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23d38 00000800:00000200:22.0:1616513080.420293:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.420295:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.420297:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.420299:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.420355:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689600197148416 00010000:00000200:20.0:1616513080.420366:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1689600197148416/t0(0) o400->67f99982-0f7e-231b-c1a7-a766fb09f550@10.31.128.217@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.420381:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689600197148416, offset 192 00000400:00000200:20.0:1616513080.420387:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.217@tcp1 00000400:00000200:20.0:1616513080.420394:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.217@tcp1 routed destination 00000800:00000200:7.2:1616513080.420403:0:49483:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513080.420409:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.217@tcp1(10.31.128.217@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.420416:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.420422:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.420425:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.420427:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.420432:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513080.420457:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:20.0:1616513080.420458:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513080.420467:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.420469:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513080.420474:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:30.0:1616513080.420476:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513080.420476:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.217@tcp1 : PUT - for me 00000800:00000200:18.0:1616513080.420478:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.420483:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.420484:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.217@tcp1 of length 224 into portal 12 MB=0x600aeb39043c0 00000400:00000200:30.0:1616513080.420487:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2770 00000800:00000200:30.0:1616513080.420490:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.420491:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.217@tcp1 of length 224/224 into md 0xf964d519 [1] + 2240 00000400:00000200:2.0:1616513080.420496:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513080.420501:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513080.420508:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.420511:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.420513:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.420571:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689600197149632 00010000:00000200:26.0:1616513080.420583:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd219200 x1689600197149632/t0(0) o400->3bfc14c5-1f2e-6bc3-4052-9937031b520f@10.31.128.217@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.420598:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689600197149632, offset 192 00000400:00000200:26.0:1616513080.420604:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.217@tcp1 00000400:00000200:26.0:1616513080.420611:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.217@tcp1 routed destination 00000400:00000200:26.0:1616513080.420626:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.217@tcp1(10.31.128.217@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.420632:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.420638:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.420641:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.420643:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.420647:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.420696:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.420750:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.420757:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.420760:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.420766:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.420770:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af4b8 00000800:00000200:12.0:1616513080.420774:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.420777:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.432773:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.432785:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.432796:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513080.432798:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513080.432803:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.132@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.432810:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.132@tcp1 of length 224 into portal 26 MB=0x5ffe07e311d80 00000400:00000200:8.0:1616513080.432815:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.132@tcp1 of length 224/224 into md 0xf97c8b89 [1] + 0 00000400:00000200:8.0:1616513080.432819:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee234b8 00000400:00000200:8.0:1616513080.432821:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.432823:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.432826:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee234b8 00000800:00000200:8.0:1616513080.432829:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.432831:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.432832:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.432834:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513080.432879:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.432888:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:20.0:1616513080.432891:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714538458496 00000400:00000200:22.0:1616513080.432897:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.132@tcp1 : PUT - for me 00010000:00000200:20.0:1616513080.432902:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688714538458496/t0(0) o400->03931b0c-cad1-4b0e-0a8f-fe68025e13f0@10.31.142.132@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513080.432906:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.132@tcp1 of length 224 into portal 12 MB=0x5ffe07e312140 00000400:00000200:22.0:1616513080.432912:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.132@tcp1 of length 224/224 into md 0xf964d519 [1] + 2464 00000100:00000200:20.0:1616513080.432916:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688714538458496, offset 192 00000400:00000200:22.0:1616513080.432917:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.432920:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513080.432925:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.132@tcp1 00000800:00000200:22.0:1616513080.432926:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.432930:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.432932:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513080.432932:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.132@tcp1 routed destination 00000800:00000200:18.0:1616513080.432934:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513080.432947:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.132@tcp1(10.31.142.132@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.432954:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.432960:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.432963:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.432965:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.432970:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.432974:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.432976:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.432983:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.432988:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.432990:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:26.0:1616513080.432993:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714538459456 00000400:00000200:30.0:1616513080.432995:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.432999:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2660 00000800:00000200:30.0:1616513080.433002:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.433004:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:26.0:1616513080.433004:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec207980 x1688714538459456/t0(0) o400->d105e0be-9718-9e5c-3031-e31ecbfb8f97@10.31.142.132@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.433018:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688714538459456, offset 192 00000400:00000200:26.0:1616513080.433023:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.132@tcp1 00000400:00000200:26.0:1616513080.433030:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.132@tcp1 routed destination 00000400:00000200:26.0:1616513080.433045:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.132@tcp1(10.31.142.132@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.433051:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.433056:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.433059:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.433061:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.433065:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.433070:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.433122:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.433129:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.433132:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.433138:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.433142:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae550 00000800:00000200:12.0:1616513080.433146:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.433149:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.437852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.437865:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.437871:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.437878:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.136@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.437885:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.136@tcp1 of length 224 into portal 12 MB=0x5ffdda4a58400 00000400:00000200:8.0:1616513080.437890:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.136@tcp1 of length 224/224 into md 0x865bbec1 [1] + 30352 00000400:00000200:8.0:1616513080.437894:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.437896:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513080.437902:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.437904:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.437906:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.437908:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.437966:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702298719232 00010000:00000200:0.0:1616513080.437979:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7080 x1688702298719232/t0(0) o400->628a24d1-ed2a-4202-48c2-e4c8a2b8a696@10.31.141.136@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.437993:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702298719232, offset 192 00000400:00000200:0.0:1616513080.438000:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.136@tcp1 00000400:00000200:0.0:1616513080.438008:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.136@tcp1 routed destination 00000400:00000200:0.0:1616513080.438023:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.136@tcp1(10.31.141.136@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.438029:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513080.438035:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.438038:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513080.438040:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513080.438045:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.438050:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.438103:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.438110:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.438113:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.438119:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.438123:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04c38 00000800:00000200:22.0:1616513080.438127:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.438130:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.438444:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.438456:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.438465:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.438473:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.146@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.438484:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.146@tcp1 of length 224 into portal 26 MB=0x5ffd4af4a5e40 00000400:00000200:12.0:1616513080.438490:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.146@tcp1 of length 224/224 into md 0xf97c8bb9 [1] + 0 00000400:00000200:12.0:1616513080.438495:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22660 00000400:00000200:12.0:1616513080.438497:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.438502:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.438506:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22660 00000800:00000200:24.0:1616513080.438508:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.438509:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.438512:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513080.438521:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.438524:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.438525:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.438533:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.146@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.438542:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.146@tcp1 of length 224 into portal 12 MB=0x5ffd4af4a6180 00000400:00000200:24.0:1616513080.438548:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.146@tcp1 of length 224/224 into md 0x865bbec1 [1] + 30576 00000400:00000200:24.0:1616513080.438554:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.438556:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.438563:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.438567:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513080.438567:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663822589504 00000800:00000200:24.0:1616513080.438569:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.438576:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663822590336 00010000:00000200:20.0:1616513080.438577:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688663822589504/t0(0) o400->2d411b80-5a75-8332-e436-3d7cd5dbb2a4@10.31.136.146@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:0.0:1616513080.438582:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d1f80 x1688663822590336/t0(0) o400->30138cc9-3849-803d-4d00-402be8117053@10.31.136.146@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513080.438591:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663822589504, offset 192 00000100:00000200:0.0:1616513080.438594:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663822590336, offset 192 00000400:00000200:20.0:1616513080.438597:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.146@tcp1 00000400:00000200:0.0:1616513080.438602:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.146@tcp1 00000400:00000200:20.0:1616513080.438604:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.146@tcp1 routed destination 00000400:00000200:20.0:1616513080.438618:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.146@tcp1(10.31.136.146@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.438624:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513080.438626:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.146@tcp1 routed destination 00000800:00000200:20.0:1616513080.438632:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.438634:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.438636:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513080.438639:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.146@tcp1(10.31.136.146@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.438641:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513080.438645:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513080.438646:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.438647:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513080.438649:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.438652:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513080.438654:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513080.438657:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513080.438658:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513080.438662:0:49485:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513080.438665:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.438667:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.438668:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513080.438672:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513080.438674:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:30.0:1616513080.438675:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.438677:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513080.438680:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e27f8 00000400:00000200:30.0:1616513080.438682:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513080.438683:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513080.438685:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000800:00000200:30.0:1616513080.438687:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.438689:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.439638:0:49485:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.439692:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.439702:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.439711:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.136@tcp1 : PUT - for me 00000800:00000200:11.2:1616513080.439712:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513080.439719:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.136@tcp1 of length 224 into portal 26 MB=0x5ffdda4a58080 00000400:00000200:22.0:1616513080.439726:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.136@tcp1 of length 224/224 into md 0xf97c8bc9 [1] + 0 00000400:00000200:22.0:1616513080.439730:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22088 00000400:00000200:22.0:1616513080.439733:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.439736:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513080.439740:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22088 00000800:00000200:22.0:1616513080.439743:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.439745:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.439746:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.439750:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.439753:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702298718336 00010000:00000200:20.0:1616513080.439758:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688702298718336/t0(0) o400->1fd30c87-4033-a554-5dc9-bbe9962a35a8@10.31.141.136@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.439781:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702298718336, offset 192 00000400:00000200:20.0:1616513080.439786:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.136@tcp1 00000400:00000200:20.0:1616513080.439791:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.136@tcp1 routed destination 00000400:00000200:20.0:1616513080.439803:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.136@tcp1(10.31.141.136@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.439812:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.439816:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.439819:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.439820:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513080.439823:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:20.0:1616513080.439827:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513080.439846:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.439855:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513080.439855:0:49485:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.439863:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513080.439864:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.49@tcp1 : PUT - for me 00000800:00000200:30.0:1616513080.439868:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513080.439874:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.49@tcp1 of length 224 into portal 26 MB=0x60182f5d61680 00000400:00000200:30.0:1616513080.439875:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.49@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.439880:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.49@tcp1 of length 224/224 into md 0xf97c8bd9 [1] + 0 00000400:00000200:30.0:1616513080.439883:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.49@tcp1 of length 224 into portal 12 MB=0x60182f5d619c0 00000400:00000200:12.0:1616513080.439885:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22330 00000400:00000200:12.0:1616513080.439888:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.439889:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.49@tcp1 of length 224/224 into md 0xf964d519 [1] + 2688 00000400:00000200:30.0:1616513080.439894:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.439894:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.439899:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22330 00000100:00000200:30.0:1616513080.439901:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.439902:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513080.439904:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:30.0:1616513080.439906:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513080.439906:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:30.0:1616513080.439908:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:20.0:1616513080.439909:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511842088576 00000800:00000200:12.0:1616513080.439909:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513080.439910:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.439912:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513080.439912:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:20.0:1616513080.439914:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1690511842088576/t0(0) o400->38363629-34e8-ff95-a72b-c56472d2fd55@10.31.138.49@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513080.439915:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.439915:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.439920:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.439924:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e37e8 00000100:00000200:20.0:1616513080.439926:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511842088576, offset 192 00000800:00000200:12.0:1616513080.439927:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:20.0:1616513080.439931:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.49@tcp1 00000400:00000200:20.0:1616513080.439937:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.49@tcp1 routed destination 00000400:00000200:20.0:1616513080.439949:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.49@tcp1(10.31.138.49@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.439954:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.439958:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.439961:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.439962:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.439965:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:26.0:1616513080.439969:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511842089408 00000100:00000200:20.0:1616513080.439969:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.439969:0:49485:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.439979:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:26.0:1616513080.439981:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf81f80 x1690511842089408/t0(0) o400->46209bb5-9898-2aed-0cd2-c59fa6331a38@10.31.138.49@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513080.439984:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.439987:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513080.439989:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513080.439993:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:26.0:1616513080.439994:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511842089408, offset 192 00000400:00000200:18.0:1616513080.439998:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3a90 00000400:00000200:26.0:1616513080.440000:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.49@tcp1 00000800:00000200:18.0:1616513080.440002:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:26.0:1616513080.440008:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.49@tcp1 routed destination 00000400:00000200:26.0:1616513080.440022:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.49@tcp1(10.31.138.49@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.440028:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.440033:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.440036:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.440038:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.440042:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.440047:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.440101:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.440108:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.440111:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.440111:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.440118:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.440122:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa08 00000800:00000200:28.0:1616513080.440126:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513080.442587:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925920, portal 4 00000100:00000200:4.0:1616513080.442592:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925920, offset 0 00000400:00000200:4.0:1616513080.442598:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:4.0:1616513080.442604:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:4.0:1616513080.442616:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513080.442623:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:4.0:1616513080.442628:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513080.442631:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:4.0:1616513080.442633:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:4.0:1616513080.442638:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513080.442692:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.442712:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513080.442718:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513080.442721:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513080.442729:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513080.442736:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affd5580 x1693067725925920/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513080.442749:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb430 00000800:00000200:1.0:1616513080.442755:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513080.442758:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513080.442867:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513080.442877:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.442884:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513080.442892:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513080.442901:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626e20 00000400:00000200:1.0:1616513080.442908:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffbe5 [1] + 192 00000400:00000200:1.0:1616513080.442914:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513080.442918:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affd5580 x1693067725925920/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513080.442927:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513080.442933:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.442936:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513080.442938:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:4.0:1616513080.442942:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affd5580 x1693067725925920/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513080.442954:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affd5580 x1693067725925920/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513080.442964:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbdc0 00000800:00000200:11.2:1616513080.454016:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.454071:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.454080:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.454089:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.240@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.454099:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.240@tcp1 of length 224 into portal 26 MB=0x5ffd3d5df17c0 00000400:00000200:12.0:1616513080.454106:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.240@tcp1 of length 224/224 into md 0xf97c8be9 [1] + 0 00000400:00000200:12.0:1616513080.454110:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23100 00000400:00000200:12.0:1616513080.454113:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.454116:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.454120:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23100 00000800:00000200:24.0:1616513080.454124:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.454125:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.454127:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.454129:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.454181:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660174903232 00010000:00000200:20.0:1616513080.454198:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688660174903232/t0(0) o400->9899a0dd-01e7-6c47-8de3-33663f1c3b5c@10.31.128.240@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513080.454210:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513080.454211:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660174903232, offset 192 00000400:00000200:20.0:1616513080.454218:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.240@tcp1 00000800:00000200:8.0:1616513080.454220:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513080.454225:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.240@tcp1 routed destination 00000800:00000200:8.0:1616513080.454227:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.454233:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.240@tcp1 : PUT - for me 00000800:00000200:28.0:1616513080.454234:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513080.454239:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.240@tcp1(10.31.128.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513080.454241:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.240@tcp1 of length 224 into portal 12 MB=0x5ffd3d5df1b00 00000800:00000200:20.0:1616513080.454245:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513080.454246:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.240@tcp1 of length 224/224 into md 0x865bbec1 [1] + 30800 00000800:00000200:20.0:1616513080.454250:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:8.0:1616513080.454251:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513080.454253:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:8.0:1616513080.454253:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513080.454257:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513080.454258:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.454260:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513080.454261:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513080.454262:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.454267:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.454269:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513080.454284:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660174904064 00000800:00000200:30.0:1616513080.454288:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.454292:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.454294:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:0.0:1616513080.454296:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5e80 x1688660174904064/t0(0) o400->12f82363-6abc-f1a6-9942-48dcbb3c264c@10.31.128.240@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:6.0:1616513080.454297:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513080.454298:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.454302:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3870 00000800:00000200:30.0:1616513080.454304:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513080.454311:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660174904064, offset 192 00000400:00000200:0.0:1616513080.454317:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.240@tcp1 00000400:00000200:0.0:1616513080.454325:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.240@tcp1 routed destination 00000400:00000200:0.0:1616513080.454343:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.240@tcp1(10.31.128.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.454349:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.454355:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.454358:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.454360:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.454365:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.454370:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.454381:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.454386:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.454389:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.454389:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.454404:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.454408:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a90 00000800:00000200:24.0:1616513080.454412:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.462211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.462270:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.462279:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513080.462279:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.462289:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.81@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.462298:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.81@tcp1 of length 224 into portal 26 MB=0x60214cf186080 00000400:00000200:28.0:1616513080.462305:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.81@tcp1 of length 224/224 into md 0xf97c8c19 [1] + 0 00000400:00000200:28.0:1616513080.462310:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23320 00000400:00000200:28.0:1616513080.462312:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.462315:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513080.462320:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23320 00000800:00000200:28.0:1616513080.462323:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.462326:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.462328:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.462383:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691138257346688 00010000:00000200:20.0:1616513080.462393:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1691138257346688/t0(0) o400->5ad24bb0-68d0-b40e-80d2-dd52db77068f@10.31.129.81@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.462406:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691138257346688, offset 192 00000400:00000200:20.0:1616513080.462412:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.81@tcp1 00000400:00000200:20.0:1616513080.462419:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.81@tcp1 routed destination 00000400:00000200:20.0:1616513080.462445:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.81@tcp1(10.31.129.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.462451:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.462457:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.462460:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.462462:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.462466:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.462470:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.462473:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513080.462525:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.462537:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.462546:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.81@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.462555:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.81@tcp1 of length 224 into portal 12 MB=0x60214cf186540 00000400:00000200:22.0:1616513080.462561:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.81@tcp1 of length 224/224 into md 0xf964d519 [1] + 2912 00000400:00000200:22.0:1616513080.462566:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.462569:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513080.462575:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513080.462578:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513080.462580:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:22.0:1616513080.462588:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513080.462589:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513080.462591:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.462592:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.462598:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.462604:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3430 00000800:00000200:18.0:1616513080.462607:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.462636:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691138257347904 00010000:00000200:26.0:1616513080.462647:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec207080 x1691138257347904/t0(0) o400->a63fc4f7-daec-675f-4608-3ed5e75a3563@10.31.129.81@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.462661:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691138257347904, offset 192 00000400:00000200:26.0:1616513080.462666:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.81@tcp1 00000400:00000200:26.0:1616513080.462673:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.81@tcp1 routed destination 00000400:00000200:26.0:1616513080.462689:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.81@tcp1(10.31.129.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.462695:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.462700:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.462702:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.462704:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.462709:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.462714:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.462765:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.462772:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.462774:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.462781:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.462784:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af7e8 00000800:00000200:12.0:1616513080.462788:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.462791:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.473833:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.473845:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.473850:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.473857:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.73@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.473863:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.73@tcp1 of length 224 into portal 26 MB=0x5ffd3ccee3340 00000400:00000200:8.0:1616513080.473868:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.73@tcp1 of length 224/224 into md 0xf97c8c29 [1] + 0 00000400:00000200:8.0:1616513080.473872:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23dc0 00000400:00000200:8.0:1616513080.473874:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.473876:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.473879:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23dc0 00000800:00000200:8.0:1616513080.473881:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.473883:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.473886:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.473888:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.473941:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660024898368 00010000:00000200:20.0:1616513080.473951:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688660024898368/t0(0) o400->7d67fdca-350a-c260-b113-526157d81268@10.31.129.73@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.473964:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660024898368, offset 192 00000400:00000200:20.0:1616513080.473970:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.73@tcp1 00000400:00000200:20.0:1616513080.473977:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.73@tcp1 routed destination 00000400:00000200:20.0:1616513080.473990:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.73@tcp1(10.31.129.73@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.473996:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.474002:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.474004:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.474006:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.474010:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.474016:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.474018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.474029:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.474034:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.474036:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.474042:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.474046:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2c38 00000800:00000200:30.0:1616513080.474048:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.474050:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.474066:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.474119:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.474128:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.474136:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.73@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.474144:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.73@tcp1 of length 224 into portal 12 MB=0x5ffd3ccee37c0 00000400:00000200:12.0:1616513080.474150:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.73@tcp1 of length 224/224 into md 0x865bbec1 [1] + 31024 00000400:00000200:12.0:1616513080.474155:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.474157:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.474163:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.474166:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.474168:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.474170:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.474227:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660024899520 00010000:00000200:0.0:1616513080.474239:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663f00 x1688660024899520/t0(0) o400->7cde937a-ea3c-2eba-35a0-7f04e772479b@10.31.129.73@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.474252:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660024899520, offset 192 00000400:00000200:0.0:1616513080.474258:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.73@tcp1 00000400:00000200:0.0:1616513080.474265:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.73@tcp1 routed destination 00000400:00000200:0.0:1616513080.474279:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.73@tcp1(10.31.129.73@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.474285:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.474290:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.474293:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.474295:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.474299:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.474304:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.474312:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.474315:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.474317:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513080.474321:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513080.474322:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.474324:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb045d8 00000800:00000200:8.0:1616513080.474326:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513080.474534:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925936, portal 4 00000100:00000200:30.0:1616513080.474538:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925936, offset 0 00000400:00000200:30.0:1616513080.474542:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513080.474547:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513080.474557:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513080.474562:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513080.474567:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513080.474569:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513080.474571:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513080.474575:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513080.474623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513080.474636:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513080.474639:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513080.474641:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.474645:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:12.0:1616513080.474647:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:12.0:1616513080.474652:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74dd21f980 x1693067725925936/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513080.474663:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee198 00000800:00000200:12.0:1616513080.474666:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513080.474761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513080.474773:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513080.474780:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:8.0:1616513080.474780:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513080.474789:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513080.474797:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626e30 00000400:00000200:28.0:1616513080.474804:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d5db1 [1] + 192 00000400:00000200:28.0:1616513080.474809:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.474814:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74dd21f980 x1693067725925936/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513080.474831:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513080.474833:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:30.0:1616513080.474835:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74dd21f980 x1693067725925936/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513080.474835:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513080.474846:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74dd21f980 x1693067725925936/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513080.474853:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:11.2:1616513080.484588:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.484644:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.484654:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.484663:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.9@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.484671:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.9@tcp1 of length 224 into portal 26 MB=0x5ffdda3052ec0 00000400:00000200:24.0:1616513080.484677:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.9@tcp1 of length 224/224 into md 0xf97c8c49 [1] + 0 00000400:00000200:24.0:1616513080.484682:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee237e8 00000400:00000200:24.0:1616513080.484685:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.484688:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.484692:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee237e8 00000800:00000200:24.0:1616513080.484695:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.484697:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.484698:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.484704:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.484707:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.484708:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.484717:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.9@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.484725:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.9@tcp1 of length 224 into portal 12 MB=0x5ffdda3053140 00000400:00000200:12.0:1616513080.484733:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.9@tcp1 of length 224/224 into md 0x865bbec1 [1] + 31248 00000400:00000200:12.0:1616513080.484738:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.484741:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.484748:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.484751:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702271434432 00000800:00000200:12.0:1616513080.484751:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.484753:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:20.0:1616513080.484760:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688702271434432/t0(0) o400->72e5b74d-2165-4e02-8f57-74c0de59a9ab@10.31.140.9@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.484774:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702271434432, offset 192 00000400:00000200:20.0:1616513080.484780:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.9@tcp1 00000400:00000200:20.0:1616513080.484786:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.9@tcp1 routed destination 00000400:00000200:20.0:1616513080.484799:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.9@tcp1(10.31.140.9@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.484805:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.484810:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.484812:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.484814:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.484818:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.484824:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.484826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513080.484827:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702271435072 00010000:00000200:0.0:1616513080.484839:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c660000 x1688702271435072/t0(0) o400->ba8f3296-7242-0aee-ced4-a02e47d75955@10.31.140.9@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.484854:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702271435072, offset 192 00000400:00000200:0.0:1616513080.484860:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.9@tcp1 00000400:00000200:0.0:1616513080.484867:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.9@tcp1 routed destination 00000800:00000200:22.0:1616513080.484881:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513080.484881:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.9@tcp1(10.31.140.9@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513080.484887:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513080.484888:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513080.484890:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513080.484894:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513080.484897:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513080.484897:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.484899:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513080.484902:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3100 00000800:00000200:0.0:1616513080.484905:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513080.484906:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.484909:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.484910:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.484967:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.484974:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.484976:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.484976:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.484983:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.484987:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb054b8 00000800:00000200:28.0:1616513080.484991:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513080.490511:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925952, portal 4 00000100:00000200:30.0:1616513080.490514:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925952, offset 0 00000400:00000200:30.0:1616513080.490519:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:30.0:1616513080.490523:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:30.0:1616513080.490533:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513080.490537:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513080.490542:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513080.490544:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513080.490545:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513080.490549:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513080.490596:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513080.490609:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:30.0:1616513080.490612:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:30.0:1616513080.490614:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.490618:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:14.0:1616513080.490619:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000100:00000200:30.0:1616513080.490621:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dec201b00 x1693067725925952/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513080.490630:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef540 00000800:00000200:30.0:1616513080.490631:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513080.490735:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513080.490747:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513080.490756:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513080.490764:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513080.490772:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626e40 00000400:00000200:22.0:1616513080.490779:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c9349 [1] + 192 00000400:00000200:22.0:1616513080.490784:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.490789:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dec201b00 x1693067725925952/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513080.490799:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513080.490804:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000100:00000200:30.0:1616513080.490807:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dec201b00 x1693067725925952/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513080.490807:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513080.490809:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:30.0:1616513080.490816:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dec201b00 x1693067725925952/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513080.490823:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:11.2:1616513080.491968:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.492024:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.492034:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.492042:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.158@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.492051:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.158@tcp1 of length 224 into portal 26 MB=0x600774778cf00 00000400:00000200:24.0:1616513080.492057:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.158@tcp1 of length 224/224 into md 0xf97c8c69 [1] + 0 00000400:00000200:24.0:1616513080.492061:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee225d8 00000400:00000200:24.0:1616513080.492064:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.492066:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.492070:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee225d8 00000800:00000200:24.0:1616513080.492073:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.492075:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.492076:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.492080:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.492128:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513080.492130:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689362160471808 00010000:00000200:20.0:1616513080.492139:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1689362160471808/t0(0) o400->78cb81a3-5113-0886-9811-046bb1b860da@10.31.131.158@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.492152:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689362160471808, offset 192 00000400:00000200:20.0:1616513080.492157:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.158@tcp1 00000400:00000200:20.0:1616513080.492164:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.158@tcp1 routed destination 00000400:00000200:20.0:1616513080.492177:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.158@tcp1(10.31.131.158@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.492183:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.492188:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.492190:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.492192:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.492196:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.492196:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000100:00000200:20.0:1616513080.492203:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.492205:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513080.492205:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.492210:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:30.0:1616513080.492212:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513080.492213:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.158@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.492217:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.492220:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3650 00000400:00000200:14.0:1616513080.492221:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.158@tcp1 of length 224 into portal 12 MB=0x600774778d300 00000800:00000200:30.0:1616513080.492223:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.492225:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513080.492229:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.158@tcp1 of length 224/224 into md 0xf964d519 [1] + 3136 00000400:00000200:14.0:1616513080.492234:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513080.492236:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513080.492242:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.492245:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.492247:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.492306:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689362160472832 00010000:00000200:26.0:1616513080.492319:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba0900 x1689362160472832/t0(0) o400->5b72f23a-bc5c-e5f0-d1f9-055da06ffe3c@10.31.131.158@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.492333:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689362160472832, offset 192 00000400:00000200:26.0:1616513080.492339:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.158@tcp1 00000400:00000200:26.0:1616513080.492347:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.158@tcp1 routed destination 00000400:00000200:26.0:1616513080.492362:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.158@tcp1(10.31.131.158@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.492368:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.492374:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.492377:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.492379:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.492384:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.492389:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.492399:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.492403:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.492405:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513080.492410:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.492413:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae110 00000800:00000200:8.0:1616513080.492416:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.492418:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.497278:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.497344:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.497352:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.497361:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.35@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.497370:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.35@tcp1 of length 224 into portal 12 MB=0x5ffd3ed905c00 00000400:00000200:12.0:1616513080.497375:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.35@tcp1 of length 224/224 into md 0x865bbec1 [1] + 31472 00000400:00000200:12.0:1616513080.497380:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.497383:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.497389:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.497392:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.497394:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.497396:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.497461:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660572396544 00010000:00000200:0.0:1616513080.497472:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c662880 x1688660572396544/t0(0) o400->8cb1308e-a8b7-fc12-8623-65856360afb8@10.31.138.35@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.497486:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660572396544, offset 192 00000400:00000200:0.0:1616513080.497491:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.35@tcp1 00000400:00000200:0.0:1616513080.497499:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.35@tcp1 routed destination 00000400:00000200:0.0:1616513080.497512:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.35@tcp1(10.31.138.35@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.497518:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513080.497524:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.497527:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513080.497529:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513080.497533:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.497540:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.497595:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.497602:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.497604:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.497605:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.497611:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.497615:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a08 00000800:00000200:22.0:1616513080.497622:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.498745:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.498755:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.498760:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.498767:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.35@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.498773:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.35@tcp1 of length 224 into portal 26 MB=0x5ffd3ed905ac0 00000400:00000200:8.0:1616513080.498779:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.35@tcp1 of length 224/224 into md 0xf97c8c79 [1] + 0 00000400:00000200:8.0:1616513080.498782:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22880 00000400:00000200:8.0:1616513080.498785:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.498787:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.498790:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22880 00000800:00000200:8.0:1616513080.498792:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.498794:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.498795:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.498797:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.498851:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660572396224 00010000:00000200:20.0:1616513080.498861:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688660572396224/t0(0) o400->ae9847c0-add6-ab49-48bd-0e4f4a630ac3@10.31.138.35@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.498873:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660572396224, offset 192 00000400:00000200:20.0:1616513080.498878:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.35@tcp1 00000400:00000200:20.0:1616513080.498885:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.35@tcp1 routed destination 00000400:00000200:20.0:1616513080.498903:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.35@tcp1(10.31.138.35@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.498909:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.498914:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.498916:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.498918:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.498922:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.498927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.498928:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513080.498981:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.498987:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.498990:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.498997:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.499002:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2a18 00000800:00000200:12.0:1616513080.499006:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.499009:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.525239:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513080.525294:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.525303:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513080.525312:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.155@tcp1 : PUT - for me 00000400:00000200:14.0:1616513080.525322:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.155@tcp1 of length 224 into portal 26 MB=0x5ffd54e431e00 00000400:00000200:14.0:1616513080.525329:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.155@tcp1 of length 224/224 into md 0xf97c8c99 [1] + 0 00000400:00000200:14.0:1616513080.525334:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22770 00000400:00000200:14.0:1616513080.525336:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513080.525339:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513080.525344:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22770 00000800:00000200:14.0:1616513080.525347:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.525348:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.525349:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.525352:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.525408:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666489691648 00010000:00000200:20.0:1616513080.525419:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688666489691648/t0(0) o400->9d70f12a-5430-edfe-c9e4-332a6b77b864@10.31.132.155@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.525443:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666489691648, offset 192 00000400:00000200:20.0:1616513080.525450:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.155@tcp1 00000400:00000200:20.0:1616513080.525463:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.155@tcp1 routed destination 00000400:00000200:20.0:1616513080.525478:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.155@tcp1(10.31.132.155@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.525485:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.525491:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.525493:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513080.525493:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.525495:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.525500:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513080.525506:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513080.525506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.525507:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513080.525512:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:30.0:1616513080.525516:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.525520:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:8.0:1616513080.525520:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.155@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.525522:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.525527:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.525527:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.155@tcp1 of length 224 into portal 12 MB=0x5ffd54e431e80 00000400:00000200:30.0:1616513080.525530:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2f68 00000800:00000200:30.0:1616513080.525533:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513080.525534:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.155@tcp1 of length 224/224 into md 0x865bbec1 [1] + 31696 00000800:00000200:30.0:1616513080.525535:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513080.525539:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.525541:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513080.525546:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.525549:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.525550:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.525552:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.525614:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666489691776 00010000:00000200:0.0:1616513080.525626:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4c80 x1688666489691776/t0(0) o400->a028ce73-58ef-44c0-5067-0489acf51e73@10.31.132.155@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.525639:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666489691776, offset 192 00000400:00000200:0.0:1616513080.525645:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.155@tcp1 00000400:00000200:0.0:1616513080.525652:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.155@tcp1 routed destination 00000400:00000200:0.0:1616513080.525666:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.155@tcp1(10.31.132.155@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.525672:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.525677:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.525679:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.525682:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.525686:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.525691:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.525703:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.525710:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.525713:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.525720:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.525724:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05188 00000800:00000200:12.0:1616513080.525728:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.525731:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.527243:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.527253:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.527257:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.527264:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.196@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.527269:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.196@tcp1 of length 224 into portal 12 MB=0x5ffdcb3f56ec0 00000400:00000200:8.0:1616513080.527274:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.196@tcp1 of length 224/224 into md 0x865bbec1 [1] + 31920 00000400:00000200:8.0:1616513080.527277:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.527279:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513080.527282:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.527284:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.527285:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.527287:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.527343:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698260647616 00010000:00000200:0.0:1616513080.527354:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0900 x1688698260647616/t0(0) o400->d6352a9a-6817-9fbe-6735-478d72120b03@10.31.141.196@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.527367:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698260647616, offset 192 00000400:00000200:0.0:1616513080.527372:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.196@tcp1 00000800:00000200:7.2:1616513080.527380:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513080.527380:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.196@tcp1 routed destination 00000800:00000200:18.0:1616513080.527393:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513080.527393:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.196@tcp1(10.31.141.196@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.527398:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513080.527402:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513080.527404:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.527406:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513080.527408:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:18.0:1616513080.527411:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.196@tcp1 : PUT - for me 00000800:00000200:0.0:1616513080.527412:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:18.0:1616513080.527419:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.196@tcp1 of length 224 into portal 26 MB=0x5ffdcb3f56c40 00000400:00000200:18.0:1616513080.527426:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.196@tcp1 of length 224/224 into md 0xf97c8cb9 [1] + 0 00000400:00000200:18.0:1616513080.527430:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22ee0 00000400:00000200:18.0:1616513080.527447:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.527448:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000100:00000200:18.0:1616513080.527450:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513080.527454:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ee0 00000800:00000200:14.0:1616513080.527456:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513080.527457:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:30.0:1616513080.527458:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513080.527458:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513080.527462:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.527464:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513080.527464:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513080.527468:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb056d8 00000800:00000200:14.0:1616513080.527472:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.527517:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698260646976 00010000:00000200:20.0:1616513080.527528:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688698260646976/t0(0) o400->cb85c312-2a44-973e-7f20-2725d7106e01@10.31.141.196@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.527543:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698260646976, offset 192 00000400:00000200:20.0:1616513080.527549:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.196@tcp1 00000400:00000200:20.0:1616513080.527556:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.196@tcp1 routed destination 00000400:00000200:20.0:1616513080.527571:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.196@tcp1(10.31.141.196@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.527577:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.527583:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.527587:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.527589:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.527594:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.527600:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.527601:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513080.527613:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.527620:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.527627:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.527634:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.527639:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2aa0 00000800:00000200:12.0:1616513080.527643:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.527646:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.541014:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.541028:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.541034:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.541042:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.154@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.541049:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.154@tcp1 of length 224 into portal 26 MB=0x5ffd4f317cd80 00000400:00000200:8.0:1616513080.541055:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.154@tcp1 of length 224/224 into md 0xf97c8cd9 [1] + 0 00000400:00000200:8.0:1616513080.541059:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee223b8 00000400:00000200:8.0:1616513080.541061:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.541063:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.541067:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee223b8 00000800:00000200:8.0:1616513080.541069:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.541071:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.541073:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.541075:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.541130:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664960126336 00010000:00000200:20.0:1616513080.541141:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688664960126336/t0(0) o400->d0cafdc2-cdc4-57d2-015d-b4b63f2d15b7@10.31.132.154@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.541155:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664960126336, offset 192 00000400:00000200:20.0:1616513080.541161:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.154@tcp1 00000800:00000200:11.2:1616513080.541161:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:20.0:1616513080.541169:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.154@tcp1 routed destination 00000400:00000200:20.0:1616513080.541184:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.154@tcp1(10.31.132.154@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.541191:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.541196:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.541199:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.541201:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.541206:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.541213:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.541213:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513080.541215:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.541224:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.541232:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.154@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.541241:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.154@tcp1 of length 224 into portal 12 MB=0x5ffd4f317cec0 00000400:00000200:12.0:1616513080.541247:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.154@tcp1 of length 224/224 into md 0x865bbec1 [1] + 32144 00000400:00000200:12.0:1616513080.541252:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.541255:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.541261:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.541264:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.541266:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513080.541269:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513080.541269:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513080.541277:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.541279:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.541279:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.541286:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.541292:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e36d8 00000800:00000200:22.0:1616513080.541295:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513080.541324:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664960126656 00010000:00000200:0.0:1616513080.541335:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663180 x1688664960126656/t0(0) o400->c1c91358-7d75-688c-fe7b-40402d3d1361@10.31.132.154@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.541347:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664960126656, offset 192 00000400:00000200:0.0:1616513080.541353:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.154@tcp1 00000400:00000200:0.0:1616513080.541359:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.154@tcp1 routed destination 00000400:00000200:0.0:1616513080.541372:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.154@tcp1(10.31.132.154@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.541378:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.541382:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.541385:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.541387:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.541391:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.541396:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.541404:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.541408:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.541409:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513080.541414:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513080.541414:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.541417:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04550 00000800:00000200:8.0:1616513080.541419:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.546150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513080.546208:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.546217:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.546227:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.184@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.546236:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.184@tcp1 of length 224 into portal 26 MB=0x6059fb564ef80 00000400:00000200:18.0:1616513080.546242:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.184@tcp1 of length 224/224 into md 0xf97c8ce9 [1] + 0 00000400:00000200:18.0:1616513080.546247:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23b18 00000400:00000200:18.0:1616513080.546249:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.546252:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513080.546256:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23b18 00000800:00000200:18.0:1616513080.546259:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.546260:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.546262:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.546264:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.546318:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695033361493888 00000800:00000200:11.2:1616513080.546327:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:20.0:1616513080.546328:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1695033361493888/t0(0) o400->2d09cfd2-3e7a-69de-68ed-27980b688948@10.31.129.184@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.546343:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695033361493888, offset 192 00000400:00000200:20.0:1616513080.546349:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.184@tcp1 00000400:00000200:20.0:1616513080.546355:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.184@tcp1 routed destination 00000400:00000200:20.0:1616513080.546370:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.184@tcp1(10.31.129.184@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.546376:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.546381:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.546383:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.546385:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513080.546385:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.546390:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513080.546394:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513080.546395:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.546396:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513080.546402:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.184@tcp1 : PUT - for me 00000800:00000200:30.0:1616513080.546403:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.546407:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.546408:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.546410:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.184@tcp1 of length 224 into portal 12 MB=0x6059fb564f000 00000400:00000200:30.0:1616513080.546413:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.546416:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.184@tcp1 of length 224/224 into md 0x865bbec1 [1] + 32368 00000400:00000200:30.0:1616513080.546421:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e36d8 00000400:00000200:12.0:1616513080.546421:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.546423:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513080.546424:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.546425:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513080.546430:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.546432:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.546449:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513080.546456:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.546459:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.546460:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.546468:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.34@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.546477:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.34@tcp1 of length 224 into portal 26 MB=0x5ffefca502440 00000400:00000200:24.0:1616513080.546484:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.34@tcp1 of length 224/224 into md 0xf97c8d09 [1] + 0 00000400:00000200:24.0:1616513080.546489:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23188 00000400:00000200:24.0:1616513080.546491:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513080.546491:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695033361494016 00000100:00000200:24.0:1616513080.546494:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.546499:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23188 00010000:00000200:0.0:1616513080.546501:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663a80 x1695033361494016/t0(0) o400->a7a79895-5a32-0a8b-66f5-c861d70e97f2@10.31.129.184@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513080.546502:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.546505:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.546507:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.546511:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688780240069696 00000100:00000200:0.0:1616513080.546513:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695033361494016, offset 192 00010000:00000200:20.0:1616513080.546516:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688780240069696/t0(0) o400->c7c5011e-b882-711a-e710-e1aa278812a6@10.31.141.34@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513080.546518:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.184@tcp1 00000400:00000200:0.0:1616513080.546524:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.184@tcp1 routed destination 00000100:00000200:20.0:1616513080.546527:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688780240069696, offset 192 00000400:00000200:20.0:1616513080.546532:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.34@tcp1 00000400:00000200:0.0:1616513080.546536:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.184@tcp1(10.31.129.184@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.546541:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513080.546543:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.34@tcp1 routed destination 00000800:00000200:0.0:1616513080.546549:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.546551:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513080.546552:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513080.546554:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513080.546555:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.34@tcp1(10.31.141.34@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.546558:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513080.546561:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513080.546563:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513080.546564:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.546565:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.546568:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.546569:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513080.546570:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.546572:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513080.546572:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:8.0:1616513080.546572:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.546575:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:20.0:1616513080.546576:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:8.0:1616513080.546577:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513080.546577:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513080.546578:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.34@tcp1 : PUT - for me 00000800:00000200:12.0:1616513080.546579:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513080.546581:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.546583:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb042a8 00000800:00000200:30.0:1616513080.546585:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:8.0:1616513080.546585:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513080.546586:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.34@tcp1 of length 224 into portal 12 MB=0x5ffefca5024c0 00000800:00000200:30.0:1616513080.546588:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:30.0:1616513080.546590:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513080.546592:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.34@tcp1 of length 224/224 into md 0xf964d519 [1] + 3360 00000400:00000200:30.0:1616513080.546594:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513080.546594:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.546597:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2f68 00000400:00000200:14.0:1616513080.546597:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513080.546599:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513080.546599:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513080.546608:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.546610:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.546612:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.546672:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688780240069824 00010000:00000200:26.0:1616513080.546684:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba5a00 x1688780240069824/t0(0) o400->d428ea2a-139f-5c06-5fb3-c4acbef5b58c@10.31.141.34@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.546698:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688780240069824, offset 192 00000400:00000200:26.0:1616513080.546705:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.34@tcp1 00000400:00000200:26.0:1616513080.546712:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.34@tcp1 routed destination 00000400:00000200:26.0:1616513080.546727:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.34@tcp1(10.31.141.34@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.546733:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.546739:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.546742:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.546746:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.546750:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.546755:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.546767:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.546774:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.546777:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.546778:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.546784:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.546788:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aec38 00000800:00000200:28.0:1616513080.546792:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:5.0:1616513080.554840:0:6488:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925968, portal 4 00000100:00000200:5.0:1616513080.554845:0:6488:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725925968, offset 0 00000400:00000200:5.0:1616513080.554851:0:6488:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:5.0:1616513080.554857:0:6488:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:5.0:1616513080.554870:0:6488:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:5.0:1616513080.554877:0:6488:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513080.554883:0:6488:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513080.554886:0:6488:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:5.0:1616513080.554888:0:6488:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513080.554893:0:6488:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513080.554938:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.554956:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513080.554962:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513080.554964:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513080.554971:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513080.554978:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8302327080 x1693067725925968/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513080.554990:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55e990 00000800:00000200:1.0:1616513080.554997:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513080.555000:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513080.555108:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513080.555118:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.555124:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513080.555126:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513080.555132:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513080.555141:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626e50 00000400:00000200:1.0:1616513080.555147:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffbf5 [1] + 192 00000400:00000200:1.0:1616513080.555152:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513080.555156:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8302327080 x1693067725925968/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513080.555170:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513080.555173:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513080.555175:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:5.0:1616513080.555176:0:6488:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8302327080 x1693067725925968/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:5.0:1616513080.555187:0:6488:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8302327080 x1693067725925968/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513080.555196:0:6488:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55e7f8 00000800:00000200:7.2:1616513080.568915:0:49498:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513080.568971:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.568987:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.568996:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.135@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.569006:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.135@tcp1 of length 224 into portal 26 MB=0x5ffd2f9e3dbc0 00000400:00000200:18.0:1616513080.569013:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.135@tcp1 of length 224/224 into md 0xf97c8d19 [1] + 0 00000400:00000200:18.0:1616513080.569017:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22198 00000400:00000200:18.0:1616513080.569020:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.569023:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513080.569027:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22198 00000800:00000200:18.0:1616513080.569031:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.569033:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.569034:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.569038:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.569092:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656484228032 00010000:00000200:20.0:1616513080.569102:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688656484228032/t0(0) o400->fe589227-62f7-366f-1908-fdbc0dc64bcf@10.31.129.135@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.569116:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688656484228032, offset 192 00000400:00000200:20.0:1616513080.569122:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.135@tcp1 00000400:00000200:20.0:1616513080.569129:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.135@tcp1 routed destination 00000400:00000200:20.0:1616513080.569143:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.135@tcp1(10.31.129.135@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.569149:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.569155:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.569158:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:7.2:1616513080.569158:0:49498:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.569160:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.569165:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.569169:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:20.0:1616513080.569171:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.569177:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.569185:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.135@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.569192:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.135@tcp1 of length 224 into portal 12 MB=0x5ffd2f9e3df00 00000400:00000200:30.0:1616513080.569198:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.135@tcp1 of length 224/224 into md 0xf964d519 [1] + 3584 00000400:00000200:30.0:1616513080.569202:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.569206:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513080.569212:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513080.569215:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513080.569216:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.569218:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.569222:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.569224:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.569224:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513080.569228:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.569232:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3650 00000800:00000200:30.0:1616513080.569234:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.569278:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656484228864 00010000:00000200:26.0:1616513080.569291:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf83600 x1688656484228864/t0(0) o400->1a4aa61b-d8e9-3593-7e9b-3880410d8c42@10.31.129.135@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.569305:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688656484228864, offset 192 00000400:00000200:26.0:1616513080.569311:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.135@tcp1 00000400:00000200:26.0:1616513080.569319:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.135@tcp1 routed destination 00000400:00000200:26.0:1616513080.569334:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.135@tcp1(10.31.129.135@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.569340:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.569346:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.569349:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.569351:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.569356:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.569363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.569418:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.569425:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.569427:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.569429:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.569446:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.569450:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae770 00000800:00000200:12.0:1616513080.569456:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.570161:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.570215:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.570225:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513080.570232:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.570234:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.177@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.570243:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.177@tcp1 of length 224 into portal 26 MB=0x603fefb654080 00000400:00000200:24.0:1616513080.570250:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.177@tcp1 of length 224/224 into md 0xf97c8d39 [1] + 0 00000400:00000200:24.0:1616513080.570254:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22e58 00000400:00000200:24.0:1616513080.570257:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.570262:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.570267:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22e58 00000800:00000200:24.0:1616513080.570270:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.570273:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.570274:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.570277:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693243534557312 00010000:00000200:20.0:1616513080.570282:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1693243534557312/t0(0) o400->1f219f48-197d-22fb-3be9-6ac72b7b1228@10.31.131.177@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.570292:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693243534557312, offset 192 00000400:00000200:20.0:1616513080.570299:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.177@tcp1 00000400:00000200:20.0:1616513080.570304:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.177@tcp1 routed destination 00000400:00000200:20.0:1616513080.570315:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.177@tcp1(10.31.131.177@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.570320:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.570324:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.570326:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.570328:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.570331:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.570334:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.570337:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.570392:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.570399:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.570402:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.570404:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513080.570409:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.570414:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2c38 00000800:00000200:18.0:1616513080.570418:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.570826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.570836:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.570843:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.570851:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.177@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.570858:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.177@tcp1 of length 224 into portal 12 MB=0x603fefb6541c0 00000400:00000200:8.0:1616513080.570864:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.177@tcp1 of length 224/224 into md 0x865bbec1 [1] + 32592 00000400:00000200:8.0:1616513080.570869:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.570871:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513080.570877:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.570880:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.570882:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.570884:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.570936:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693243534557632 00010000:00000200:0.0:1616513080.570946:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5580 x1693243534557632/t0(0) o400->7481fdd4-8a88-021b-7ce9-be0334a2032b@10.31.131.177@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.570957:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693243534557632, offset 192 00000400:00000200:0.0:1616513080.570962:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.177@tcp1 00000400:00000200:0.0:1616513080.570969:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.177@tcp1 routed destination 00000400:00000200:0.0:1616513080.570981:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.177@tcp1(10.31.131.177@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.570986:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.570991:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.570993:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.570994:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.570998:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.571003:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.571014:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.571019:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.571022:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.571024:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.571029:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.571032:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb043b8 00000800:00000200:28.0:1616513080.571036:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.575309:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.575365:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.575374:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.575374:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.575380:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.575383:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.138@tcp1 : PUT - for me 00000800:00000200:14.0:1616513080.575384:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.575386:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.138@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.575393:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.138@tcp1 of length 224 into portal 12 MB=0x5ffd6e0f7e600 00000400:00000200:22.0:1616513080.575394:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.138@tcp1 of length 224 into portal 26 MB=0x5ffd6e0f7e580 00000400:00000200:30.0:1616513080.575398:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.138@tcp1 of length 224/224 into md 0xf964d519 [1] + 3808 00000400:00000200:30.0:1616513080.575401:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.575404:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.138@tcp1 of length 224/224 into md 0xf97c8d69 [1] + 0 00000400:00000200:22.0:1616513080.575409:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23650 00000100:00000200:30.0:1616513080.575411:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:22.0:1616513080.575412:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513080.575416:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:22.0:1616513080.575416:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:30.0:1616513080.575418:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.575419:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.575420:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23650 00000800:00000200:22.0:1616513080.575423:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.575425:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.575427:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.575481:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673245980160 00010000:00000200:26.0:1616513080.575493:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf85a00 x1688673245980160/t0(0) o400->6b4d635f-9636-fbee-03d0-854a43493aa8@10.31.143.138@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513080.575499:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673245980032 00000100:00000200:26.0:1616513080.575508:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688673245980160, offset 192 00010000:00000200:20.0:1616513080.575508:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688673245980032/t0(0) o400->732359c3-69db-04a9-8d5c-8c2df32fc693@10.31.143.138@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513080.575514:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.138@tcp1 00000400:00000200:26.0:1616513080.575521:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.138@tcp1 routed destination 00000100:00000200:20.0:1616513080.575521:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688673245980032, offset 192 00000400:00000200:20.0:1616513080.575526:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.138@tcp1 00000400:00000200:20.0:1616513080.575533:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.138@tcp1 routed destination 00000400:00000200:26.0:1616513080.575536:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.138@tcp1(10.31.143.138@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.575543:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513080.575546:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.138@tcp1(10.31.143.138@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.575548:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.575551:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.575552:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.575553:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.575557:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.575558:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513080.575559:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.575561:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513080.575563:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.575565:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513080.575571:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513080.575572:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.575581:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.575585:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.575587:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513080.575592:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.575596:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3870 00000800:00000200:8.0:1616513080.575598:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.575600:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513080.575616:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.575623:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.575626:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.575626:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513080.575632:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.575636:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae220 00000800:00000200:18.0:1616513080.575640:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.596822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.596877:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.596887:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.596896:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.94@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.596905:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.94@tcp1 of length 224 into portal 26 MB=0x602be2fcc6000 00000400:00000200:24.0:1616513080.596912:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.94@tcp1 of length 224/224 into md 0xf97c8d79 [1] + 0 00000400:00000200:24.0:1616513080.596917:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22990 00000400:00000200:24.0:1616513080.596920:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.596923:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.596927:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22990 00000800:00000200:28.0:1616513080.596931:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.596931:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.596933:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.596935:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.596992:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691865729228800 00010000:00000200:20.0:1616513080.597003:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1691865729228800/t0(0) o400->d6b25d83-987f-56bd-807d-691a5375d85f@10.31.136.94@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.597017:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691865729228800, offset 192 00000400:00000200:20.0:1616513080.597024:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.94@tcp1 00000400:00000200:20.0:1616513080.597031:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.94@tcp1 routed destination 00000400:00000200:20.0:1616513080.597046:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.94@tcp1(10.31.136.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.597061:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.597067:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.597070:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.597072:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.597077:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.597084:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.597084:0:49501:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.597139:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.597146:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.597149:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.597156:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.597161:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e37e8 00000800:00000200:30.0:1616513080.597165:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.597168:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.597235:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.597245:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.597252:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.597259:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.94@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.597267:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.94@tcp1 of length 224 into portal 12 MB=0x602be2fcc6180 00000400:00000200:8.0:1616513080.597273:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.94@tcp1 of length 224/224 into md 0x865bbec1 [1] + 32816 00000400:00000200:8.0:1616513080.597278:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.597280:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513080.597287:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.597289:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.597291:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.597293:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.597351:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691865729229184 00010000:00000200:0.0:1616513080.597364:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7500 x1691865729229184/t0(0) o400->09b740f8-55a2-b92c-8bb2-7655eaaae045@10.31.136.94@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.597378:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691865729229184, offset 192 00000400:00000200:0.0:1616513080.597384:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.94@tcp1 00000400:00000200:0.0:1616513080.597392:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.94@tcp1 routed destination 00000400:00000200:0.0:1616513080.597407:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.94@tcp1(10.31.136.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.597413:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.597419:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.597422:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.597424:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.597429:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.597434:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.597447:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.597453:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.597455:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.597457:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.597462:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.597466:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05980 00000800:00000200:28.0:1616513080.597470:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.614404:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513080.614416:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.614423:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.614431:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.181@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.614446:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.181@tcp1 of length 224 into portal 26 MB=0x5ffd40b0e5bc0 00000400:00000200:8.0:1616513080.614451:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.181@tcp1 of length 224/224 into md 0xf97c8da9 [1] + 0 00000400:00000200:8.0:1616513080.614455:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22550 00000400:00000200:8.0:1616513080.614458:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.614460:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.614464:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22550 00000800:00000200:8.0:1616513080.614467:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.614469:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513080.614469:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.614475:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513080.614478:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.614479:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.614488:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.181@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.614497:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.181@tcp1 of length 224 into portal 12 MB=0x5ffd40b0e5c40 00000400:00000200:12.0:1616513080.614503:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.181@tcp1 of length 224/224 into md 0x865bbec1 [1] + 33040 00000400:00000200:12.0:1616513080.614509:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.614511:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.614518:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.614521:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.614523:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.614544:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661067193280 00010000:00000200:20.0:1616513080.614555:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688661067193280/t0(0) o400->87cf82b0-b5ee-5e42-4263-b4ffcdd0f483@10.31.131.181@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.614568:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661067193280, offset 192 00000400:00000200:20.0:1616513080.614574:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.181@tcp1 00000400:00000200:20.0:1616513080.614581:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.181@tcp1 routed destination 00000100:00000200:0.0:1616513080.614582:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661067193408 00000400:00000200:20.0:1616513080.614595:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.181@tcp1(10.31.131.181@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513080.614595:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661200 x1688661067193408/t0(0) o400->2e09752f-9343-8861-83e4-7e4998a59305@10.31.131.181@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.614601:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.614606:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.614609:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513080.614609:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661067193408, offset 192 00000800:00000200:20.0:1616513080.614611:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.614615:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513080.614618:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.181@tcp1 00000100:00000200:20.0:1616513080.614622:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.614622:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513080.614625:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.181@tcp1 routed destination 00000800:00000200:14.0:1616513080.614636:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513080.614640:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.181@tcp1(10.31.131.181@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513080.614643:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513080.614645:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513080.614647:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513080.614652:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513080.614653:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.614655:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513080.614658:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2660 00000800:00000200:0.0:1616513080.614658:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513080.614661:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513080.614663:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513080.614664:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.614668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.614722:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.614729:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.614732:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.614732:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.614739:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.614745:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05298 00000800:00000200:28.0:1616513080.614749:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.615478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.615487:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.615493:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513080.615497:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.615500:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.162@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.615507:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.162@tcp1 of length 224 into portal 26 MB=0x5ffe852b569c0 00000400:00000200:12.0:1616513080.615513:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.162@tcp1 of length 224/224 into md 0xf97c8db9 [1] + 0 00000400:00000200:12.0:1616513080.615517:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee227f8 00000400:00000200:12.0:1616513080.615519:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.615522:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.615526:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee227f8 00000800:00000200:12.0:1616513080.615528:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.615530:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.615532:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.615538:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748168669632 00010000:00000200:20.0:1616513080.615543:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688748168669632/t0(0) o400->807b7a98-ed9f-6251-afad-abd40146696a@10.31.141.162@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.615555:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688748168669632, offset 192 00000400:00000200:20.0:1616513080.615561:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.162@tcp1 00000400:00000200:20.0:1616513080.615567:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.162@tcp1 routed destination 00000400:00000200:20.0:1616513080.615580:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.162@tcp1(10.31.141.162@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.615586:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.615590:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.615593:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.615597:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.615600:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.615604:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.615605:0:49503:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.615616:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.615622:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.615624:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.615631:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.615635:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2660 00000800:00000200:30.0:1616513080.615639:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.615641:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.615671:0:49503:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513080.615687:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.615694:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513080.615702:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.615702:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513080.615702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513080.615704:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.615706:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513080.615710:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.615715:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513080.615719:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513080.615721:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.162@tcp1 : PUT - for me 00000400:00000200:8.0:1616513080.615727:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.162@tcp1 of length 224 into portal 12 MB=0x5ffe852b56b00 00000400:00000200:8.0:1616513080.615731:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.162@tcp1 of length 224/224 into md 0x865bbec1 [1] + 33264 00000400:00000200:8.0:1616513080.615735:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.615737:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513080.615741:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.615743:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.615745:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.615753:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748168669952 00010000:00000200:0.0:1616513080.615759:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7980 x1688748168669952/t0(0) o400->e76cf9d3-4360-d05d-2bb7-f782684b3496@10.31.141.162@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.615770:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688748168669952, offset 192 00000400:00000200:0.0:1616513080.615775:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.162@tcp1 00000400:00000200:0.0:1616513080.615780:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.162@tcp1 routed destination 00000400:00000200:0.0:1616513080.615792:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.162@tcp1(10.31.141.162@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.615797:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.615802:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.615804:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.615806:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.615808:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.615813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.615823:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.615829:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.615831:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.615832:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.615838:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.615842:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04198 00000800:00000200:24.0:1616513080.615846:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.626952:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.627008:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.627018:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513080.627019:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513080.627026:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.151@tcp1 : PUT - for me 00000800:00000200:8.0:1616513080.627027:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513080.627035:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.151@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.627036:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.151@tcp1 of length 224 into portal 26 MB=0x6059534005bc0 00000400:00000200:28.0:1616513080.627042:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.151@tcp1 of length 224/224 into md 0xf97c8dd9 [1] + 0 00000400:00000200:8.0:1616513080.627043:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.151@tcp1 of length 224 into portal 12 MB=0x6059534005d00 00000400:00000200:28.0:1616513080.627048:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22b28 00000400:00000200:8.0:1616513080.627049:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.151@tcp1 of length 224/224 into md 0x865bbec1 [1] + 33488 00000400:00000200:28.0:1616513080.627050:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.627053:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513080.627053:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513080.627056:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513080.627058:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22b28 00000800:00000200:28.0:1616513080.627061:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513080.627062:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513080.627064:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:8.0:1616513080.627065:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:28.0:1616513080.627066:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513080.627068:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.627070:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.627121:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694988240968640 00010000:00000200:20.0:1616513080.627132:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1694988240968640/t0(0) o400->4d970e93-bed3-b6ac-5a2b-ce3754c3921a@10.31.143.151@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513080.627144:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694988240968960 00000100:00000200:20.0:1616513080.627146:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694988240968640, offset 192 00000400:00000200:20.0:1616513080.627152:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.151@tcp1 00010000:00000200:0.0:1616513080.627156:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5a00 x1694988240968960/t0(0) o400->9fdf5074-0788-d889-6363-8ace46dcb975@10.31.143.151@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513080.627160:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.151@tcp1 routed destination 00000100:00000200:0.0:1616513080.627171:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694988240968960, offset 192 00000400:00000200:20.0:1616513080.627175:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.151@tcp1(10.31.143.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.627182:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.627187:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:0.0:1616513080.627187:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.151@tcp1 00000800:00000200:20.0:1616513080.627190:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.627193:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513080.627194:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.151@tcp1 routed destination 00000800:00000200:20.0:1616513080.627197:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.627204:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.627204:0:49504:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513080.627208:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.151@tcp1(10.31.143.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513080.627214:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513080.627214:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513080.627218:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513080.627219:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513080.627220:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513080.627224:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:30.0:1616513080.627225:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513080.627226:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513080.627229:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3870 00000800:00000200:0.0:1616513080.627230:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513080.627231:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.627234:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.627235:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.627293:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.627301:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.627303:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513080.627303:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.627310:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.627314:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04000 00000800:00000200:24.0:1616513080.627318:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.640254:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.640310:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.640319:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.640328:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.197@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.640338:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.197@tcp1 of length 224 into portal 26 MB=0x605a073b08fc0 00000400:00000200:28.0:1616513080.640345:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.197@tcp1 of length 224/224 into md 0xf97c8de9 [1] + 0 00000400:00000200:28.0:1616513080.640349:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23a08 00000400:00000200:28.0:1616513080.640352:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.640355:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513080.640359:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23a08 00000800:00000200:28.0:1616513080.640362:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.640364:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513080.640371:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513080.640375:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.640377:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513080.640433:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.640455:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036554121152 00000800:00000200:18.0:1616513080.640455:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.640464:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.197@tcp1 : PUT - for me 00010000:00000200:20.0:1616513080.640466:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1695036554121152/t0(0) o400->45c822dc-e496-50d2-4e25-f396f71f2771@10.31.129.197@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513080.640473:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.197@tcp1 of length 224 into portal 12 MB=0x605a073b09040 00000100:00000200:20.0:1616513080.640479:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036554121152, offset 192 00000400:00000200:18.0:1616513080.640480:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.197@tcp1 of length 224/224 into md 0xf964d519 [1] + 4032 00000400:00000200:20.0:1616513080.640485:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.197@tcp1 00000400:00000200:18.0:1616513080.640485:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.640487:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:18.0:1616513080.640488:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513080.640492:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.197@tcp1 routed destination 00000800:00000200:18.0:1616513080.640496:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.640499:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.640500:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513080.640506:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.197@tcp1(10.31.129.197@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.640512:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.640518:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.640520:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.640522:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.640527:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.640532:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.640533:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513080.640562:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036554121280 00010000:00000200:26.0:1616513080.640575:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bff500 x1695036554121280/t0(0) o400->6d1f388c-979f-d306-0852-da7644527a7c@10.31.129.197@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:30.0:1616513080.640584:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:26.0:1616513080.640589:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036554121280, offset 192 00000800:00000200:30.0:1616513080.640590:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.640593:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513080.640598:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.197@tcp1 00000400:00000200:30.0:1616513080.640599:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.640604:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3650 00000400:00000200:26.0:1616513080.640606:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.197@tcp1 routed destination 00000800:00000200:30.0:1616513080.640612:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.640615:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:26.0:1616513080.640622:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.197@tcp1(10.31.129.197@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.640628:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.640634:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.640636:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.640639:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.640644:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.640649:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513080.640658:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513080.640663:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513080.640666:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513080.640672:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513080.640675:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae6e8 00000800:00000200:8.0:1616513080.640679:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513080.640681:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.647107:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.647162:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.647172:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.647181:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.18@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.647190:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.18@tcp1 of length 224 into portal 26 MB=0x60393ee82b040 00000400:00000200:12.0:1616513080.647196:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.18@tcp1 of length 224/224 into md 0xf97c8e09 [1] + 0 00000400:00000200:12.0:1616513080.647201:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22c38 00000400:00000200:12.0:1616513080.647204:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.647213:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513080.647216:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513080.647217:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22c38 00000800:00000200:12.0:1616513080.647220:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513080.647223:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513080.647225:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513080.647226:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513080.647228:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.647235:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.18@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.647243:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.18@tcp1 of length 224 into portal 12 MB=0x60393ee82b100 00000400:00000200:28.0:1616513080.647250:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.18@tcp1 of length 224/224 into md 0x865bbec1 [1] + 33712 00000400:00000200:28.0:1616513080.647257:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.647260:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513080.647266:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.647270:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.647272:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.647281:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783756881984 00010000:00000200:20.0:1616513080.647292:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1692783756881984/t0(0) o400->69788477-677f-a1e0-e4dc-bb97c16b0241@10.31.140.18@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.647308:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783756881984, offset 192 00000400:00000200:20.0:1616513080.647314:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.18@tcp1 00000400:00000200:20.0:1616513080.647321:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.18@tcp1 routed destination 00000100:00000200:0.0:1616513080.647328:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783756882176 00000400:00000200:20.0:1616513080.647336:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.18@tcp1(10.31.140.18@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513080.647339:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9ba80 x1692783756882176/t0(0) o400->7d657e5b-9e16-264a-fae8-2ab9bfe746b0@10.31.140.18@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.647342:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.647348:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.647351:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.647353:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513080.647353:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783756882176, offset 192 00000800:00000200:20.0:1616513080.647358:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513080.647358:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.18@tcp1 00000100:00000200:20.0:1616513080.647364:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.647364:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513080.647365:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.18@tcp1 routed destination 00000800:00000200:14.0:1616513080.647376:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513080.647379:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.18@tcp1(10.31.140.18@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513080.647382:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513080.647385:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513080.647385:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513080.647391:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513080.647391:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513080.647391:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.647393:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.647395:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513080.647396:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e36d8 00000800:00000200:14.0:1616513080.647400:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513080.647400:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.647405:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.647458:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.647467:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.647470:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.647471:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.647478:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.647482:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:24.0:1616513080.647486:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.650491:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.650546:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.650555:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.650564:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.136@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.650573:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.136@tcp1 of length 224 into portal 26 MB=0x5ffd473327c40 00000400:00000200:12.0:1616513080.650580:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.136@tcp1 of length 224/224 into md 0xf97c8e29 [1] + 0 00000400:00000200:12.0:1616513080.650585:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22ff0 00000400:00000200:12.0:1616513080.650588:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.650590:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.650595:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22ff0 00000800:00000200:12.0:1616513080.650598:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.650599:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.650600:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.650603:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.650658:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662814391360 00010000:00000200:20.0:1616513080.650669:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688662814391360/t0(0) o400->fbae7fb4-fdb4-7b0e-fdfc-36880c376678@10.31.136.136@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.650684:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662814391360, offset 192 00000400:00000200:20.0:1616513080.650689:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.136@tcp1 00000400:00000200:20.0:1616513080.650697:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.136@tcp1 routed destination 00000400:00000200:20.0:1616513080.650712:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.136@tcp1(10.31.136.136@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513080.650713:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.650718:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513080.650722:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.650724:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.650727:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513080.650728:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.650729:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.650734:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513080.650734:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.136@tcp1 : PUT - for me 00000800:00000200:7.2:1616513080.650738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.650740:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513080.650740:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.136@tcp1 of length 224 into portal 12 MB=0x5ffd473327d80 00000400:00000200:12.0:1616513080.650746:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.136@tcp1 of length 224/224 into md 0x865bbec1 [1] + 33936 00000800:00000200:30.0:1616513080.650749:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513080.650750:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.650752:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513080.650755:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513080.650757:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:30.0:1616513080.650758:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.650759:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.650761:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.650763:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513080.650764:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.650769:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3870 00000800:00000200:30.0:1616513080.650772:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.650775:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513080.650821:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662814391680 00010000:00000200:0.0:1616513080.650834:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c662d00 x1688662814391680/t0(0) o400->594e140a-1663-a4e2-7779-e498fe706fa3@10.31.136.136@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.650848:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662814391680, offset 192 00000400:00000200:0.0:1616513080.650854:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.136@tcp1 00000400:00000200:0.0:1616513080.650862:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.136@tcp1 routed destination 00000400:00000200:0.0:1616513080.650877:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.136@tcp1(10.31.136.136@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.650883:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.650889:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.650891:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.650894:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.650898:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.650903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.650914:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.650919:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.650921:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.650922:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.650928:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.650932:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb047f8 00000800:00000200:28.0:1616513080.650936:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.664534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.664590:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.664599:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.664610:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.27@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.664619:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.27@tcp1 of length 224 into portal 26 MB=0x5ffd31d29e1c0 00000400:00000200:12.0:1616513080.664625:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.27@tcp1 of length 224/224 into md 0xf97c8e49 [1] + 0 00000400:00000200:12.0:1616513080.664631:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22110 00000400:00000200:12.0:1616513080.664633:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.664636:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.664641:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22110 00000800:00000200:24.0:1616513080.664644:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.664644:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.664646:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.664648:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.664705:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657076019648 00010000:00000200:20.0:1616513080.664716:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688657076019648/t0(0) o400->8d3141b4-c16b-bcb3-6fd9-84f5da35e32d@10.31.129.27@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.664731:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657076019648, offset 192 00000400:00000200:20.0:1616513080.664737:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.27@tcp1 00000400:00000200:20.0:1616513080.664744:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.27@tcp1 routed destination 00000400:00000200:20.0:1616513080.664761:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.27@tcp1(10.31.129.27@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.664768:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.664774:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.664776:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.664779:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.664783:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.664790:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.664790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.664851:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.664858:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.664861:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.664867:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513080.664868:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.664873:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3870 00000800:00000200:18.0:1616513080.664877:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.664877:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513080.664879:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.664880:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.664882:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.664883:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.664918:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.664971:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.664981:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513080.664982:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.664990:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.27@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.664998:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.27@tcp1 of length 224 into portal 12 MB=0x5ffd31d29e600 00000400:00000200:22.0:1616513080.665005:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.27@tcp1 of length 224/224 into md 0xf964d519 [1] + 4256 00000400:00000200:22.0:1616513080.665010:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.665012:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513080.665020:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.665023:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.665025:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.665085:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657076020736 00010000:00000200:26.0:1616513080.665098:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec200480 x1688657076020736/t0(0) o400->76a12088-f8ef-ac88-41ec-0cd6006bee5e@10.31.129.27@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.665112:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657076020736, offset 192 00000400:00000200:26.0:1616513080.665119:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.27@tcp1 00000400:00000200:26.0:1616513080.665126:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.27@tcp1 routed destination 00000400:00000200:26.0:1616513080.665141:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.27@tcp1(10.31.129.27@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.665148:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.665153:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.665156:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.665158:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.665162:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.665167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.665177:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.665181:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.665183:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.665188:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.665192:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af980 00000800:00000200:12.0:1616513080.665195:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.665197:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.687305:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.687316:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.687323:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513080.687327:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:30.0:1616513080.687331:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.37@tcp1 : PUT - for me 00000800:00000200:14.0:1616513080.687336:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.687341:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.37@tcp1 of length 224 into portal 26 MB=0x6007983da0d00 00000400:00000200:14.0:1616513080.687344:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.37@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.687346:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.37@tcp1 of length 224/224 into md 0xf97c8e89 [1] + 0 00000400:00000200:30.0:1616513080.687351:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22000 00000400:00000200:14.0:1616513080.687352:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.37@tcp1 of length 224 into portal 12 MB=0x6007983da0e00 00000400:00000200:30.0:1616513080.687353:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.687356:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.687360:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22000 00000800:00000200:30.0:1616513080.687363:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513080.687364:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.37@tcp1 of length 224/224 into md 0xf964d519 [1] + 4480 00000800:00000200:30.0:1616513080.687365:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513080.687367:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.687370:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513080.687371:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513080.687373:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513080.687380:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.687383:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.687385:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.687429:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689371763412224 00010000:00000200:20.0:1616513080.687452:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1689371763412224/t0(0) o400->ef65aeee-30bc-8a04-fa5f-729f3a92f8e9@10.31.133.37@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.687467:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689371763412224, offset 192 00000400:00000200:20.0:1616513080.687473:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.37@tcp1 00000100:00000200:26.0:1616513080.687479:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689371763412480 00000400:00000200:20.0:1616513080.687480:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.37@tcp1 routed destination 00010000:00000200:26.0:1616513080.687491:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba2d00 x1689371763412480/t0(0) o400->898aee04-90ae-756d-382d-33a47e03154d@10.31.133.37@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513080.687496:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.37@tcp1(10.31.133.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.687503:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:26.0:1616513080.687506:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689371763412480, offset 192 00000800:00000200:20.0:1616513080.687508:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.687511:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:26.0:1616513080.687512:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.37@tcp1 00000800:00000200:20.0:1616513080.687513:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.687518:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513080.687519:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.37@tcp1 routed destination 00000800:00000200:7.2:1616513080.687523:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.687525:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513080.687534:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.37@tcp1(10.31.133.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.687541:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.687546:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.687551:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.687553:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.687558:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513080.687576:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.687584:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.687586:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.687588:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.687595:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.687601:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3870 00000800:00000200:11.2:1616513080.687601:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513080.687605:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513080.687657:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.687664:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.687666:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.687673:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.687677:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af320 00000800:00000200:24.0:1616513080.687681:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.687684:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.689778:0:49511:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513080.689833:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.689843:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.689862:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.170@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.689871:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.170@tcp1 of length 224 into portal 26 MB=0x60384ea5fb940 00000400:00000200:18.0:1616513080.689877:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.170@tcp1 of length 224/224 into md 0xf97c8e99 [1] + 0 00000400:00000200:18.0:1616513080.689882:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22aa0 00000400:00000200:18.0:1616513080.689884:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.689887:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513080.689893:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22aa0 00000800:00000200:18.0:1616513080.689897:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.689898:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.689899:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.689902:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.689956:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692719262972224 00010000:00000200:20.0:1616513080.689967:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1692719262972224/t0(0) o400->277989e3-e86f-d500-9530-e94c89ccc8b1@10.31.138.170@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.689981:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692719262972224, offset 192 00000400:00000200:20.0:1616513080.689987:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.170@tcp1 00000400:00000200:20.0:1616513080.689996:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.170@tcp1 routed destination 00000400:00000200:20.0:1616513080.690011:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.170@tcp1(10.31.138.170@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.690017:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.690023:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.690028:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.690030:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.690035:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.690040:0:49511:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.690042:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.690047:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.690051:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.690052:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.690057:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.690061:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6376d8 00000800:00000200:30.0:1616513080.690064:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.690066:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.690450:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.690510:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.690519:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.690528:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.170@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.690537:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.170@tcp1 of length 224 into portal 12 MB=0x60384ea5fbb00 00000400:00000200:12.0:1616513080.690543:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.170@tcp1 of length 224/224 into md 0x865bbec1 [1] + 34160 00000400:00000200:12.0:1616513080.690549:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.690551:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.690557:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.690560:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.690562:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.690565:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.690622:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692719262972672 00010000:00000200:0.0:1616513080.690634:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c660480 x1692719262972672/t0(0) o400->ce70e366-8bb3-9bc5-df25-5cf071d1014b@10.31.138.170@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.690660:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692719262972672, offset 192 00000400:00000200:0.0:1616513080.690666:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.170@tcp1 00000400:00000200:0.0:1616513080.690673:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.170@tcp1 routed destination 00000400:00000200:0.0:1616513080.690688:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.170@tcp1(10.31.138.170@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.690695:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.690700:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.690703:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.690705:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.690710:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.690715:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.690769:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.690776:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.690779:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.690780:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.690786:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.690790:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:24.0:1616513080.690793:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.735454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513080.735468:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.735477:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513080.735487:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.168@tcp1 : PUT - for me 00000400:00000200:14.0:1616513080.735496:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.168@tcp1 of length 224 into portal 26 MB=0x5ffd7c9767700 00000400:00000200:14.0:1616513080.735503:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.168@tcp1 of length 224/224 into md 0xf97c8eb9 [1] + 0 00000400:00000200:14.0:1616513080.735508:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23c28 00000400:00000200:14.0:1616513080.735511:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513080.735514:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513080.735518:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23c28 00000800:00000200:14.0:1616513080.735522:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.735523:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.735525:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.735527:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.735584:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688677146588928 00010000:00000200:20.0:1616513080.735596:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688677146588928/t0(0) o400->638bff82-d6e5-2363-ffd1-e3855117cce8@10.31.129.168@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.735611:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688677146588928, offset 192 00000400:00000200:20.0:1616513080.735617:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.168@tcp1 00000400:00000200:20.0:1616513080.735625:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.168@tcp1 routed destination 00000400:00000200:20.0:1616513080.735640:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.168@tcp1(10.31.129.168@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.735647:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.735653:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.735656:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.735658:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.735664:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.735668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.735671:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.735678:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.735684:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.735686:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.735692:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.735697:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636dd0 00000800:00000200:30.0:1616513080.735701:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.735703:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.735827:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.735884:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.735894:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.735903:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.168@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.735911:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.168@tcp1 of length 224 into portal 12 MB=0x5ffd7c97679c0 00000400:00000200:28.0:1616513080.735917:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.168@tcp1 of length 224/224 into md 0x865bbec1 [1] + 34384 00000400:00000200:28.0:1616513080.735923:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.735925:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513080.735933:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.735936:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.735938:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.735942:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.735998:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688677146589632 00010000:00000200:0.0:1616513080.736011:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b99f80 x1688677146589632/t0(0) o400->99d7f769-5b1d-f7ae-1caf-3d06c371fd19@10.31.129.168@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.736025:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688677146589632, offset 192 00000400:00000200:0.0:1616513080.736031:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.168@tcp1 00000400:00000200:0.0:1616513080.736039:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.168@tcp1 routed destination 00000400:00000200:0.0:1616513080.736054:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.168@tcp1(10.31.129.168@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.736061:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.736066:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.736069:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.736071:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.736077:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.736082:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.736093:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.736098:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.736100:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.736109:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.736113:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05430 00000800:00000200:12.0:1616513080.736117:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.736120:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.740785:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.740843:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.740853:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.740862:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.186@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.740870:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.186@tcp1 of length 224 into portal 26 MB=0x5ffdc43810800 00000400:00000200:12.0:1616513080.740877:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.186@tcp1 of length 224/224 into md 0xf97c8ed9 [1] + 0 00000400:00000200:12.0:1616513080.740882:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23760 00000400:00000200:12.0:1616513080.740884:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.740887:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.740892:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23760 00000800:00000200:12.0:1616513080.740895:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.740897:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.740899:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.740902:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.740913:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513080.740916:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.740922:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.186@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.740930:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.186@tcp1 of length 224 into portal 12 MB=0x5ffdc43810880 00000400:00000200:28.0:1616513080.740937:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.186@tcp1 of length 224/224 into md 0x865bbec1 [1] + 34608 00000400:00000200:28.0:1616513080.740942:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.740945:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513080.740952:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.740954:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696373970944 00000800:00000200:28.0:1616513080.740955:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.740957:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:20.0:1616513080.740964:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688696373970944/t0(0) o400->8924eb1f-1966-3be8-213c-d3d349ba768d@10.31.141.186@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.740978:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696373970944, offset 192 00000400:00000200:20.0:1616513080.740983:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.186@tcp1 00000400:00000200:20.0:1616513080.740990:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.186@tcp1 routed destination 00000400:00000200:20.0:1616513080.741004:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.186@tcp1(10.31.141.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.741010:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.741015:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513080.741017:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696373971072 00000800:00000200:20.0:1616513080.741018:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.741020:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.741025:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513080.741030:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9d580 x1688696373971072/t0(0) o400->5159402c-6fc3-cf91-eb10-2d6a09ca565d@10.31.141.186@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513080.741031:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.741031:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513080.741045:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696373971072, offset 192 00000400:00000200:0.0:1616513080.741051:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.186@tcp1 00000400:00000200:0.0:1616513080.741058:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.186@tcp1 routed destination 00000800:00000200:18.0:1616513080.741066:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.741073:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:0.0:1616513080.741073:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.186@tcp1(10.31.141.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513080.741076:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513080.741082:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513080.741083:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.741088:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6373a8 00000800:00000200:0.0:1616513080.741088:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.741091:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513080.741092:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513080.741093:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513080.741095:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513080.741098:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.741103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.741157:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.741164:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.741167:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.741167:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.741174:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.741178:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04b28 00000800:00000200:24.0:1616513080.741181:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.751200:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.751211:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.751220:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.751229:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.83@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.751239:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.83@tcp1 of length 224 into portal 26 MB=0x5ffd4306ee880 00000400:00000200:30.0:1616513080.751245:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.83@tcp1 of length 224/224 into md 0xf97c8f09 [1] + 0 00000400:00000200:30.0:1616513080.751250:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23078 00000400:00000200:30.0:1616513080.751253:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.751255:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.751259:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23078 00000800:00000200:30.0:1616513080.751262:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.751264:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.751265:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.751269:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.751323:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661694277760 00010000:00000200:20.0:1616513080.751334:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688661694277760/t0(0) o400->902740ae-1ac5-38df-bf1f-f82aea10e5c6@10.31.138.83@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.751348:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661694277760, offset 192 00000400:00000200:20.0:1616513080.751354:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.83@tcp1 00000400:00000200:20.0:1616513080.751363:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.83@tcp1 routed destination 00000400:00000200:20.0:1616513080.751378:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.83@tcp1(10.31.138.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.751384:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.751390:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.751393:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.751395:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.751400:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.751404:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.751407:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513080.751457:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.751464:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.751467:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.751470:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513080.751473:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.751479:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637430 00000800:00000200:18.0:1616513080.751482:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.754017:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.754070:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.754080:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.754080:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.754089:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.83@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.754098:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.83@tcp1 of length 224 into portal 12 MB=0x5ffd4306eea40 00000400:00000200:22.0:1616513080.754105:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.83@tcp1 of length 224/224 into md 0xf964d519 [1] + 4704 00000400:00000200:22.0:1616513080.754110:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.754112:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513080.754120:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.754123:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.754125:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.754182:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661694278208 00010000:00000200:26.0:1616513080.754194:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec200000 x1688661694278208/t0(0) o400->8752acce-a4de-5672-4341-59b06707a449@10.31.138.83@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.754207:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661694278208, offset 192 00000400:00000200:26.0:1616513080.754212:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.83@tcp1 00000400:00000200:26.0:1616513080.754219:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.83@tcp1 routed destination 00000400:00000200:26.0:1616513080.754233:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.83@tcp1(10.31.138.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.754239:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.754244:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.754247:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.754249:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.754253:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.754300:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.754354:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.754361:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.754363:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.754369:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.754373:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae7f8 00000800:00000200:12.0:1616513080.754377:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.754379:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.762038:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513080.762091:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.762100:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513080.762103:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513080.762108:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.114@tcp1 : PUT - for me 00000400:00000200:14.0:1616513080.762116:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.114@tcp1 of length 224 into portal 12 MB=0x5ffffbddad540 00000400:00000200:14.0:1616513080.762122:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.114@tcp1 of length 224/224 into md 0xf964d519 [1] + 4928 00000400:00000200:14.0:1616513080.762127:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513080.762129:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513080.762136:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.762138:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.762140:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.762198:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688848750531904 00010000:00000200:26.0:1616513080.762211:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba5580 x1688848750531904/t0(0) o400->eaf41792-597b-db54-189f-1344556c3b0f@10.31.138.114@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.762225:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688848750531904, offset 192 00000400:00000200:26.0:1616513080.762231:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.114@tcp1 00000400:00000200:26.0:1616513080.762238:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.114@tcp1 routed destination 00000400:00000200:26.0:1616513080.762253:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.114@tcp1(10.31.138.114@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513080.762253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.762260:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513080.762266:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.762268:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513080.762271:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513080.762275:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.762279:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.762288:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.762293:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.762295:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.762301:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.762304:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af8f8 00000800:00000200:12.0:1616513080.762307:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513080.762308:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.762310:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513080.762315:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.762324:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.114@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.762332:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.114@tcp1 of length 224 into portal 26 MB=0x5ffffbddad480 00000400:00000200:12.0:1616513080.762338:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.114@tcp1 of length 224/224 into md 0xf97c8f29 [1] + 0 00000400:00000200:12.0:1616513080.762343:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22220 00000400:00000200:12.0:1616513080.762345:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.762348:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.762354:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22220 00000800:00000200:12.0:1616513080.762357:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.762359:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.762361:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.762364:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.762416:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688848750531712 00010000:00000200:20.0:1616513080.762426:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688848750531712/t0(0) o400->692b686e-ce14-5d77-9f9b-d949b9be7ca0@10.31.138.114@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.762451:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688848750531712, offset 192 00000400:00000200:20.0:1616513080.762456:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.114@tcp1 00000400:00000200:20.0:1616513080.762463:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.114@tcp1 routed destination 00000400:00000200:20.0:1616513080.762477:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.114@tcp1(10.31.138.114@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.762483:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513080.762489:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.762491:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.762493:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.762498:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513080.762504:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513080.762504:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.762513:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.762517:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.762519:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.762524:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.762529:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6362a8 00000800:00000200:12.0:1616513080.762531:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.762534:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.774784:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.774841:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.774851:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.774860:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.168@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.774868:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.168@tcp1 of length 224 into portal 26 MB=0x6052799307b80 00000400:00000200:24.0:1616513080.774875:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.168@tcp1 of length 224/224 into md 0xf97c8f39 [1] + 0 00000400:00000200:24.0:1616513080.774880:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee22f68 00000400:00000200:24.0:1616513080.774883:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.774885:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.774890:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee22f68 00000800:00000200:24.0:1616513080.774893:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.774894:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.774897:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.774902:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.774905:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.774906:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.774914:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.168@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.774923:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.168@tcp1 of length 224 into portal 12 MB=0x6052799307c80 00000400:00000200:12.0:1616513080.774929:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.168@tcp1 of length 224/224 into md 0x865bbec1 [1] + 34832 00000400:00000200:12.0:1616513080.774934:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.774937:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.774944:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.774947:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.774949:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.774955:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694517492218752 00010000:00000200:20.0:1616513080.774965:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1694517492218752/t0(0) o400->b258a059-70e1-853d-9a1c-2449c42ae11c@10.31.140.168@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.774980:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694517492218752, offset 192 00000400:00000200:20.0:1616513080.774988:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.168@tcp1 00000400:00000200:20.0:1616513080.774995:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.168@tcp1 routed destination 00000400:00000200:20.0:1616513080.775010:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.168@tcp1(10.31.140.168@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513080.775016:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694517492219008 00000800:00000200:20.0:1616513080.775017:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.775023:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.775025:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.775028:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513080.775029:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661b00 x1694517492219008/t0(0) o400->44371bb1-e23f-bee5-1c92-7fe0eb7530bd@10.31.140.168@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.775032:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.775038:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.775039:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513080.775043:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694517492219008, offset 192 00000400:00000200:0.0:1616513080.775050:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.168@tcp1 00000400:00000200:0.0:1616513080.775058:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.168@tcp1 routed destination 00000400:00000200:0.0:1616513080.775072:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.168@tcp1(10.31.140.168@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.775079:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.775084:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.775087:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.775089:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513080.775091:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513080.775094:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513080.775098:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513080.775099:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513080.775100:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.775103:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513080.775106:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.775112:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637ed0 00000800:00000200:28.0:1616513080.775115:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513080.775115:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513080.775122:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.775125:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.775126:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.775132:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.775136:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04088 00000800:00000200:28.0:1616513080.775139:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.776329:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.776383:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.776393:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.776402:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.139@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.776410:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.139@tcp1 of length 224 into portal 26 MB=0x603fcda429ac0 00000400:00000200:12.0:1616513080.776417:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.139@tcp1 of length 224/224 into md 0xf97c8f49 [1] + 0 00000400:00000200:12.0:1616513080.776422:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee235c8 00000400:00000200:12.0:1616513080.776425:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.776428:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.776432:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee235c8 00000800:00000200:7.2:1616513080.776436:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.776448:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513080.776448:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.776451:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:30.0:1616513080.776455:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:12.0:1616513080.776455:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513080.776457:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513080.776463:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.139@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.776472:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.139@tcp1 of length 224 into portal 12 MB=0x603fcda429bc0 00000400:00000200:30.0:1616513080.776478:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.139@tcp1 of length 224/224 into md 0xf964d519 [1] + 5152 00000400:00000200:30.0:1616513080.776483:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.776485:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513080.776492:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.776494:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.776496:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.776496:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234388703936 00000800:00000200:30.0:1616513080.776498:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:20.0:1616513080.776506:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1693234388703936/t0(0) o400->29adbec7-ac71-bf21-3986-f3a81177222d@10.31.138.139@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.776519:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234388703936, offset 192 00000400:00000200:20.0:1616513080.776525:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.139@tcp1 00000400:00000200:20.0:1616513080.776532:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.139@tcp1 routed destination 00000400:00000200:20.0:1616513080.776545:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.139@tcp1(10.31.138.139@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.776551:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:26.0:1616513080.776556:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234388704192 00000800:00000200:20.0:1616513080.776557:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.776559:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.776561:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.776566:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:26.0:1616513080.776568:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf80900 x1693234388704192/t0(0) o400->5f69b0f8-f20a-4392-b907-b75fa57ccc92@10.31.138.139@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513080.776570:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.776572:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513080.776594:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234388704192, offset 192 00000400:00000200:26.0:1616513080.776600:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.139@tcp1 00000400:00000200:26.0:1616513080.776607:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.139@tcp1 routed destination 00000400:00000200:26.0:1616513080.776622:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.139@tcp1(10.31.138.139@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.776629:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.776634:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.776637:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.776639:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513080.776641:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513080.776644:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513080.776647:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513080.776650:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513080.776650:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513080.776652:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513080.776657:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513080.776662:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513080.776662:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637298 00000800:00000200:14.0:1616513080.776666:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513080.776669:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.776671:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.776678:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.776682:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeee0 00000800:00000200:24.0:1616513080.776686:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.776689:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.787710:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.787767:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.787776:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.787784:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.79@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.787794:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.79@tcp1 of length 224 into portal 26 MB=0x5ffd438c27a40 00000400:00000200:12.0:1616513080.787801:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.79@tcp1 of length 224/224 into md 0xf97c8f59 [1] + 0 00000400:00000200:12.0:1616513080.787806:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825ee23430 00000400:00000200:12.0:1616513080.787809:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.787812:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.787816:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825ee23430 00000800:00000200:16.0:1616513080.787819:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.787819:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.787822:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.787824:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.787877:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661833972288 00010000:00000200:20.0:1616513080.787888:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688661833972288/t0(0) o400->8944c226-9aee-a68a-3457-d276d5f05a46@10.31.133.79@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.787901:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661833972288, offset 192 00000400:00000200:20.0:1616513080.787907:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.79@tcp1 00000400:00000200:20.0:1616513080.787914:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.79@tcp1 routed destination 00000400:00000200:20.0:1616513080.787928:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.79@tcp1(10.31.133.79@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.787938:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.787943:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.787946:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.787948:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.787952:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.787955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.787958:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.787965:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.787971:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.787979:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.79@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.787986:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.79@tcp1 of length 224 into portal 12 MB=0x5ffd438c27ac0 00000400:00000200:30.0:1616513080.787991:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.79@tcp1 of length 224/224 into md 0xf964d519 [1] + 5376 00000400:00000200:30.0:1616513080.787995:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.787997:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513080.788003:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513080.788005:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513080.788006:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.788008:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.788012:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.788014:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513080.788019:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.788023:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636b28 00000800:00000200:22.0:1616513080.788023:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.788027:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.788081:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661833972416 00010000:00000200:26.0:1616513080.788092:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf86c00 x1688661833972416/t0(0) o400->458b7e92-254b-d28b-31c9-b152e03e1041@10.31.133.79@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.788105:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661833972416, offset 192 00000400:00000200:26.0:1616513080.788111:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.79@tcp1 00000400:00000200:26.0:1616513080.788118:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.79@tcp1 routed destination 00000400:00000200:26.0:1616513080.788132:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.79@tcp1(10.31.133.79@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.788138:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.788143:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.788146:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.788148:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.788152:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.788158:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.788170:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.788177:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.788179:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.788186:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.788190:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afd38 00000800:00000200:24.0:1616513080.788194:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.788197:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.796761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513080.796819:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.796828:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513080.796837:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.148@tcp1 : PUT - for me 00000400:00000200:16.0:1616513080.796846:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.148@tcp1 of length 224 into portal 26 MB=0x603e1677bfec0 00000400:00000200:16.0:1616513080.796852:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.148@tcp1 of length 224/224 into md 0xf97c8f69 [1] + 0 00000400:00000200:16.0:1616513080.796858:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2dd0 00000400:00000200:16.0:1616513080.796860:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513080.796863:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513080.796867:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2dd0 00000800:00000200:16.0:1616513080.796871:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.796873:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.796873:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.796879:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513080.796882:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.796883:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.796892:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.148@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.796900:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.148@tcp1 of length 224 into portal 12 MB=0x603e1677bff80 00000400:00000200:12.0:1616513080.796907:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.148@tcp1 of length 224/224 into md 0x865bbec1 [1] + 35056 00000400:00000200:12.0:1616513080.796912:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.796914:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.796921:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.796924:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.796926:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.796932:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116498968256 00010000:00000200:20.0:1616513080.796943:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1693116498968256/t0(0) o400->b966db97-5d72-ecda-ca7b-440a37138991@10.31.132.148@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.796957:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693116498968256, offset 192 00000400:00000200:20.0:1616513080.796963:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.148@tcp1 00000400:00000200:20.0:1616513080.796971:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.148@tcp1 routed destination 00000100:00000200:0.0:1616513080.796985:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116498968448 00000400:00000200:20.0:1616513080.796986:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.148@tcp1(10.31.132.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.796992:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.796998:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513080.796998:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661680 x1693116498968448/t0(0) o400->a5d8c0c6-c61f-d5f9-3680-43b07427c5f1@10.31.132.148@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.797001:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.797003:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.797008:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513080.797012:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693116498968448, offset 192 00000100:00000200:20.0:1616513080.797014:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.797014:0:49524:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513080.797019:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.148@tcp1 00000400:00000200:0.0:1616513080.797026:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.148@tcp1 routed destination 00000400:00000200:0.0:1616513080.797041:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.148@tcp1(10.31.132.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.797048:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.797053:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.797056:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.797058:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.797063:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.797068:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513080.797069:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.797076:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513080.797079:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513080.797081:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513080.797085:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513080.797088:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.797091:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513080.797091:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636d48 00000800:00000200:16.0:1616513080.797092:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513080.797094:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513080.797097:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513080.797097:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513080.797101:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:28.0:1616513080.797105:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.799478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513080.799488:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513080.799495:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.799502:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.6@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.799510:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.6@tcp1 of length 224 into portal 26 MB=0x5ffdaa4616c00 00000400:00000200:30.0:1616513080.799516:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.6@tcp1 of length 224/224 into md 0xf97c8f79 [1] + 0 00000400:00000200:30.0:1616513080.799520:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3320 00000400:00000200:30.0:1616513080.799522:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.799524:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513080.799527:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3320 00000800:00000200:30.0:1616513080.799530:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.799532:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.799533:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513080.799535:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.799587:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688689409354752 00010000:00000200:20.0:1616513080.799596:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688689409354752/t0(0) o400->f5a94ed0-1217-0f4c-6fc7-bf4a5daca46e@10.31.140.6@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.799608:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688689409354752, offset 192 00000400:00000200:20.0:1616513080.799614:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.6@tcp1 00000400:00000200:20.0:1616513080.799620:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.6@tcp1 routed destination 00000400:00000200:20.0:1616513080.799633:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.6@tcp1(10.31.140.6@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.799639:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.799644:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.799646:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.799650:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.799654:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.799658:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.799660:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513080.799670:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513080.799709:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.799716:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513080.799718:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513080.799724:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513080.799729:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637100 00000800:00000200:12.0:1616513080.799730:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513080.799733:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.799735:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513080.799739:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.799747:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.6@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.799755:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.6@tcp1 of length 224 into portal 12 MB=0x5ffdaa4616c80 00000400:00000200:12.0:1616513080.799761:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.6@tcp1 of length 224/224 into md 0x865bbec1 [1] + 35280 00000400:00000200:12.0:1616513080.799766:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.799768:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513080.799774:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.799776:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.799780:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.799783:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.799839:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688689409354880 00010000:00000200:0.0:1616513080.799851:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c667980 x1688689409354880/t0(0) o400->44ae6c5e-c424-e077-8583-8c38dc927b1a@10.31.140.6@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.799865:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688689409354880, offset 192 00000400:00000200:0.0:1616513080.799871:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.6@tcp1 00000400:00000200:0.0:1616513080.799879:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.6@tcp1 routed destination 00000400:00000200:0.0:1616513080.799893:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.6@tcp1(10.31.140.6@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.799900:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.799906:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.799908:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.799911:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.799915:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.799921:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.799975:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.799982:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513080.799985:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513080.799988:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513080.799992:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513080.799996:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05c28 00000800:00000200:16.0:1616513080.800000:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.802209:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.802266:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.802275:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.802284:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.89@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.802293:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.89@tcp1 of length 224 into portal 26 MB=0x601833c693000 00000400:00000200:12.0:1616513080.802299:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.89@tcp1 of length 224/224 into md 0xf97c8fb9 [1] + 0 00000400:00000200:12.0:1616513080.802304:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3f58 00000400:00000200:12.0:1616513080.802307:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.802310:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.802314:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3f58 00000800:00000200:12.0:1616513080.802317:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.802319:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.802320:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.802323:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.802393:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513026134016 00010000:00000200:20.0:1616513080.802403:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1690513026134016/t0(0) o400->ba331d69-f884-9208-1427-06db0c1297ba@10.31.138.89@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.802416:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513026134016, offset 192 00000400:00000200:20.0:1616513080.802422:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.89@tcp1 00000400:00000200:20.0:1616513080.802429:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.89@tcp1 routed destination 00000400:00000200:20.0:1616513080.802454:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.89@tcp1(10.31.138.89@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.802460:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.802466:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.802468:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.802470:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:7.2:1616513080.802471:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.802475:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.802478:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:20.0:1616513080.802482:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513080.802483:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513080.802489:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.89@tcp1 : PUT - for me 00000400:00000200:30.0:1616513080.802495:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.89@tcp1 of length 224 into portal 12 MB=0x601833c693100 00000400:00000200:30.0:1616513080.802500:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.89@tcp1 of length 224/224 into md 0xf964d519 [1] + 5600 00000400:00000200:30.0:1616513080.802504:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513080.802506:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513080.802510:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513080.802512:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:30.0:1616513080.802513:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513080.802515:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513080.802519:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513080.802520:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.802524:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513080.802525:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513080.802528:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6377e8 00000800:00000200:30.0:1616513080.802530:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.802574:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513026134272 00010000:00000200:26.0:1616513080.802586:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf84c80 x1690513026134272/t0(0) o400->39a6ce64-18b6-0dd7-40c6-a6b11d6a93fb@10.31.138.89@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.802599:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513026134272, offset 192 00000400:00000200:26.0:1616513080.802605:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.89@tcp1 00000400:00000200:26.0:1616513080.802611:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.89@tcp1 routed destination 00000400:00000200:26.0:1616513080.802626:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.89@tcp1(10.31.138.89@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.802633:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.802638:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.802641:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.802643:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.802647:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.802653:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.802665:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.802672:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.802674:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513080.802681:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.802685:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af760 00000800:00000200:28.0:1616513080.802689:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.802692:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.803630:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.803685:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.803694:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.803703:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.129@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.803712:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.129@tcp1 of length 224 into portal 26 MB=0x602eab91b8000 00000400:00000200:24.0:1616513080.803718:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.129@tcp1 of length 224/224 into md 0xf97c8fc9 [1] + 0 00000400:00000200:24.0:1616513080.803723:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3c28 00000400:00000200:24.0:1616513080.803726:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.803729:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.803733:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3c28 00000800:00000200:24.0:1616513080.803736:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.803739:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.803740:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.803743:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.803748:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513080.803748:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.803755:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.129@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.803760:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.129@tcp1 of length 224 into portal 12 MB=0x602eab91b80c0 00000400:00000200:24.0:1616513080.803767:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.129@tcp1 of length 224/224 into md 0x865bbec1 [1] + 35504 00000400:00000200:24.0:1616513080.803771:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.803773:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.803778:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.803780:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.803782:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.803795:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692057011453952 00010000:00000200:20.0:1616513080.803806:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1692057011453952/t0(0) o400->02e3fadb-347f-a3d5-5ad5-eaeeaee50270@10.31.129.129@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.803819:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692057011453952, offset 192 00000400:00000200:20.0:1616513080.803825:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.129@tcp1 00000400:00000200:20.0:1616513080.803832:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.129@tcp1 routed destination 00000100:00000200:0.0:1616513080.803843:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692057011454144 00000400:00000200:20.0:1616513080.803846:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.129@tcp1(10.31.129.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.803852:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:0.0:1616513080.803855:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3a80 x1692057011454144/t0(0) o400->f080fcb1-14da-874a-204c-00b0b07ba7d3@10.31.129.129@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.803857:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.803860:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.803862:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.803866:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.803870:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513080.803870:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692057011454144, offset 192 00000100:00000200:20.0:1616513080.803873:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513080.803876:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.129@tcp1 00000400:00000200:0.0:1616513080.803883:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.129@tcp1 routed destination 00000400:00000200:0.0:1616513080.803898:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.129@tcp1(10.31.129.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.803905:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.803910:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.803913:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.803915:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.803920:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513080.803923:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513080.803925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513080.803930:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.803933:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.803939:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.803945:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636440 00000800:00000200:22.0:1616513080.803948:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.803951:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513080.803979:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.803985:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513080.803988:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.803988:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513080.803994:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513080.803998:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05f58 00000800:00000200:16.0:1616513080.804001:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.807123:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513080.807204:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.807214:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.807223:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.16@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.807233:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.16@tcp1 of length 224 into portal 26 MB=0x5ffd46d047600 00000400:00000200:18.0:1616513080.807240:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.16@tcp1 of length 224/224 into md 0xf97c8fe9 [1] + 0 00000400:00000200:18.0:1616513080.807244:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3cb0 00000400:00000200:18.0:1616513080.807247:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.807250:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513080.807255:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3cb0 00000800:00000200:2.0:1616513080.807258:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.807259:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.807261:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.807263:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.807319:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662710711808 00010000:00000200:20.0:1616513080.807330:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688662710711808/t0(0) o400->6886e9d9-1887-689f-aa1a-7089e9581977@10.31.133.16@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.807346:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662710711808, offset 192 00000400:00000200:20.0:1616513080.807352:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.16@tcp1 00000400:00000200:20.0:1616513080.807359:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.16@tcp1 routed destination 00000400:00000200:20.0:1616513080.807375:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.16@tcp1(10.31.133.16@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.807381:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.807386:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.807389:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.807391:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.807396:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.807400:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.807402:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513080.807453:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.807460:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.807463:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.807466:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.807469:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.807475:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637c28 00000800:00000200:22.0:1616513080.807479:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.807868:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513080.807878:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.807886:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513080.807888:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.807895:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.16@tcp1 : PUT - for me 00000400:00000200:2.0:1616513080.807904:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.16@tcp1 of length 224 into portal 12 MB=0x5ffd46d047780 00000400:00000200:2.0:1616513080.807910:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.16@tcp1 of length 224/224 into md 0xf964d519 [1] + 5824 00000400:00000200:2.0:1616513080.807915:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513080.807918:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513080.807925:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.807928:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.807930:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.807987:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662710712192 00010000:00000200:26.0:1616513080.808000:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21e300 x1688662710712192/t0(0) o400->875d0aa3-f8a4-12c1-2f70-2312e81eb192@10.31.133.16@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.808014:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662710712192, offset 192 00000400:00000200:26.0:1616513080.808021:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.16@tcp1 00000400:00000200:26.0:1616513080.808028:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.16@tcp1 routed destination 00000400:00000200:26.0:1616513080.808043:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.16@tcp1(10.31.133.16@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.808049:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.808055:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.808058:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.808060:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.808064:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.808110:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.808125:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.808131:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.808134:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513080.808141:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.808145:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae000 00000800:00000200:28.0:1616513080.808149:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.808152:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.809156:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.809209:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.809218:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.809226:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.220@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.809236:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.220@tcp1 of length 224 into portal 26 MB=0x60068891c2b00 00000400:00000200:12.0:1616513080.809242:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.220@tcp1 of length 224/224 into md 0xf97c8ff9 [1] + 0 00000400:00000200:12.0:1616513080.809247:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3d38 00000400:00000200:12.0:1616513080.809250:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.809253:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.809257:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3d38 00000800:00000200:12.0:1616513080.809260:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.809263:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.809265:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.809267:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.809319:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689298837187328 00010000:00000200:20.0:1616513080.809330:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1689298837187328/t0(0) o400->8b4923e3-759b-258c-ec3a-573eaec769a8@10.31.141.220@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.809345:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689298837187328, offset 192 00000400:00000200:20.0:1616513080.809351:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.220@tcp1 00000400:00000200:20.0:1616513080.809358:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.220@tcp1 routed destination 00000400:00000200:20.0:1616513080.809373:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.220@tcp1(10.31.141.220@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.809380:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513080.809381:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513080.809385:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.809388:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.809390:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513080.809393:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513080.809395:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.809399:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.809401:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513080.809402:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.809411:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.220@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.809419:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.220@tcp1 of length 224 into portal 12 MB=0x60068891c3000 00000400:00000200:28.0:1616513080.809425:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.220@tcp1 of length 224/224 into md 0x865bbec1 [1] + 35728 00000400:00000200:28.0:1616513080.809430:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.809442:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.809448:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.809452:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.809455:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.809457:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513080.809466:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.809472:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513080.809475:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513080.809482:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513080.809487:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636330 00000800:00000200:14.0:1616513080.809491:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.809494:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513080.809527:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689298837188608 00010000:00000200:0.0:1616513080.809540:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9c800 x1689298837188608/t0(0) o400->c94ce14c-34e2-7f19-4dad-dfef9b35644d@10.31.141.220@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.809554:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689298837188608, offset 192 00000400:00000200:0.0:1616513080.809560:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.220@tcp1 00000400:00000200:0.0:1616513080.809567:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.220@tcp1 routed destination 00000400:00000200:0.0:1616513080.809584:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.220@tcp1(10.31.141.220@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.809590:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.809596:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.809599:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.809601:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.809606:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.809611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.809620:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.809624:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.809626:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513080.809629:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.809631:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.809636:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05078 00000800:00000200:12.0:1616513080.809639:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.818987:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.819044:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.819053:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.819062:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.86@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.819071:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.86@tcp1 of length 224 into portal 26 MB=0x5ffd6ad253380 00000400:00000200:24.0:1616513080.819077:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.86@tcp1 of length 224/224 into md 0xf97c9029 [1] + 0 00000400:00000200:24.0:1616513080.819082:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2990 00000400:00000200:24.0:1616513080.819085:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.819087:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.819092:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2990 00000800:00000200:24.0:1616513080.819095:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.819097:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.819097:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.819101:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.819157:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672376533888 00010000:00000200:20.0:1616513080.819167:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688672376533888/t0(0) o400->bddc7c5b-83dd-0052-5ce6-5edf05d54603@10.31.136.86@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.819180:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688672376533888, offset 192 00000400:00000200:20.0:1616513080.819186:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.86@tcp1 00000400:00000200:20.0:1616513080.819193:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.86@tcp1 routed destination 00000400:00000200:20.0:1616513080.819208:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.86@tcp1(10.31.136.86@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.819214:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.819219:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.819222:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.819224:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.819228:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.819234:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.819234:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.819287:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.819294:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.819297:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.819303:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.819309:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636bb0 00000800:00000200:18.0:1616513080.819312:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.819315:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.819384:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513080.819395:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.819403:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513080.819411:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.86@tcp1 : PUT - for me 00000400:00000200:14.0:1616513080.819419:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.86@tcp1 of length 224 into portal 12 MB=0x5ffd6ad2534c0 00000400:00000200:14.0:1616513080.819425:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.86@tcp1 of length 224/224 into md 0xf964d519 [1] + 6048 00000400:00000200:14.0:1616513080.819429:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513080.819432:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513080.819450:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.819450:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.819455:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.819457:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.819523:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672376534208 00010000:00000200:26.0:1616513080.819536:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba0d80 x1688672376534208/t0(0) o400->352f1307-2145-1837-c857-1f670afb230f@10.31.136.86@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.819550:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688672376534208, offset 192 00000400:00000200:26.0:1616513080.819558:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.86@tcp1 00000400:00000200:26.0:1616513080.819565:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.86@tcp1 routed destination 00000400:00000200:26.0:1616513080.819580:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.86@tcp1(10.31.136.86@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.819586:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.819592:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.819595:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.819597:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.819602:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.819607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.819659:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.819665:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513080.819668:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513080.819674:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513080.819678:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae5d8 00000800:00000200:16.0:1616513080.819681:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513080.819684:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.827092:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.827147:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.827155:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.827163:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.27@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.827172:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.27@tcp1 of length 224 into portal 26 MB=0x5ffe0364e2b00 00000400:00000200:28.0:1616513080.827178:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.27@tcp1 of length 224/224 into md 0xf97c9069 [1] + 0 00000400:00000200:28.0:1616513080.827182:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3298 00000400:00000200:28.0:1616513080.827185:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.827187:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513080.827191:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3298 00000800:00000200:28.0:1616513080.827194:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.827197:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.827197:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513080.827204:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513080.827207:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.827207:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.827215:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.27@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.827223:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.27@tcp1 of length 224 into portal 12 MB=0x5ffe0364e3080 00000400:00000200:24.0:1616513080.827229:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.27@tcp1 of length 224/224 into md 0x865bbec1 [1] + 35952 00000400:00000200:24.0:1616513080.827234:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.827236:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.827242:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.827245:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.827247:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.827253:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688713332402944 00010000:00000200:20.0:1616513080.827263:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688713332402944/t0(0) o400->66c8a8d1-a1d8-cb12-c696-0ddc15d646db@10.31.141.27@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.827276:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688713332402944, offset 192 00000400:00000200:20.0:1616513080.827282:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.27@tcp1 00000400:00000200:20.0:1616513080.827288:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.27@tcp1 routed destination 00000400:00000200:20.0:1616513080.827302:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.27@tcp1(10.31.141.27@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513080.827305:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688713332404352 00000800:00000200:20.0:1616513080.827308:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.827314:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.827316:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:0.0:1616513080.827317:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d6300 x1688713332404352/t0(0) o400->d220cf23-5159-dbf1-fa9b-af724def5dde@10.31.141.27@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.827318:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.827322:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.827328:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.827329:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513080.827330:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688713332404352, offset 192 00000400:00000200:0.0:1616513080.827336:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.27@tcp1 00000400:00000200:0.0:1616513080.827343:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.27@tcp1 routed destination 00000400:00000200:0.0:1616513080.827357:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.27@tcp1(10.31.141.27@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.827363:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.827369:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.827371:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.827373:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.827377:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513080.827380:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513080.827383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513080.827386:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.827389:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513080.827394:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513080.827395:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.827400:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637540 00000800:00000200:12.0:1616513080.827401:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513080.827403:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513080.827403:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513080.827404:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513080.827406:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513080.827409:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.827413:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:12.0:1616513080.827417:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.831250:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.831307:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.831316:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.831325:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.112@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.831335:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.112@tcp1 of length 224 into portal 26 MB=0x5ffe843aef9c0 00000400:00000200:24.0:1616513080.831342:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.112@tcp1 of length 224/224 into md 0xf97c9079 [1] + 0 00000400:00000200:24.0:1616513080.831347:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3a08 00000400:00000200:24.0:1616513080.831349:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.831352:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.831356:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3a08 00000800:00000200:24.0:1616513080.831359:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.831359:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.831362:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.831366:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.831369:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513080.831370:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513080.831378:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.112@tcp1 : PUT - for me 00000400:00000200:16.0:1616513080.831387:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.112@tcp1 of length 224 into portal 12 MB=0x5ffe843aefa40 00000400:00000200:16.0:1616513080.831393:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.112@tcp1 of length 224/224 into md 0x865bbec1 [1] + 36176 00000400:00000200:16.0:1616513080.831398:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513080.831401:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513080.831407:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.831410:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513080.831412:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.831421:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688747916589504 00010000:00000200:20.0:1616513080.831432:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688747916589504/t0(0) o400->f45599d0-d656-4bf1-877f-c47c6ce62b61@10.31.141.112@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.831457:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688747916589504, offset 192 00000400:00000200:20.0:1616513080.831463:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.112@tcp1 00000400:00000200:20.0:1616513080.831471:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.112@tcp1 routed destination 00000100:00000200:0.0:1616513080.831472:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688747916589632 00010000:00000200:0.0:1616513080.831485:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd6780 x1688747916589632/t0(0) o400->a28e251f-28c3-63db-4091-86a58e464d6b@10.31.141.112@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513080.831486:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.112@tcp1(10.31.141.112@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.831492:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.831498:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513080.831499:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688747916589632, offset 192 00000800:00000200:20.0:1616513080.831502:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.831504:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513080.831505:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.112@tcp1 00000800:00000200:20.0:1616513080.831509:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513080.831513:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.112@tcp1 routed destination 00000800:00000200:7.2:1616513080.831514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.831516:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513080.831528:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.112@tcp1(10.31.141.112@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.831535:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.831541:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.831543:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.831547:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.831552:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.831557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513080.831568:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513080.831569:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513080.831575:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513080.831576:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513080.831578:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.831578:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.831580:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513080.831581:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.831585:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.831585:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.831589:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000800:00000200:12.0:1616513080.831593:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513080.831594:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637870 00000800:00000200:22.0:1616513080.831597:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.850377:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513080.850432:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.850459:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513080.850469:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.189@tcp1 : PUT - for me 00000400:00000200:16.0:1616513080.850479:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.189@tcp1 of length 224 into portal 26 MB=0x5ffd3d5dd7a00 00000400:00000200:16.0:1616513080.850486:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.189@tcp1 of length 224/224 into md 0xf97c90a9 [1] + 0 00000800:00000200:28.0:1616513080.850488:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513080.850490:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3980 00000400:00000200:16.0:1616513080.850494:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513080.850496:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513080.850501:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3980 00000800:00000200:16.0:1616513080.850504:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.850507:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513080.850509:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.850563:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660174797312 00010000:00000200:20.0:1616513080.850573:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688660174797312/t0(0) o400->fca86026-5dfd-39cc-678a-a737e896f46a@10.31.128.189@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.850586:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660174797312, offset 192 00000400:00000200:20.0:1616513080.850592:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.189@tcp1 00000400:00000200:20.0:1616513080.850600:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.189@tcp1 routed destination 00000400:00000200:20.0:1616513080.850614:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.189@tcp1(10.31.128.189@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.850620:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.850626:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.850628:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.850630:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.850635:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.850641:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.850641:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.850693:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.850700:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.850702:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.850708:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.850714:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637a90 00000800:00000200:18.0:1616513080.850717:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.850719:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.851200:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.851254:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.851263:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.851272:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.189@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.851281:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.189@tcp1 of length 224 into portal 12 MB=0x5ffd3d5dd7b80 00000400:00000200:24.0:1616513080.851287:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.189@tcp1 of length 224/224 into md 0x865bbec1 [1] + 36400 00000400:00000200:24.0:1616513080.851292:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.851295:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.851301:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.851305:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.851306:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.851309:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.851366:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660174797696 00010000:00000200:0.0:1616513080.851378:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d2880 x1688660174797696/t0(0) o400->35b0c0bc-4ef4-382b-58cc-284212a9e47f@10.31.128.189@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.851392:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660174797696, offset 192 00000400:00000200:0.0:1616513080.851399:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.189@tcp1 00000400:00000200:0.0:1616513080.851406:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.189@tcp1 routed destination 00000400:00000200:0.0:1616513080.851421:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.189@tcp1(10.31.128.189@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.851427:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.851433:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.851436:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.851454:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.851459:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.851464:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.851475:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.851480:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.851483:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513080.851485:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.851489:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.851493:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05e48 00000800:00000200:28.0:1616513080.851497:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.852278:0:49531:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513080.852333:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.852342:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513080.852352:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.13@tcp1 : PUT - for me 00000400:00000200:14.0:1616513080.852361:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.13@tcp1 of length 224 into portal 26 MB=0x60008cf299bc0 00000400:00000200:14.0:1616513080.852367:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.13@tcp1 of length 224/224 into md 0xf97c90b9 [1] + 0 00000400:00000200:14.0:1616513080.852372:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2440 00000400:00000200:14.0:1616513080.852375:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513080.852378:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513080.852382:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2440 00000800:00000200:14.0:1616513080.852384:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.852387:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.852387:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.852391:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.852462:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688887695612864 00010000:00000200:20.0:1616513080.852473:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688887695612864/t0(0) o400->a28f1447-1415-9a3a-dd9a-87bc1893da30@10.31.136.13@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.852488:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688887695612864, offset 192 00000400:00000200:20.0:1616513080.852494:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.13@tcp1 00000400:00000200:20.0:1616513080.852502:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.13@tcp1 routed destination 00000400:00000200:20.0:1616513080.852517:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.13@tcp1(10.31.136.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.852523:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.852529:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.852532:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.852534:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.852539:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.852543:0:49531:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.852546:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513080.852595:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.852602:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.852605:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.852611:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.852617:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636c38 00000800:00000200:18.0:1616513080.852620:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.852623:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.852873:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.852930:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.852939:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.852949:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.13@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.852957:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.13@tcp1 of length 224 into portal 12 MB=0x60008cf299c80 00000400:00000200:24.0:1616513080.852963:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.13@tcp1 of length 224/224 into md 0x865bbec1 [1] + 36624 00000400:00000200:24.0:1616513080.852969:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.852971:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.852978:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.852981:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.852982:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.852985:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.853041:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688887695613056 00010000:00000200:0.0:1616513080.853053:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3f00 x1688887695613056/t0(0) o400->d767c1cc-a310-438d-e181-d7ce2ae545a4@10.31.136.13@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.853067:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688887695613056, offset 192 00000400:00000200:0.0:1616513080.853074:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.13@tcp1 00000400:00000200:0.0:1616513080.853081:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.13@tcp1 routed destination 00000400:00000200:0.0:1616513080.853096:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.13@tcp1(10.31.136.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.853102:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.853108:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.853110:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.853112:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.853117:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.853122:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.853177:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.853183:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513080.853186:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513080.853193:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513080.853197:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb057e8 00000800:00000200:16.0:1616513080.853201:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513080.853204:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.860387:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.860466:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.860475:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.860484:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.175@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.860493:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.175@tcp1 of length 224 into portal 26 MB=0x5ffde35c96680 00000400:00000200:22.0:1616513080.860500:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.175@tcp1 of length 224/224 into md 0xf97c90d9 [1] + 0 00000400:00000200:22.0:1616513080.860504:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2d48 00000400:00000200:22.0:1616513080.860507:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.860510:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513080.860514:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2d48 00000800:00000200:22.0:1616513080.860517:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.860519:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.860520:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.860523:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.860589:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704733767296 00010000:00000200:20.0:1616513080.860599:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688704733767296/t0(0) o400->1b3a82f4-9898-806b-7bf2-7f5c88bcc870@10.31.132.175@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.860612:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704733767296, offset 192 00000400:00000200:20.0:1616513080.860618:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.175@tcp1 00000400:00000200:20.0:1616513080.860625:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.175@tcp1 routed destination 00000400:00000200:20.0:1616513080.860639:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.175@tcp1(10.31.132.175@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.860645:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.860650:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.860653:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.860655:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:7.2:1616513080.860657:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.860660:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:20.0:1616513080.860666:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513080.860672:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.860681:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.860690:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.175@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.860700:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.175@tcp1 of length 224 into portal 12 MB=0x5ffde35c96780 00000400:00000200:18.0:1616513080.860707:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.175@tcp1 of length 224/224 into md 0xf964d519 [1] + 6272 00000400:00000200:18.0:1616513080.860712:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.860715:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513080.860722:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513080.860725:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513080.860725:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:18.0:1616513080.860732:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513080.860733:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513080.860735:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.860736:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513080.860738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513080.860743:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.860748:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637dc0 00000800:00000200:2.0:1616513080.860752:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.860785:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704733767552 00010000:00000200:26.0:1616513080.860796:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfda00 x1688704733767552/t0(0) o400->ae7ff24f-7cd4-52e9-2963-a5d985dc4e32@10.31.132.175@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513080.860804:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513080.860810:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704733767552, offset 192 00000800:00000200:24.0:1616513080.860814:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513080.860815:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.175@tcp1 00000400:00000200:26.0:1616513080.860822:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.175@tcp1 routed destination 00000400:00000200:24.0:1616513080.860824:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.14@tcp1 : PUT - for me 00000400:00000200:26.0:1616513080.860836:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.175@tcp1(10.31.132.175@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.860842:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.860846:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.14@tcp1 of length 224 into portal 26 MB=0x60008478d6340 00000800:00000200:26.0:1616513080.860848:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.860850:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513080.860852:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:24.0:1616513080.860853:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.14@tcp1 of length 224/224 into md 0xf97c90f9 [1] + 0 00000800:00000200:26.0:1616513080.860857:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:24.0:1616513080.860857:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e23b8 00000800:00000200:12.0:1616513080.860858:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:24.0:1616513080.860860:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:11.2:1616513080.860862:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513080.860863:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.860868:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e23b8 00000800:00000200:24.0:1616513080.860871:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513080.860873:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513080.860876:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.860878:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:20.0:1616513080.860879:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688885420450624 00000800:00000200:24.0:1616513080.860882:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.860884:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:20.0:1616513080.860884:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688885420450624/t0(0) o400->629f31b3-e2e0-89a7-9c5f-375995e55dbc@10.31.136.14@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:24.0:1616513080.860890:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.860893:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeaa0 00000800:00000200:24.0:1616513080.860895:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513080.860895:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688885420450624, offset 192 00000800:00000200:24.0:1616513080.860898:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:20.0:1616513080.860900:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.14@tcp1 00000400:00000200:20.0:1616513080.860905:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.14@tcp1 routed destination 00000400:00000200:20.0:1616513080.860917:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.14@tcp1(10.31.136.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513080.860920:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.860923:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.860927:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.860929:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.860930:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.860931:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.860934:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513080.860936:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:20.0:1616513080.860937:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513080.860938:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513080.860942:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:22.0:1616513080.860943:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.14@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.860944:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.860946:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.860950:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.14@tcp1 of length 224 into portal 12 MB=0x60008478d6500 00000400:00000200:18.0:1616513080.860950:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.860954:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6364c8 00000400:00000200:22.0:1616513080.860955:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.14@tcp1 of length 224/224 into md 0xf964d519 [1] + 6496 00000800:00000200:18.0:1616513080.860957:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.860959:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.860962:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513080.860967:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.860969:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.860971:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.860979:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688885420451072 00010000:00000200:26.0:1616513080.860984:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec206300 x1688885420451072/t0(0) o400->a9446681-f6d7-a18f-8cfb-7d8d26dce36b@10.31.136.14@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.860995:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688885420451072, offset 192 00000400:00000200:26.0:1616513080.860999:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.14@tcp1 00000400:00000200:26.0:1616513080.861005:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.14@tcp1 routed destination 00000400:00000200:26.0:1616513080.861017:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.14@tcp1(10.31.136.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.861022:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.861027:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.861029:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.861030:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.861033:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.861038:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.861051:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.861056:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.861059:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513080.861060:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.861065:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.861069:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afcb0 00000800:00000200:12.0:1616513080.861073:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.867648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513080.867706:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.867716:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513080.867725:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.97@tcp1 : PUT - for me 00000400:00000200:16.0:1616513080.867735:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.97@tcp1 of length 224 into portal 26 MB=0x6059dff8addc0 00000400:00000200:16.0:1616513080.867741:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.97@tcp1 of length 224/224 into md 0xf97c9109 [1] + 0 00000400:00000200:16.0:1616513080.867746:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2e58 00000400:00000200:16.0:1616513080.867749:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513080.867751:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513080.867756:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2e58 00000800:00000200:16.0:1616513080.867759:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.867761:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513080.867762:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513080.867765:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.867820:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695026015559104 00010000:00000200:20.0:1616513080.867831:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1695026015559104/t0(0) o400->ef1e02a1-a5d1-af17-1655-84b9285b5084@10.31.136.97@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.867845:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695026015559104, offset 192 00000400:00000200:20.0:1616513080.867853:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.97@tcp1 00000400:00000200:20.0:1616513080.867861:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.97@tcp1 routed destination 00000400:00000200:20.0:1616513080.867876:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.97@tcp1(10.31.136.97@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.867883:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.867888:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.867891:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.867893:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.867898:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.867904:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.867905:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513080.867957:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.867964:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.867967:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513080.867974:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.867979:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637980 00000800:00000200:2.0:1616513080.867983:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.867986:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.868251:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513080.868266:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.868275:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.868285:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.97@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.868294:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.97@tcp1 of length 224 into portal 12 MB=0x6059dff8adf80 00000400:00000200:18.0:1616513080.868300:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.97@tcp1 of length 224/224 into md 0xf964d519 [1] + 6720 00000400:00000200:18.0:1616513080.868305:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.868308:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513080.868314:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.868317:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.868320:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.868324:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.868379:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695026015559552 00010000:00000200:26.0:1616513080.868392:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf9f80 x1695026015559552/t0(0) o400->b7bdf782-a843-8c7c-d3f1-b90309cdb2ea@10.31.136.97@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.868406:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695026015559552, offset 192 00000400:00000200:26.0:1616513080.868412:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.97@tcp1 00000400:00000200:26.0:1616513080.868420:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.97@tcp1 routed destination 00000400:00000200:26.0:1616513080.868450:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.97@tcp1(10.31.136.97@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.868457:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.868463:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.868465:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.868468:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.868472:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.868478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.868531:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.868538:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.868541:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.868547:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.868551:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afc28 00000800:00000200:24.0:1616513080.868555:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.868558:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:27.0:1616513080.870570:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:27.0:1616513080.870577:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513080.871878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.871935:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.871944:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.871953:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.141@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.871963:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.141@tcp1 of length 224 into portal 26 MB=0x5ffe021a5ac00 00000400:00000200:28.0:1616513080.871969:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.141@tcp1 of length 224/224 into md 0xf97c9119 [1] + 0 00000400:00000200:28.0:1616513080.871974:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2908 00000400:00000200:28.0:1616513080.871977:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.871981:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513080.871985:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2908 00000800:00000200:16.0:1616513080.871987:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.871988:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.871991:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.871993:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.872046:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688712985816064 00010000:00000200:20.0:1616513080.872056:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688712985816064/t0(0) o400->dfd21c04-7de5-d3ee-e8ee-972666fe86e0@10.31.143.141@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.872069:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688712985816064, offset 192 00000800:00000200:7.2:1616513080.872074:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513080.872075:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.141@tcp1 00000400:00000200:20.0:1616513080.872082:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.141@tcp1 routed destination 00000400:00000200:20.0:1616513080.872096:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.141@tcp1(10.31.143.141@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.872102:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.872108:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.872110:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.872112:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.872117:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:20.0:1616513080.872123:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513080.872126:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.872136:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513080.872144:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.141@tcp1 : PUT - for me 00000400:00000200:2.0:1616513080.872153:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.141@tcp1 of length 224 into portal 12 MB=0x5ffe021a5ac80 00000400:00000200:2.0:1616513080.872159:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.141@tcp1 of length 224/224 into md 0xf964d519 [1] + 6944 00000400:00000200:2.0:1616513080.872167:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513080.872169:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513080.872176:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513080.872178:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513080.872179:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513080.872184:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513080.872186:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513080.872188:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513080.872188:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513080.872194:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513080.872199:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637980 00000800:00000200:14.0:1616513080.872203:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.872241:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688712985816192 00010000:00000200:26.0:1616513080.872253:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21f080 x1688712985816192/t0(0) o400->6a8afd93-80ce-d836-675d-55d60d1fabf4@10.31.143.141@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.872267:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688712985816192, offset 192 00000400:00000200:26.0:1616513080.872274:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.141@tcp1 00000400:00000200:26.0:1616513080.872282:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.141@tcp1 routed destination 00000400:00000200:26.0:1616513080.872297:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.141@tcp1(10.31.143.141@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.872303:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.872309:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.872312:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.872314:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.872319:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.872324:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.872378:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.872384:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.872387:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.872393:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.872396:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afb18 00000800:00000200:24.0:1616513080.872400:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.872403:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.877508:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513080.877561:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.877570:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.877579:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.138@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.877588:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.138@tcp1 of length 224 into portal 26 MB=0x5ffe361c99840 00000400:00000200:18.0:1616513080.877595:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.138@tcp1 of length 224/224 into md 0xf97c9129 [1] + 0 00000400:00000200:18.0:1616513080.877600:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2330 00000400:00000200:18.0:1616513080.877603:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.877607:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513080.877611:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2330 00000800:00000200:18.0:1616513080.877614:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.877615:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.877617:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.877622:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.877625:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.877626:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513080.877634:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.138@tcp1 : PUT - for me 00000400:00000200:2.0:1616513080.877643:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.138@tcp1 of length 224 into portal 12 MB=0x5ffe361c99b40 00000400:00000200:2.0:1616513080.877649:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.138@tcp1 of length 224/224 into md 0xf964d519 [1] + 7168 00000400:00000200:2.0:1616513080.877654:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513080.877657:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513080.877663:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.877683:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.877684:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.877691:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726946814016 00010000:00000200:20.0:1616513080.877701:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688726946814016/t0(0) o400->7d8e3399-e22b-0e2c-8298-8b41b9a2d6ba@10.31.141.138@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.877714:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726946814016, offset 192 00000400:00000200:20.0:1616513080.877720:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.138@tcp1 00000400:00000200:20.0:1616513080.877727:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.138@tcp1 routed destination 00000100:00000200:26.0:1616513080.877730:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726946814784 00000400:00000200:20.0:1616513080.877741:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.138@tcp1(10.31.141.138@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:26.0:1616513080.877742:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd219b00 x1688726946814784/t0(0) o400->9cedaa22-c031-ebfa-8e89-ef8166c1a289@10.31.141.138@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.877751:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:26.0:1616513080.877755:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726946814784, offset 192 00000800:00000200:20.0:1616513080.877756:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.877759:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:26.0:1616513080.877761:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.138@tcp1 00000800:00000200:20.0:1616513080.877761:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.877765:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513080.877768:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.138@tcp1 routed destination 00000800:00000200:7.2:1616513080.877769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.877772:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513080.877782:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.138@tcp1(10.31.141.138@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.877788:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.877793:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.877795:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.877800:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.877804:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513080.877820:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.877827:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.877829:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513080.877830:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.877836:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.877841:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637dc0 00000800:00000200:22.0:1616513080.877844:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.877851:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.877909:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.877917:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513080.877920:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513080.877926:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513080.877931:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af540 00000800:00000200:16.0:1616513080.877935:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513080.877938:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513080.890481:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725925984, portal 4 00000100:00000200:30.0:1616513080.890486:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 24, xid 1693067725925984, offset 0 00000400:00000200:30.0:1616513080.890493:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:30.0:1616513080.890499:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:30.0:1616513080.890511:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513080.890518:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:30.0:1616513080.890524:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513080.890528:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:30.0:1616513080.890530:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:30.0:1616513080.890535:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513080.890583:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:2.0:1616513080.890640:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:2.0:1616513080.890647:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:2.0:1616513080.890650:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.890653:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:2.0:1616513080.890667:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000100:00000200:2.0:1616513080.890673:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9640480 x1693067725925984/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513080.890684:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef540 00000800:00000200:2.0:1616513080.890688:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:31.2:1616513080.890752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:18.0:1616513080.890805:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:18.0:1616513080.890814:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.234@o2ib1 00000400:00000200:18.0:1616513080.890822:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513080.890829:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 336 into portal 4 MB=0x603d60c626e60 00000400:00000200:18.0:1616513080.890835:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.234@o2ib1 of length 336/336 into md 0xf97c9749 [1] + 192 00000400:00000200:18.0:1616513080.890840:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.890845:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9640480 x1693067725925984/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513080.890858:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:18.0:1616513080.890859:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:18.0:1616513080.890862:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000100:00000200:30.0:1616513080.890864:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9640480 x1693067725925984/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513080.890864:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:30.0:1616513080.890875:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9640480 x1693067725925984/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513080.890885:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:7.2:1616513080.902786:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513080.902799:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.902807:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513080.902815:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.227@tcp1 : PUT - for me 00000400:00000200:14.0:1616513080.902824:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.227@tcp1 of length 224 into portal 26 MB=0x5ffd48fbddc00 00000400:00000200:14.0:1616513080.902831:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.227@tcp1 of length 224/224 into md 0xf97c9139 [1] + 0 00000400:00000200:14.0:1616513080.902835:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e26e8 00000400:00000200:14.0:1616513080.902837:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513080.902840:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513080.902844:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e26e8 00000800:00000200:14.0:1616513080.902849:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.902851:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.902852:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.902855:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.902907:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663293287424 00010000:00000200:20.0:1616513080.902918:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688663293287424/t0(0) o400->45dfeb1b-d1b1-b131-0bca-63e6326a400c@10.31.132.227@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.902931:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663293287424, offset 192 00000400:00000200:20.0:1616513080.902937:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.227@tcp1 00000400:00000200:20.0:1616513080.902944:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.227@tcp1 routed destination 00000400:00000200:20.0:1616513080.902958:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.227@tcp1(10.31.132.227@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.902964:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.902969:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.902972:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.902974:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.902978:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.902984:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.902985:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513080.903037:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.903044:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.903046:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.903053:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.903058:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637a90 00000800:00000200:22.0:1616513080.903062:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.903065:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.903445:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.903501:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.903511:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.903519:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.227@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.903527:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.227@tcp1 of length 224 into portal 12 MB=0x5ffd48fbdddc0 00000400:00000200:24.0:1616513080.903533:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.227@tcp1 of length 224/224 into md 0x865bbec1 [1] + 36848 00000400:00000200:24.0:1616513080.903538:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.903540:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.903547:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.903550:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.903551:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.903554:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.903612:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663293287872 00010000:00000200:0.0:1616513080.903624:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d2d00 x1688663293287872/t0(0) o400->5eb2ee84-6c84-15a7-be07-4b1e987ac795@10.31.132.227@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.903639:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663293287872, offset 192 00000400:00000200:0.0:1616513080.903647:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.227@tcp1 00000400:00000200:0.0:1616513080.903654:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.227@tcp1 routed destination 00000400:00000200:0.0:1616513080.903669:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.227@tcp1(10.31.132.227@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.903676:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.903682:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.903684:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.903686:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.903691:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.903696:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.903750:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.903758:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513080.903760:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513080.903767:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513080.903771:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04110 00000800:00000200:16.0:1616513080.903775:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513080.903778:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.918782:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.918838:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.918847:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.918856:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.17@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.918865:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.17@tcp1 of length 224 into portal 26 MB=0x602197088fc00 00000400:00000200:12.0:1616513080.918871:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.17@tcp1 of length 224/224 into md 0xf97c9179 [1] + 0 00000400:00000200:12.0:1616513080.918876:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2088 00000400:00000200:12.0:1616513080.918878:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.918881:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.918886:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2088 00000800:00000200:12.0:1616513080.918889:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.918891:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.918895:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.918897:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.918950:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691158145727488 00010000:00000200:20.0:1616513080.918961:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1691158145727488/t0(0) o400->b3e13609-051f-46ad-3de9-19495b010df9@10.31.133.17@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.918975:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691158145727488, offset 192 00000400:00000200:20.0:1616513080.918981:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.17@tcp1 00000400:00000200:20.0:1616513080.918989:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.17@tcp1 routed destination 00000400:00000200:20.0:1616513080.919004:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.17@tcp1(10.31.133.17@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.919010:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.919016:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.919019:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.919023:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.919027:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.919033:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.919034:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513080.919095:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.919102:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.919105:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.919107:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.919111:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.919117:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637540 00000800:00000200:2.0:1616513080.919120:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513080.919443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.919497:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.919507:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.919516:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.17@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.919525:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.17@tcp1 of length 224 into portal 12 MB=0x602197088fd80 00000400:00000200:22.0:1616513080.919531:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.17@tcp1 of length 224/224 into md 0xf964d519 [1] + 7392 00000400:00000200:22.0:1616513080.919536:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.919539:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513080.919547:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.919550:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.919552:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.919556:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.919579:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513080.919610:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691158145727872 00010000:00000200:26.0:1616513080.919622:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec204c80 x1691158145727872/t0(0) o400->f14982cf-ac5f-1486-ceb5-ddfee8f150e0@10.31.133.17@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513080.919633:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513080.919636:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691158145727872, offset 192 00000400:00000200:26.0:1616513080.919642:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.17@tcp1 00000800:00000200:16.0:1616513080.919643:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513080.919649:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.17@tcp1 routed destination 00000400:00000200:16.0:1616513080.919652:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.112@tcp1 : PUT - for me 00000400:00000200:26.0:1616513080.919663:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.17@tcp1(10.31.133.17@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.919669:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513080.919673:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.112@tcp1 of length 224 into portal 26 MB=0x5fff276933f80 00000800:00000200:26.0:1616513080.919675:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.919677:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513080.919679:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:16.0:1616513080.919680:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.112@tcp1 of length 224/224 into md 0xf97c9189 [1] + 0 00000800:00000200:26.0:1616513080.919684:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513080.919685:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2550 00000800:00000200:28.0:1616513080.919688:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:16.0:1616513080.919688:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513080.919691:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513080.919696:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2550 00000800:00000200:28.0:1616513080.919697:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513080.919699:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513080.919699:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513080.919701:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513080.919702:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513080.919704:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.919706:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:20.0:1616513080.919708:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688791720083328 00000400:00000200:28.0:1616513080.919710:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af430 00000800:00000200:28.0:1616513080.919714:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:20.0:1616513080.919714:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688791720083328/t0(0) o400->e269b3ec-e155-3d6a-093e-803274d3b10e@10.31.136.112@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.919725:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688791720083328, offset 192 00000400:00000200:20.0:1616513080.919730:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.112@tcp1 00000400:00000200:20.0:1616513080.919737:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.112@tcp1 routed destination 00000400:00000200:20.0:1616513080.919750:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.112@tcp1(10.31.136.112@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.919756:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.919761:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.919763:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.919765:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.919768:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.919772:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.919772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.919780:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.919784:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513080.919787:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513080.919788:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513080.919793:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513080.919798:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636330 00000800:00000200:14.0:1616513080.919802:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.920389:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.920457:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.920466:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513080.920467:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.920475:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.112@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.920484:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.112@tcp1 of length 224 into portal 12 MB=0x5fff2769340c0 00000400:00000200:24.0:1616513080.920490:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.112@tcp1 of length 224/224 into md 0x865bbec1 [1] + 37072 00000400:00000200:24.0:1616513080.920495:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.920498:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.920504:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.920507:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.920509:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.920568:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688791720083648 00010000:00000200:0.0:1616513080.920581:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d7500 x1688791720083648/t0(0) o400->6366e0cb-c6a4-2589-cb49-fa8139809a59@10.31.136.112@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.920595:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688791720083648, offset 192 00000400:00000200:0.0:1616513080.920601:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.112@tcp1 00000400:00000200:0.0:1616513080.920609:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.112@tcp1 routed destination 00000400:00000200:0.0:1616513080.920624:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.112@tcp1(10.31.136.112@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.920630:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.920636:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.920639:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.920641:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.920646:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.920651:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.920661:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.920665:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513080.920667:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513080.920669:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513080.920673:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513080.920676:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:16.0:1616513080.920679:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.921173:0:49541:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513080.921227:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.921236:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513080.921245:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.40@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.921254:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.40@tcp1 of length 224 into portal 26 MB=0x5ffd3d64512c0 00000400:00000200:18.0:1616513080.921260:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.40@tcp1 of length 224/224 into md 0xf97c9199 [1] + 0 00000400:00000200:18.0:1616513080.921272:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3b18 00000400:00000200:18.0:1616513080.921275:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.921277:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513080.921281:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513080.921281:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3b18 00000800:00000200:18.0:1616513080.921285:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513080.921287:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.921289:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.921345:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660181586624 00010000:00000200:20.0:1616513080.921356:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688660181586624/t0(0) o400->96aedb00-2601-e1bc-d2a1-0a87dc6ca49e@10.31.129.40@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.921370:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660181586624, offset 192 00000400:00000200:20.0:1616513080.921376:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.40@tcp1 00000400:00000200:20.0:1616513080.921383:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.40@tcp1 routed destination 00000400:00000200:20.0:1616513080.921398:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.40@tcp1(10.31.129.40@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.921405:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.921410:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.921413:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.921415:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.921420:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.921425:0:49541:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.921426:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513080.921480:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.921489:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.921492:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513080.921499:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.921504:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636440 00000800:00000200:2.0:1616513080.921507:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.921510:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.921516:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513080.921519:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513080.921524:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.40@tcp1 : PUT - for me 00000400:00000200:2.0:1616513080.921532:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.40@tcp1 of length 224 into portal 12 MB=0x5ffd3d6451680 00000400:00000200:2.0:1616513080.921538:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.40@tcp1 of length 224/224 into md 0xf964d519 [1] + 7616 00000400:00000200:2.0:1616513080.921543:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513080.921546:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513080.921551:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513080.921554:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.921556:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.921614:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660181587584 00010000:00000200:26.0:1616513080.921626:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21e780 x1688660181587584/t0(0) o400->1318df1b-33b5-0ff0-ca74-2e6862367db3@10.31.129.40@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.921641:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660181587584, offset 192 00000400:00000200:26.0:1616513080.921647:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.40@tcp1 00000400:00000200:26.0:1616513080.921654:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.40@tcp1 routed destination 00000400:00000200:26.0:1616513080.921669:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.40@tcp1(10.31.129.40@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.921675:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.921681:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.921684:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.921686:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.921691:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.921696:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.921763:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.921769:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.921772:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.921779:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.921782:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aebb0 00000800:00000200:12.0:1616513080.921787:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.921790:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.930608:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513080.930664:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.930673:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513080.930682:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.143@tcp1 : PUT - for me 00000400:00000200:28.0:1616513080.930691:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.143@tcp1 of length 224 into portal 26 MB=0x6000797688d80 00000400:00000200:28.0:1616513080.930697:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.143@tcp1 of length 224/224 into md 0xf97c91a9 [1] + 0 00000400:00000200:28.0:1616513080.930702:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2bb0 00000400:00000200:28.0:1616513080.930705:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.930708:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513080.930712:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2bb0 00000800:00000200:28.0:1616513080.930715:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.930716:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.930718:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513080.930720:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.930772:0:49542:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513080.930775:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688882465246592 00010000:00000200:20.0:1616513080.930786:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688882465246592/t0(0) o400->c9097233-8e80-5e71-0166-6d7bd829339d@10.31.131.143@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.930802:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688882465246592, offset 192 00000400:00000200:20.0:1616513080.930808:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.143@tcp1 00000400:00000200:20.0:1616513080.930816:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.143@tcp1 routed destination 00000400:00000200:20.0:1616513080.930831:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.143@tcp1(10.31.131.143@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.930838:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513080.930842:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.930843:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.930846:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513080.930848:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513080.930851:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.930853:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:20.0:1616513080.930860:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513080.930861:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.143@tcp1 : PUT - for me 00000400:00000200:18.0:1616513080.930870:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.143@tcp1 of length 224 into portal 12 MB=0x6000797689200 00000400:00000200:18.0:1616513080.930877:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.143@tcp1 of length 224/224 into md 0xf964d519 [1] + 7840 00000400:00000200:18.0:1616513080.930882:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.930884:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513080.930891:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513080.930894:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513080.930896:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513080.930898:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.930902:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.930904:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.930907:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513080.930909:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.930914:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637100 00000800:00000200:18.0:1616513080.930917:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513080.930957:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688882465247744 00010000:00000200:26.0:1616513080.930969:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfe780 x1688882465247744/t0(0) o400->c87c2bc8-1de6-9d04-5769-efe7ed403116@10.31.131.143@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513080.930983:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688882465247744, offset 192 00000400:00000200:26.0:1616513080.930991:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.143@tcp1 00000400:00000200:26.0:1616513080.930998:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.143@tcp1 routed destination 00000400:00000200:26.0:1616513080.931013:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.143@tcp1(10.31.131.143@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.931020:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.931026:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.931028:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.931030:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.931035:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.931041:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.931106:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.931113:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.931115:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.931122:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.931126:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:12.0:1616513080.931130:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.931133:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.936374:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513080.936430:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.936450:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513080.936460:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.38@tcp1 : PUT - for me 00000400:00000200:16.0:1616513080.936470:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.38@tcp1 of length 224 into portal 26 MB=0x5f91eeb1df600 00000400:00000200:16.0:1616513080.936477:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.38@tcp1 of length 224/224 into md 0xf97c91c9 [1] + 0 00000400:00000200:16.0:1616513080.936482:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3210 00000800:00000200:28.0:1616513080.936483:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513080.936484:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513080.936487:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513080.936492:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3210 00000800:00000200:28.0:1616513080.936493:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513080.936495:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513080.936498:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513080.936500:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.936502:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.38@tcp1 : PUT - for me 00000800:00000200:16.0:1616513080.936503:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513080.936511:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.38@tcp1 of length 224 into portal 12 MB=0x5f91eeb1df680 00000400:00000200:28.0:1616513080.936517:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.38@tcp1 of length 224/224 into md 0x865bbec1 [1] + 37296 00000400:00000200:28.0:1616513080.936522:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513080.936525:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513080.936532:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.936535:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.936538:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.936570:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681286072497664 00010000:00000200:20.0:1616513080.936581:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1681286072497664/t0(0) o400->c1a08d6b-6aa8-8e21-422c-4704e29ffed2@10.31.132.38@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.936594:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681286072497664, offset 192 00000100:00000200:0.0:1616513080.936599:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681286072497792 00000400:00000200:20.0:1616513080.936600:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.38@tcp1 00000400:00000200:20.0:1616513080.936607:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.38@tcp1 routed destination 00010000:00000200:0.0:1616513080.936612:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9b600 x1681286072497792/t0(0) o400->d027e780-d178-ea5b-689a-0b56cdb31243@10.31.132.38@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513080.936620:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.38@tcp1(10.31.132.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.936626:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:0.0:1616513080.936626:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681286072497792, offset 192 00000800:00000200:20.0:1616513080.936632:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:0.0:1616513080.936633:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.38@tcp1 00000800:00000200:20.0:1616513080.936634:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.936636:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513080.936640:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.38@tcp1 routed destination 00000800:00000200:20.0:1616513080.936641:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.936647:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.936648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513080.936655:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.38@tcp1(10.31.132.38@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.936662:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.936667:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.936672:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.936674:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.936679:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.936684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513080.936702:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513080.936709:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.936711:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513080.936718:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513080.936723:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636b28 00000800:00000200:2.0:1616513080.936726:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513080.936729:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513080.936736:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.936743:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513080.936746:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513080.936747:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.936752:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.936755:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ba0 00000800:00000200:24.0:1616513080.936759:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.944473:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513080.944527:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.944536:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513080.944544:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.69@tcp1 : PUT - for me 00000400:00000200:12.0:1616513080.944552:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.69@tcp1 of length 224 into portal 26 MB=0x5f908f2a51740 00000400:00000200:12.0:1616513080.944559:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.69@tcp1 of length 224/224 into md 0xf97c91d9 [1] + 0 00000400:00000200:12.0:1616513080.944563:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2220 00000400:00000200:12.0:1616513080.944565:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513080.944568:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513080.944572:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2220 00000800:00000200:12.0:1616513080.944575:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.944577:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513080.944579:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.944581:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.944633:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681191709513536 00010000:00000200:20.0:1616513080.944643:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1681191709513536/t0(0) o400->f44f004b-d92c-9804-7cd4-eb17a4b5409f@10.31.131.69@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.944656:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681191709513536, offset 192 00000400:00000200:20.0:1616513080.944662:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.69@tcp1 00000400:00000200:20.0:1616513080.944669:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.69@tcp1 routed destination 00000400:00000200:20.0:1616513080.944684:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.69@tcp1(10.31.131.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.944690:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.944696:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.944698:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.944700:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.944705:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.944711:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.944712:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.944768:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513080.944771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513080.944775:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513080.944778:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513080.944785:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513080.944791:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637ed0 00000800:00000200:22.0:1616513080.944794:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.944797:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513080.944824:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.944834:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513080.944842:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.69@tcp1 : PUT - for me 00000400:00000200:16.0:1616513080.944851:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.69@tcp1 of length 224 into portal 12 MB=0x5f908f2a517c0 00000400:00000200:16.0:1616513080.944857:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.69@tcp1 of length 224/224 into md 0x865bbec1 [1] + 37520 00000400:00000200:16.0:1616513080.944863:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513080.944865:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513080.944872:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.944875:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513080.944877:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.944879:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.944933:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681191709513664 00010000:00000200:0.0:1616513080.944945:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5e80 x1681191709513664/t0(0) o400->81b758a3-55ba-386c-3ed1-953a8baf2913@10.31.131.69@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.944958:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681191709513664, offset 192 00000400:00000200:0.0:1616513080.944964:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.69@tcp1 00000400:00000200:0.0:1616513080.944971:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.69@tcp1 routed destination 00000400:00000200:0.0:1616513080.944985:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.69@tcp1(10.31.131.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.944991:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.944996:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.944998:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.945001:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.945005:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.945010:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.945028:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.945032:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.945035:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513080.945037:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513080.945040:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.945044:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05320 00000800:00000200:12.0:1616513080.945048:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.954858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513080.954915:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.954924:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513080.954933:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.95@tcp1 : PUT - for me 00000400:00000200:16.0:1616513080.954942:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.95@tcp1 of length 224 into portal 26 MB=0x5ffe089c1b3c0 00000400:00000200:16.0:1616513080.954949:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.95@tcp1 of length 224/224 into md 0xf97c9209 [1] + 0 00000400:00000200:16.0:1616513080.954954:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2ee0 00000400:00000200:16.0:1616513080.954957:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513080.954960:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513080.954964:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2ee0 00000800:00000200:16.0:1616513080.954967:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.954968:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.954976:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513080.954979:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513080.954979:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513080.954981:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.954987:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.95@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.954996:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.95@tcp1 of length 224 into portal 12 MB=0x5ffe089c1b440 00000400:00000200:24.0:1616513080.955004:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.95@tcp1 of length 224/224 into md 0x865bbec1 [1] + 37744 00000400:00000200:24.0:1616513080.955009:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.955012:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513080.955018:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.955022:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513080.955024:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513080.955028:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714732483520 00010000:00000200:20.0:1616513080.955039:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688714732483520/t0(0) o400->67859123-528a-9200-4fe0-c5d9b15d2878@10.31.140.95@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.955053:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688714732483520, offset 192 00000400:00000200:20.0:1616513080.955060:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.95@tcp1 00000400:00000200:20.0:1616513080.955067:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.95@tcp1 routed destination 00000400:00000200:20.0:1616513080.955082:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.95@tcp1(10.31.140.95@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513080.955082:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714732483648 00000800:00000200:20.0:1616513080.955088:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.955094:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513080.955095:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d0d80 x1688714732483648/t0(0) o400->fe62f3a8-78c1-343f-bbf5-a1d7b22b505a@10.31.140.95@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.955097:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.955099:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.955104:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.955109:0:49545:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513080.955109:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688714732483648, offset 192 00000100:00000200:20.0:1616513080.955111:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513080.955116:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.95@tcp1 00000400:00000200:0.0:1616513080.955123:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.95@tcp1 routed destination 00000400:00000200:0.0:1616513080.955138:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.95@tcp1(10.31.140.95@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.955145:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.955151:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.955154:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.955156:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.955161:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513080.955164:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513080.955166:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513080.955172:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513080.955174:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.955178:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513080.955178:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513080.955181:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513080.955185:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513080.955187:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513080.955187:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513080.955187:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637430 00000800:00000200:16.0:1616513080.955189:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513080.955190:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513080.955190:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.955193:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:12.0:1616513080.955194:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513080.955194:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.955196:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513080.955198:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05dc0 00000800:00000200:12.0:1616513080.955202:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.958022:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513080.958078:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.958087:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513080.958096:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.124@tcp1 : PUT - for me 00000400:00000200:24.0:1616513080.958105:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.124@tcp1 of length 224 into portal 26 MB=0x603fe9f651e00 00000400:00000200:24.0:1616513080.958111:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.124@tcp1 of length 224/224 into md 0xf97c9219 [1] + 0 00000400:00000200:24.0:1616513080.958116:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e24c8 00000400:00000200:24.0:1616513080.958119:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513080.958122:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513080.958126:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e24c8 00000800:00000200:24.0:1616513080.958129:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.958131:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.958133:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.958136:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513080.958151:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513080.958191:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693241991044608 00010000:00000200:20.0:1616513080.958202:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1693241991044608/t0(0) o400->223cf57d-2548-f420-39f3-2a4bfc1d9768@10.31.143.124@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513080.958209:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513080.958216:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693241991044608, offset 192 00000800:00000200:16.0:1616513080.958219:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513080.958222:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.124@tcp1 00000400:00000200:16.0:1616513080.958227:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.124@tcp1 : PUT - for me 00000400:00000200:20.0:1616513080.958229:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.124@tcp1 routed destination 00000400:00000200:16.0:1616513080.958239:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.124@tcp1 of length 224 into portal 12 MB=0x603fe9f651e80 00000400:00000200:20.0:1616513080.958244:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.124@tcp1(10.31.143.124@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513080.958245:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.124@tcp1 of length 224/224 into md 0x865bbec1 [1] + 37968 00000400:00000200:16.0:1616513080.958250:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513080.958251:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:16.0:1616513080.958253:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513080.958256:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513080.958259:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513080.958261:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513080.958262:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513080.958263:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513080.958264:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.958267:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:20.0:1616513080.958268:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513080.958274:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513080.958275:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513080.958324:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693241991044736 00000800:00000200:18.0:1616513080.958335:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:0.0:1616513080.958337:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1b00 x1693241991044736/t0(0) o400->030d8229-c6f1-7df2-d0d7-4cbeef7b3ac6@10.31.143.124@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513080.958346:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.958349:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.958356:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.958361:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636dd0 00000800:00000200:18.0:1616513080.958365:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.958368:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513080.958370:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693241991044736, offset 192 00000400:00000200:0.0:1616513080.958376:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.124@tcp1 00000400:00000200:0.0:1616513080.958382:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.124@tcp1 routed destination 00000400:00000200:0.0:1616513080.958396:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.124@tcp1(10.31.143.124@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.958402:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.958408:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.958410:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.958412:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.958417:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.958422:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513080.958432:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.958442:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513080.958452:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513080.958454:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513080.958460:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513080.958463:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04ee0 00000800:00000200:24.0:1616513080.958466:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.964634:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513080.964689:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.964698:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513080.964707:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.75@tcp1 : PUT - for me 00000400:00000200:14.0:1616513080.964718:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.75@tcp1 of length 224 into portal 26 MB=0x5ffd3f3bf3a00 00000400:00000200:14.0:1616513080.964724:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.75@tcp1 of length 224/224 into md 0xf97c9229 [1] + 0 00000400:00000200:14.0:1616513080.964729:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2ff0 00000400:00000200:14.0:1616513080.964732:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513080.964734:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513080.964739:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2ff0 00000800:00000200:14.0:1616513080.964742:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.964743:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513080.964745:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513080.964747:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.964802:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660676131328 00010000:00000200:20.0:1616513080.964820:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688660676131328/t0(0) o400->fd96f19b-1d3e-031e-fa89-e010d96d7ff1@10.31.133.75@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.964834:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660676131328, offset 192 00000400:00000200:20.0:1616513080.964840:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.75@tcp1 00000400:00000200:20.0:1616513080.964848:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.75@tcp1 routed destination 00000400:00000200:20.0:1616513080.964863:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.75@tcp1(10.31.133.75@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.964870:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513080.964876:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.964879:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.964881:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.964886:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513080.964891:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.964892:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513080.964907:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.964920:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.964923:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513080.964930:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.964935:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636dd0 00000800:00000200:18.0:1616513080.964939:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.964942:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.965237:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513080.965298:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.965307:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513080.965316:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.75@tcp1 : PUT - for me 00000400:00000200:16.0:1616513080.965325:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.75@tcp1 of length 224 into portal 12 MB=0x5ffd3f3bf3b00 00000400:00000200:16.0:1616513080.965331:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.75@tcp1 of length 224/224 into md 0x865bbec1 [1] + 38192 00000400:00000200:16.0:1616513080.965337:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513080.965339:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513080.965345:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513080.965348:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513080.965350:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.965353:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513080.965412:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660676131584 00010000:00000200:0.0:1616513080.965424:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1f80 x1688660676131584/t0(0) o400->7ee83bbb-2a3f-9d6f-9f0b-0a16599df0ea@10.31.133.75@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513080.965491:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660676131584, offset 192 00000400:00000200:0.0:1616513080.965497:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.75@tcp1 00000400:00000200:0.0:1616513080.965504:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.75@tcp1 routed destination 00000400:00000200:0.0:1616513080.965518:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.75@tcp1(10.31.133.75@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.965524:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.965529:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.965532:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.965542:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.965547:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.965552:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513080.965605:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513080.965612:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513080.965615:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513080.965621:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.965625:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04330 00000800:00000200:28.0:1616513080.965629:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513080.965632:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513080.966125:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513080.966179:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.966189:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513080.966198:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.85@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.966206:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.85@tcp1 of length 224 into portal 26 MB=0x5ffe0e9586200 00000400:00000200:22.0:1616513080.966213:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.85@tcp1 of length 224/224 into md 0xf97c9239 [1] + 0 00000400:00000200:22.0:1616513080.966218:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e35c8 00000400:00000200:22.0:1616513080.966223:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.966226:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513080.966230:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e35c8 00000800:00000200:14.0:1616513080.966232:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.966233:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513080.966236:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.966237:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.966292:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513080.966294:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716336194048 00000800:00000200:16.0:1616513080.966301:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:20.0:1616513080.966305:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688716336194048/t0(0) o400->b5176c7e-5ad3-9f5d-7564-437bf40bb1a0@10.31.133.85@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513080.966307:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513080.966309:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513080.966314:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.85@tcp1 : PUT - for me 00000100:00000200:20.0:1616513080.966319:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688716336194048, offset 192 00000400:00000200:16.0:1616513080.966320:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.85@tcp1 of length 224 into portal 12 MB=0x5ffe0e9586280 00000400:00000200:20.0:1616513080.966326:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.85@tcp1 00000400:00000200:16.0:1616513080.966330:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.85@tcp1 of length 224/224 into md 0x865bbec1 [1] + 38416 00000400:00000200:20.0:1616513080.966333:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.85@tcp1 routed destination 00000400:00000200:16.0:1616513080.966334:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513080.966336:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513080.966341:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513080.966345:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513080.966347:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:20.0:1616513080.966348:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.85@tcp1(10.31.133.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.966355:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:0.0:1616513080.966356:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716336194176 00000800:00000200:20.0:1616513080.966360:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.966363:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:0.0:1616513080.966363:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd7500 x1688716336194176/t0(0) o400->36428bd2-c661-6e27-a631-cd5142e5ce3a@10.31.133.85@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.966365:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.966370:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513080.966374:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688716336194176, offset 192 00000800:00000200:7.2:1616513080.966375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.966377:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513080.966379:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.85@tcp1 00000400:00000200:0.0:1616513080.966385:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.85@tcp1 routed destination 00000400:00000200:0.0:1616513080.966398:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.85@tcp1(10.31.133.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513080.966406:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513080.966411:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513080.966413:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513080.966415:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513080.966417:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513080.966423:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513080.966428:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513080.966433:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513080.966446:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513080.966446:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513080.966449:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513080.966449:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513080.966451:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513080.966456:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.966456:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513080.966459:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb053a8 00000800:00000200:28.0:1616513080.966461:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513080.966464:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637ed0 00000800:00000200:18.0:1616513080.966467:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513080.966472:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:25.0:1616513080.986655:0:6489:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926000, portal 4 00000100:00000200:25.0:1616513080.986660:0:6489:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926000, offset 0 00000400:00000200:25.0:1616513080.986667:0:6489:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:25.0:1616513080.986674:0:6489:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:25.0:1616513080.986688:0:6489:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:25.0:1616513080.986695:0:6489:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:25.0:1616513080.986702:0:6489:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:25.0:1616513080.986705:0:6489:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:25.0:1616513080.986707:0:6489:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:25.0:1616513080.986713:0:6489:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513080.986720:0:6489:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513080.986734:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513080.986741:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513080.986744:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513080.986750:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513080.986756:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cab103f00 x1693067725926000/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513080.986779:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492672550 00000800:00000200:14.0:1616513080.986784:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513080.986786:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513080.986939:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513080.986992:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513080.987001:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513080.987010:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513080.987017:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626e70 00000400:00000200:18.0:1616513080.987024:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c9849 [1] + 192 00000400:00000200:18.0:1616513080.987029:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513080.987034:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cab103f00 x1693067725926000/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513080.987050:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513080.987050:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513080.987052:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:18.0:1616513080.987054:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:25.0:1616513080.987059:0:6489:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cab103f00 x1693067725926000/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:25.0:1616513080.987072:0:6489:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cab103f00 x1693067725926000/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513087 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:25.0:1616513080.987082:0:6489:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492673650 00000800:00000200:7.2:1616513080.995209:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513080.995267:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513080.995276:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513080.995285:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.102@tcp1 : PUT - for me 00000400:00000200:14.0:1616513080.995294:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.102@tcp1 of length 224 into portal 26 MB=0x6001f83286d80 00000400:00000200:14.0:1616513080.995301:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.102@tcp1 of length 224/224 into md 0xf97c9249 [1] + 0 00000400:00000200:14.0:1616513080.995306:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3760 00000400:00000200:14.0:1616513080.995309:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513080.995312:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513080.995316:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3760 00000800:00000200:22.0:1616513080.995318:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.995320:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513080.995322:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513080.995324:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513080.995327:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513080.995327:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513080.995335:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.102@tcp1 : PUT - for me 00000400:00000200:22.0:1616513080.995343:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.102@tcp1 of length 224 into portal 12 MB=0x6001f83286ec0 00000400:00000200:22.0:1616513080.995348:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.102@tcp1 of length 224/224 into md 0xf964d519 [1] + 8064 00000400:00000200:22.0:1616513080.995353:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513080.995355:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513080.995363:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513080.995366:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513080.995367:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513080.995381:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688985204714880 00010000:00000200:20.0:1616513080.995392:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688985204714880/t0(0) o400->c4d3ac0e-1182-a63e-9e7f-5b65f13323f3@10.31.142.102@tcp1:37/0 lens 224/224 e 0 to 0 dl 1616513172 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513080.995406:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688985204714880, offset 192 00000400:00000200:20.0:1616513080.995411:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.102@tcp1 00000400:00000200:20.0:1616513080.995418:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.102@tcp1 routed destination 00000100:00000200:26.0:1616513080.995431:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688985204715200 00000400:00000200:20.0:1616513080.995432:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.102@tcp1(10.31.142.102@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513080.995453:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:26.0:1616513080.995459:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec205100 x1688985204715200/t0(0) o400->4db62ebc-a774-0c64-543b-93eda1f50d9a@10.31.142.102@tcp1:706/0 lens 224/224 e 0 to 0 dl 1616513086 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513080.995459:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513080.995461:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513080.995463:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513080.995468:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:26.0:1616513080.995474:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688985204715200, offset 192 00000800:00000200:7.2:1616513080.995474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513080.995475:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513080.995480:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.102@tcp1 00000400:00000200:26.0:1616513080.995488:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.102@tcp1 routed destination 00000400:00000200:26.0:1616513080.995503:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.102@tcp1(10.31.142.102@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513080.995509:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513080.995515:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513080.995518:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513080.995520:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513080.995524:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513080.995527:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513080.995534:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513080.995537:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513080.995538:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513080.995543:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513080.995548:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637100 00000800:00000200:18.0:1616513080.995552:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513080.995571:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513080.995626:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513080.995632:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513080.995635:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513080.995641:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513080.995644:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aedd0 00000800:00000200:12.0:1616513080.995648:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513080.995651:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.003103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.003161:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.003170:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.003180:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.217@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.003189:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.217@tcp1 of length 224 into portal 26 MB=0x5ffd4de15ee40 00000400:00000200:22.0:1616513081.003196:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.217@tcp1 of length 224/224 into md 0xf97c9269 [1] + 0 00000400:00000200:22.0:1616513081.003200:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2000 00000400:00000200:22.0:1616513081.003203:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.003206:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513081.003210:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2000 00000800:00000200:22.0:1616513081.003213:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.003215:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.003215:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.003219:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.003274:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664607682112 00010000:00000200:20.0:1616513081.003285:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688664607682112/t0(0) o400->aa762f93-1b8e-b331-cc7f-688bb402d9d2@10.31.132.217@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.003299:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664607682112, offset 192 00000400:00000200:20.0:1616513081.003305:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.217@tcp1 00000400:00000200:20.0:1616513081.003313:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.217@tcp1 routed destination 00000400:00000200:20.0:1616513081.003328:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.217@tcp1(10.31.132.217@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.003335:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.003341:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.003344:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.003346:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.003351:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.003356:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.003357:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513081.003411:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.003417:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.003420:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513081.003427:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.003447:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636330 00000800:00000200:14.0:1616513081.003451:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.003454:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513081.003947:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513081.003958:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.003966:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513081.003967:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513081.003975:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.217@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.003985:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.217@tcp1 of length 224 into portal 12 MB=0x5ffd4de15efc0 00000400:00000200:2.0:1616513081.003992:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.217@tcp1 of length 224/224 into md 0xf964d519 [1] + 8288 00000400:00000200:2.0:1616513081.003997:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.004000:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513081.004007:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.004009:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.004011:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.004076:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664607682496 00010000:00000200:26.0:1616513081.004088:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd218d80 x1688664607682496/t0(0) o400->6f89170a-6cce-ea3e-7957-9785a5a43938@10.31.132.217@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.004103:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664607682496, offset 192 00000400:00000200:26.0:1616513081.004109:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.217@tcp1 00000400:00000200:26.0:1616513081.004116:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.217@tcp1 routed destination 00000400:00000200:26.0:1616513081.004131:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.217@tcp1(10.31.132.217@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.004137:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.004143:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.004147:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.004150:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.004154:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.004162:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.004218:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.004225:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.004228:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.004234:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.004237:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae3b8 00000800:00000200:24.0:1616513081.004241:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.004244:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.011213:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513081.011268:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.011277:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.011286:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.179@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.011295:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.179@tcp1 of length 224 into portal 26 MB=0x60182975275c0 00000400:00000200:12.0:1616513081.011302:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.179@tcp1 of length 224/224 into md 0xf97c9289 [1] + 0 00000400:00000200:12.0:1616513081.011307:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2b28 00000400:00000200:12.0:1616513081.011309:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.011312:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513081.011316:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2b28 00000800:00000200:12.0:1616513081.011319:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.011322:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.011323:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.011327:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.011380:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510256403904 00010000:00000200:20.0:1616513081.011391:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1690510256403904/t0(0) o400->ddd9c5da-476c-f45f-482c-f971b5236694@10.31.140.179@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513081.011402:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513081.011405:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510256403904, offset 192 00000400:00000200:20.0:1616513081.011413:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.179@tcp1 00000400:00000200:20.0:1616513081.011420:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.179@tcp1 routed destination 00000400:00000200:20.0:1616513081.011449:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.179@tcp1(10.31.140.179@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.011456:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513081.011459:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.011461:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.011464:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.011466:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513081.011470:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.011471:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.011478:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513081.011479:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.179@tcp1 : PUT - for me 00000800:00000200:7.2:1616513081.011479:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513081.011487:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.179@tcp1 of length 224 into portal 12 MB=0x60182975276c0 00000400:00000200:24.0:1616513081.011494:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.179@tcp1 of length 224/224 into md 0x865bbec1 [1] + 38640 00000400:00000200:24.0:1616513081.011499:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.011502:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.011509:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.011512:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.011514:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.011517:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513081.011541:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.011547:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.011550:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513081.011556:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.011556:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.011561:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637a90 00000800:00000200:14.0:1616513081.011564:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513081.011576:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510256404160 00010000:00000200:0.0:1616513081.011588:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3180 x1690510256404160/t0(0) o400->34f02572-be7d-c10e-0eaa-5b5871adffdc@10.31.140.179@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.011602:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510256404160, offset 192 00000400:00000200:0.0:1616513081.011607:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.179@tcp1 00000400:00000200:0.0:1616513081.011614:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.179@tcp1 routed destination 00000400:00000200:0.0:1616513081.011628:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.179@tcp1(10.31.140.179@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.011634:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.011639:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.011642:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.011644:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.011648:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.011654:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.011665:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.011670:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.011673:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513081.011674:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.011680:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.011684:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:16.0:1616513081.011687:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.014373:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.014426:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.014446:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.014455:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.208@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.014480:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.208@tcp1 of length 224 into portal 26 MB=0x5ffd4e2d660c0 00000400:00000200:22.0:1616513081.014486:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.208@tcp1 of length 224/224 into md 0xf97c92a9 [1] + 0 00000800:00000200:2.0:1616513081.014488:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513081.014491:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3078 00000400:00000200:22.0:1616513081.014493:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.014496:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513081.014500:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3078 00000800:00000200:22.0:1616513081.014502:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.014505:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.014507:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.014560:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664687403200 00010000:00000200:20.0:1616513081.014570:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688664687403200/t0(0) o400->37aa35fe-4e92-4df9-04f6-9e97c4f93ffa@10.31.132.208@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.014583:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664687403200, offset 192 00000400:00000200:20.0:1616513081.014589:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.208@tcp1 00000400:00000200:20.0:1616513081.014596:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.208@tcp1 routed destination 00000400:00000200:20.0:1616513081.014610:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.208@tcp1(10.31.132.208@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.014616:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.014621:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.014624:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.014626:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.014630:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.014635:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.014637:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513081.014655:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513081.014687:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.014694:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.014696:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513081.014703:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513081.014708:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637980 00000800:00000200:28.0:1616513081.014712:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513081.014712:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.014715:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513081.014721:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513081.014730:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.208@tcp1 : PUT - for me 00000400:00000200:28.0:1616513081.014739:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.208@tcp1 of length 224 into portal 12 MB=0x5ffd4e2d66240 00000400:00000200:28.0:1616513081.014745:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.208@tcp1 of length 224/224 into md 0x865bbec1 [1] + 38864 00000400:00000200:28.0:1616513081.014751:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513081.014753:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513081.014760:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.014763:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513081.014765:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.014768:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.014824:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664687403584 00010000:00000200:0.0:1616513081.014836:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9cc80 x1688664687403584/t0(0) o400->d62dcb77-2811-d3a7-ccb6-d311bdef97c7@10.31.132.208@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.014851:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664687403584, offset 192 00000400:00000200:0.0:1616513081.014857:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.208@tcp1 00000400:00000200:0.0:1616513081.014865:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.208@tcp1 routed destination 00000400:00000200:0.0:1616513081.014879:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.208@tcp1(10.31.132.208@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.014886:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.014892:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.014894:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.014896:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.014901:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.014905:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.014917:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.014924:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.014926:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.014933:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.014939:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000800:00000200:12.0:1616513081.014943:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.014946:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.017928:0:49552:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513081.017992:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.018014:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.018022:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.71@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.018030:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.71@tcp1 of length 224 into portal 26 MB=0x603ff925a5880 00000400:00000200:2.0:1616513081.018036:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.71@tcp1 of length 224/224 into md 0xf97c92b9 [1] + 0 00000400:00000200:2.0:1616513081.018041:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3188 00000400:00000200:2.0:1616513081.018043:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.018046:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513081.018049:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3188 00000800:00000200:2.0:1616513081.018052:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.018055:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.018056:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.018058:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.018110:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693246067202176 00000800:00000200:7.2:1616513081.018110:0:49552:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:20.0:1616513081.018120:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1693246067202176/t0(0) o400->04fa827e-3599-c137-54a9-04b2c00e1c7d@10.31.132.71@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.018133:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693246067202176, offset 192 00000400:00000200:20.0:1616513081.018138:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.71@tcp1 00000400:00000200:20.0:1616513081.018145:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.71@tcp1 routed destination 00000400:00000200:20.0:1616513081.018159:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.71@tcp1(10.31.132.71@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.018169:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.018175:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513081.018176:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.018177:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.018179:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513081.018184:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513081.018186:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:20.0:1616513081.018190:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513081.018194:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.71@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.018203:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.71@tcp1 of length 224 into portal 12 MB=0x603ff925a5980 00000400:00000200:18.0:1616513081.018209:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.71@tcp1 of length 224/224 into md 0xf964d519 [1] + 8512 00000400:00000200:18.0:1616513081.018215:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.018219:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513081.018226:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513081.018228:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513081.018230:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513081.018235:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513081.018236:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513081.018238:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.018238:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513081.018245:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.018251:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636330 00000800:00000200:14.0:1616513081.018255:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.018293:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693246067202432 00010000:00000200:26.0:1616513081.018305:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfc800 x1693246067202432/t0(0) o400->5ac0bcfc-1a27-13a0-ce6e-43358c67e172@10.31.132.71@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.018319:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693246067202432, offset 192 00000400:00000200:26.0:1616513081.018325:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.71@tcp1 00000400:00000200:26.0:1616513081.018332:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.71@tcp1 routed destination 00000400:00000200:26.0:1616513081.018348:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.71@tcp1(10.31.132.71@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.018354:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.018360:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.018362:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.018364:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.018369:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.018375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.018429:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.018446:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.018448:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.018454:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.018458:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aff58 00000800:00000200:24.0:1616513081.018462:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.018464:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:17.0:1616513081.051844:0:6490:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926016, portal 4 00000100:00000200:17.0:1616513081.051849:0:6490:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926016, offset 0 00000400:00000200:17.0:1616513081.051856:0:6490:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:17.0:1616513081.051863:0:6490:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:17.0:1616513081.051878:0:6490:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:17.0:1616513081.051884:0:6490:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:17.0:1616513081.051891:0:6490:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:17.0:1616513081.051894:0:6490:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:17.0:1616513081.051896:0:6490:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:17.0:1616513081.051901:0:6490:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513081.051909:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513081.051924:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513081.051930:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513081.051933:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.051940:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513081.051946:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843a2ef500 x1693067725926016/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513081.051957:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3e550 00000800:00000200:22.0:1616513081.051961:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513081.051964:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513081.052116:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513081.052171:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513081.052180:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:2.0:1616513081.052188:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513081.052196:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626e80 00000400:00000200:2.0:1616513081.052202:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c98d9 [1] + 192 00000400:00000200:2.0:1616513081.052207:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.052212:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843a2ef500 x1693067725926016/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513081.052223:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513081.052226:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513081.052230:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513081.052231:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:17.0:1616513081.052236:0:6490:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843a2ef500 x1693067725926016/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:17.0:1616513081.052249:0:6490:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843a2ef500 x1693067725926016/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:17.0:1616513081.052260:0:6490:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3f980 00000800:00000200:7.2:1616513081.052492:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.052504:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.052510:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513081.052514:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513081.052518:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.204@tcp1 : PUT - for me 00000800:00000200:18.0:1616513081.052525:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513081.052526:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.052527:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.204@tcp1 of length 224 into portal 26 MB=0x5ffd5617540c0 00000400:00000200:22.0:1616513081.052533:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.204@tcp1 of length 224/224 into md 0xf97c92c9 [1] + 0 00000400:00000200:18.0:1616513081.052533:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.204@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.052537:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e34b8 00000400:00000200:22.0:1616513081.052539:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.052542:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513081.052542:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.204@tcp1 of length 224 into portal 12 MB=0x5ffd5617541c0 00000400:00000200:22.0:1616513081.052545:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e34b8 00000800:00000200:22.0:1616513081.052548:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.052550:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.052550:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.204@tcp1 of length 224/224 into md 0xf964d519 [1] + 8736 00000800:00000200:22.0:1616513081.052552:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513081.052555:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.052558:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513081.052565:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.052569:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.052571:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.052610:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666811744448 00010000:00000200:20.0:1616513081.052621:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688666811744448/t0(0) o400->18f6135f-fa58-e65c-5f22-c81ebe2788bc@10.31.136.204@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513081.052630:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666811744704 00000100:00000200:20.0:1616513081.052634:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666811744448, offset 192 00000400:00000200:20.0:1616513081.052640:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.204@tcp1 00010000:00000200:26.0:1616513081.052642:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfa400 x1688666811744704/t0(0) o400->b1983fc3-8e2f-c02b-6a34-af04f96344bb@10.31.136.204@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513081.052647:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.204@tcp1 routed destination 00000100:00000200:26.0:1616513081.052656:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666811744704, offset 192 00000400:00000200:20.0:1616513081.052661:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.204@tcp1(10.31.136.204@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513081.052662:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.204@tcp1 00000800:00000200:20.0:1616513081.052668:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513081.052670:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.204@tcp1 routed destination 00000800:00000200:20.0:1616513081.052674:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.052677:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.052679:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.052683:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513081.052684:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.204@tcp1(10.31.136.204@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513081.052689:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513081.052690:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:20.0:1616513081.052690:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513081.052695:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.052697:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.052699:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513081.052699:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.052703:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513081.052704:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513081.052705:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513081.052707:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513081.052711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513081.052711:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.052716:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637ed0 00000800:00000200:2.0:1616513081.052719:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513081.052767:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.052774:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.052777:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.052784:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.052788:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aff58 00000800:00000200:12.0:1616513081.052791:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.052794:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:17.0:1616513081.098669:0:6493:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926048, portal 4 00000100:00000200:17.0:1616513081.098675:0:6493:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926048, offset 0 00000400:00000200:17.0:1616513081.098681:0:6493:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:13.0:1616513081.098686:0:6491:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926032, portal 4 00000400:00000200:17.0:1616513081.098688:0:6493:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000100:00000200:13.0:1616513081.098692:0:6491:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926032, offset 0 00000400:00000200:13.0:1616513081.098699:0:6491:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:17.0:1616513081.098701:0:6493:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:13.0:1616513081.098705:0:6491:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:17.0:1616513081.098708:0:6493:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:13.0:1616513081.098718:0:6491:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:13.0:1616513081.098725:0:6491:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:17.0:1616513081.098726:0:6493:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:17.0:1616513081.098729:0:6493:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:17.0:1616513081.098731:0:6493:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:13.0:1616513081.098731:0:6491:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:17.0:1616513081.098736:0:6493:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:13.0:1616513081.098736:0:6491:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:13.0:1616513081.098738:0:6491:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:13.0:1616513081.098743:0:6491:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:27.2:1616513081.098783:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513081.098804:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:1.2:1616513081.098804:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:27.0:1616513081.098810:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513081.098814:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513081.098822:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:1.0:1616513081.098824:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000100:00000200:27.0:1616513081.098829:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4558ad7500 x1693067725926048/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513081.098829:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513081.098832:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:9.0:1616513081.098834:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513081.098838:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:27.0:1616513081.098841:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3e3b8 00000800:00000200:27.0:1616513081.098845:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:1.0:1616513081.098847:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdd689200 x1693067725926032/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513081.098848:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:1.0:1616513081.098859:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81783457e8 00000800:00000200:1.0:1616513081.098864:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513081.098888:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513081.098898:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513081.098905:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513081.098913:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:27.2:1616513081.098920:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000400:00000200:1.0:1616513081.098921:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626e90 00000400:00000200:1.0:1616513081.098928:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffc45 [1] + 192 00000800:00000200:27.0:1616513081.098929:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:1.0:1616513081.098933:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:27.0:1616513081.098935:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000100:00000200:1.0:1616513081.098937:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdd689200 x1693067725926032/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513081.098939:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513081.098943:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:5.0:1616513081.098950:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:27.0:1616513081.098951:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626ea0 00000800:00000200:1.0:1616513081.098952:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513081.098955:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:27.0:1616513081.098957:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166bdd [1] + 192 00000800:00000200:1.0:1616513081.098957:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:13.0:1616513081.098958:0:6491:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdd689200 x1693067725926032/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513081.098962:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513081.098966:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4558ad7500 x1693067725926048/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:13.0:1616513081.098971:0:6491:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdd689200 x1693067725926032/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513081.098980:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:13.0:1616513081.098981:0:6491:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345210 00000800:00000200:27.0:1616513081.098982:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513081.098984:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:17.0:1616513081.098987:0:6493:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4558ad7500 x1693067725926048/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:17.0:1616513081.099000:0:6493:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4558ad7500 x1693067725926048/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:17.0:1616513081.099010:0:6493:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3f980 00000100:00000200:25.0:1616513081.099487:0:6493:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926064, portal 4 00000100:00000200:25.0:1616513081.099492:0:6493:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926064, offset 0 00000100:00000200:5.0:1616513081.099498:0:6494:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926080, portal 4 00000400:00000200:25.0:1616513081.099499:0:6493:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000100:00000200:5.0:1616513081.099503:0:6494:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926080, offset 0 00000400:00000200:25.0:1616513081.099505:0:6493:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:5.0:1616513081.099510:0:6494:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:5.0:1616513081.099516:0:6494:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:25.0:1616513081.099518:0:6493:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:25.0:1616513081.099525:0:6493:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:5.0:1616513081.099529:0:6494:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:25.0:1616513081.099531:0:6493:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:25.0:1616513081.099534:0:6493:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:25.0:1616513081.099536:0:6493:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:5.0:1616513081.099536:0:6494:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:25.0:1616513081.099541:0:6493:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:5.0:1616513081.099542:0:6494:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513081.099545:0:6494:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:25.2:1616513081.099547:0:6493:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (22)++ 00000800:00000200:5.0:1616513081.099547:0:6494:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:5.0:1616513081.099551:0:6494:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513081.099558:0:6494:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:22.0:1616513081.099561:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513081.099569:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513081.099574:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.099581:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513081.099588:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdd68c800 x1693067725926064/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513081.099599:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492673188 00000800:00000200:22.0:1616513081.099604:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513081.099607:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:24.0:1616513081.099613:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513081.099621:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:24.0:1616513081.099624:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.099630:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:24.0:1616513081.099637:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8302326300 x1693067725926080/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513081.099648:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55f980 00000800:00000200:24.0:1616513081.099653:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513081.099655:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:25.2:1616513081.099770:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:5.2:1616513081.099785:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:14.0:1616513081.099825:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513081.099836:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513081.099845:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:12.0:1616513081.099845:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:14.0:1616513081.099854:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626eb0 00000800:00000200:12.0:1616513081.099855:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:14.0:1616513081.099862:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97c9909 [1] + 192 00000400:00000200:12.0:1616513081.099864:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513081.099867:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.099872:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdd68c800 x1693067725926064/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513081.099872:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626ec0 00000400:00000200:12.0:1616513081.099879:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d63c1 [1] + 192 00000800:00000200:2.0:1616513081.099883:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:12.0:1616513081.099885:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513081.099888:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000100:00000200:12.0:1616513081.099890:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8302326300 x1693067725926080/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513081.099891:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513081.099892:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:25.0:1616513081.099897:0:6493:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdd68c800 x1693067725926064/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513081.099901:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513081.099906:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513081.099909:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:25.0:1616513081.099910:0:6493:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdd68c800 x1693067725926064/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513081.099911:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:5.0:1616513081.099915:0:6494:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8302326300 x1693067725926080/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:25.0:1616513081.099920:0:6493:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492673650 00000100:00000200:5.0:1616513081.099928:0:6494:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8302326300 x1693067725926080/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513081.099938:0:6494:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55e7f8 00000800:00000200:7.2:1616513081.105770:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.105828:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513081.105849:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513081.105850:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.105858:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.72@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.105867:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.72@tcp1 of length 224 into portal 26 MB=0x5ffe870760400 00000400:00000200:22.0:1616513081.105874:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.72@tcp1 of length 224/224 into md 0xf97c92e9 [1] + 0 00000400:00000200:22.0:1616513081.105878:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e38f8 00000400:00000200:22.0:1616513081.105880:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.105883:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513081.105884:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513081.105888:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e38f8 00000800:00000200:22.0:1616513081.105890:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.105893:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.105895:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513081.105908:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.105918:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.105927:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.72@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.105936:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.72@tcp1 of length 224 into portal 12 MB=0x5ffe8707606c0 00000400:00000200:24.0:1616513081.105943:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.72@tcp1 of length 224/224 into md 0x865bbec1 [1] + 39088 00000400:00000200:24.0:1616513081.105948:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.105952:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:20.0:1616513081.105957:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748667831296 00000800:00000200:24.0:1616513081.105960:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.105963:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.105965:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.105968:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:20.0:1616513081.105970:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688748667831296/t0(0) o400->d179fd01-46da-fd2d-2e46-3d82c899cc08@10.31.142.72@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.105986:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688748667831296, offset 192 00000400:00000200:20.0:1616513081.105992:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.72@tcp1 00000400:00000200:20.0:1616513081.106000:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.72@tcp1 routed destination 00000400:00000200:20.0:1616513081.106016:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.72@tcp1(10.31.142.72@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.106022:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.106029:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513081.106031:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748667832000 00000800:00000200:20.0:1616513081.106032:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.106036:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.106042:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513081.106045:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d2400 x1688748667832000/t0(0) o400->462c1d49-7216-2529-5b4c-f1ac43445960@10.31.142.72@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513081.106048:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.106049:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513081.106060:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688748667832000, offset 192 00000400:00000200:0.0:1616513081.106067:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.72@tcp1 00000400:00000200:0.0:1616513081.106075:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.72@tcp1 routed destination 00000400:00000200:0.0:1616513081.106090:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.72@tcp1(10.31.142.72@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.106097:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513081.106101:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513081.106103:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.106105:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513081.106108:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513081.106108:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513081.106111:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513081.106113:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513081.106117:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513081.106118:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513081.106123:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637ed0 00000800:00000200:2.0:1616513081.106127:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.106130:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513081.106178:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.106185:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.106188:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513081.106190:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.106195:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.106199:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000800:00000200:16.0:1616513081.106203:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.120838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.120894:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.120903:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.120912:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.30@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.120922:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.30@tcp1 of length 224 into portal 26 MB=0x5ffd43acb6140 00000400:00000200:24.0:1616513081.120929:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.30@tcp1 of length 224/224 into md 0xf97c92f9 [1] + 0 00000400:00000200:24.0:1616513081.120933:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3ba0 00000400:00000200:24.0:1616513081.120936:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.120939:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513081.120943:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3ba0 00000800:00000200:24.0:1616513081.120946:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.120947:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.120949:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.120951:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.121006:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661868110144 00010000:00000200:20.0:1616513081.121017:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688661868110144/t0(0) o400->3545b267-e391-f1d4-e679-dc502daefd40@10.31.133.30@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.121031:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661868110144, offset 192 00000400:00000200:20.0:1616513081.121037:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.30@tcp1 00000400:00000200:20.0:1616513081.121046:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.30@tcp1 routed destination 00000400:00000200:20.0:1616513081.121061:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.30@tcp1(10.31.133.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.121067:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.121073:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.121076:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.121078:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.121083:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.121089:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.121090:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.121149:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.121156:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.121158:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513081.121165:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513081.121171:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637980 00000800:00000200:18.0:1616513081.121175:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.121178:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.121369:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513081.121379:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.121387:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.121395:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.30@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.121403:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.30@tcp1 of length 224 into portal 12 MB=0x5ffd43acb62c0 00000400:00000200:12.0:1616513081.121409:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.30@tcp1 of length 224/224 into md 0x865bbec1 [1] + 39312 00000400:00000200:12.0:1616513081.121414:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.121416:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513081.121422:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.121425:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.121427:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.121430:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.121488:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661868110528 00010000:00000200:0.0:1616513081.121501:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d8d80 x1688661868110528/t0(0) o400->f72b97b6-daf8-7e2b-3124-4b46a97f52ef@10.31.133.30@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.121515:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661868110528, offset 192 00000400:00000200:0.0:1616513081.121521:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.30@tcp1 00000400:00000200:0.0:1616513081.121529:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.30@tcp1 routed destination 00000400:00000200:0.0:1616513081.121544:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.30@tcp1(10.31.133.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.121551:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.121557:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.121559:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.121561:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.121566:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.121572:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.121583:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.121588:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.121591:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513081.121593:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.121599:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.121603:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:28.0:1616513081.121608:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.125542:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.125598:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.125607:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.125608:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513081.125612:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513081.125617:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.76@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.125619:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.76@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.125628:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.76@tcp1 of length 224 into portal 12 MB=0x5ffd45f1aebc0 00000400:00000200:16.0:1616513081.125629:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.76@tcp1 of length 224 into portal 26 MB=0x5ffd45f1aeb00 00000400:00000200:12.0:1616513081.125633:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.76@tcp1 of length 224/224 into md 0x865bbec1 [1] + 39536 00000400:00000200:16.0:1616513081.125635:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.76@tcp1 of length 224/224 into md 0xf97c9319 [1] + 0 00000400:00000200:12.0:1616513081.125637:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.125638:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513081.125640:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2cc0 00000800:00000200:12.0:1616513081.125642:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:16.0:1616513081.125643:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513081.125644:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:16.0:1616513081.125646:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513081.125646:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.125648:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.125651:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2cc0 00000800:00000200:16.0:1616513081.125654:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.125657:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.125658:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.125705:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662477302720 00010000:00000200:0.0:1616513081.125717:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d8000 x1688662477302720/t0(0) o400->d5df828c-2d2a-2f8a-ee3b-039281dad81d@10.31.133.76@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513081.125727:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662477302528 00000100:00000200:0.0:1616513081.125730:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662477302720, offset 192 00000400:00000200:0.0:1616513081.125736:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.76@tcp1 00010000:00000200:20.0:1616513081.125738:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688662477302528/t0(0) o400->6b644c4c-ffbc-e741-5876-6d65b49a097d@10.31.133.76@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513081.125743:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.76@tcp1 routed destination 00000100:00000200:20.0:1616513081.125752:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662477302528, offset 192 00000400:00000200:0.0:1616513081.125758:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.76@tcp1(10.31.133.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513081.125759:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.76@tcp1 00000800:00000200:0.0:1616513081.125765:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513081.125766:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.76@tcp1 routed destination 00000400:00000200:20.0:1616513081.125781:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.76@tcp1(10.31.133.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.125788:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.125788:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.125791:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513081.125793:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.125794:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.125796:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.125797:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513081.125798:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.125803:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513081.125805:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513081.125808:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.125809:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513081.125860:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.125867:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.125870:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513081.125873:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513081.125876:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513081.125877:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513081.125880:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb053a8 00000800:00000200:28.0:1616513081.125884:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513081.125884:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513081.125886:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.125887:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513081.125893:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.125899:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637980 00000800:00000200:28.0:1616513081.125903:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.168869:0:49568:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513081.168925:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.168935:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513081.168944:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.177@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.168954:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.177@tcp1 of length 224 into portal 26 MB=0x5ffd3d90ff2c0 00000400:00000200:18.0:1616513081.168961:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.177@tcp1 of length 224/224 into md 0xf97c9339 [1] + 0 00000400:00000200:18.0:1616513081.168966:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2198 00000400:00000200:18.0:1616513081.168968:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.168971:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513081.168975:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2198 00000800:00000200:18.0:1616513081.168979:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.168984:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.168984:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.168987:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.169040:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660228436672 00010000:00000200:20.0:1616513081.169050:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688660228436672/t0(0) o400->ece651ef-212c-4e84-69db-10093e1f8911@10.31.129.177@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.169066:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660228436672, offset 192 00000400:00000200:20.0:1616513081.169073:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.177@tcp1 00000400:00000200:20.0:1616513081.169080:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.177@tcp1 routed destination 00000400:00000200:20.0:1616513081.169096:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.177@tcp1(10.31.129.177@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.169102:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.169108:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.169111:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.169113:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.169118:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.169123:0:49568:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.169125:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513081.169175:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.169183:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.169185:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513081.169192:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.169198:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388110 00000800:00000200:14.0:1616513081.169201:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.169204:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513081.169261:0:49568:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.169272:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.169279:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513081.169281:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513081.169287:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.177@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.169296:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.177@tcp1 of length 224 into portal 12 MB=0x5ffd3d90ff700 00000400:00000200:22.0:1616513081.169301:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.177@tcp1 of length 224/224 into md 0xf964d519 [1] + 8960 00000400:00000200:22.0:1616513081.169306:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.169309:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513081.169316:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.169319:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.169321:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.169382:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660228437760 00010000:00000200:26.0:1616513081.169394:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec201680 x1688660228437760/t0(0) o400->4bc32416-913c-5802-080f-451c029466d5@10.31.129.177@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.169420:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660228437760, offset 192 00000400:00000200:26.0:1616513081.169426:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.177@tcp1 00000400:00000200:26.0:1616513081.169448:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.177@tcp1 routed destination 00000400:00000200:26.0:1616513081.169464:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.177@tcp1(10.31.129.177@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.169470:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.169476:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.169479:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.169481:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.169486:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.169531:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.169588:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.169595:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.169598:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.169604:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.169608:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae3b8 00000800:00000200:24.0:1616513081.169612:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.169615:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.225752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513081.225812:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.225823:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513081.225832:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.111@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.225843:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.111@tcp1 of length 224 into portal 26 MB=0x600073726d400 00000400:00000200:14.0:1616513081.225850:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.111@tcp1 of length 224/224 into md 0xf97c9369 [1] + 0 00000400:00000200:14.0:1616513081.225855:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2110 00000400:00000200:14.0:1616513081.225858:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.225861:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513081.225868:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2110 00000800:00000200:2.0:1616513081.225869:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.225872:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.225875:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513081.225877:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.225880:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.225881:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.225889:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.111@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.225899:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.111@tcp1 of length 224 into portal 12 MB=0x600073726d780 00000400:00000200:2.0:1616513081.225906:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.111@tcp1 of length 224/224 into md 0xf964d519 [1] + 9184 00000400:00000200:2.0:1616513081.225912:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.225915:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513081.225923:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.225926:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.225928:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.225937:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688880850326528 00010000:00000200:20.0:1616513081.225949:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688880850326528/t0(0) o400->88c54824-bce5-bda6-ebc5-e83fbf4673b9@10.31.136.111@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.225964:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688880850326528, offset 192 00000400:00000200:20.0:1616513081.225971:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.111@tcp1 00000400:00000200:20.0:1616513081.225978:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.111@tcp1 routed destination 00000100:00000200:26.0:1616513081.225990:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688880850327424 00000400:00000200:20.0:1616513081.225994:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.111@tcp1(10.31.136.111@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.226001:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:26.0:1616513081.226003:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21da00 x1688880850327424/t0(0) o400->75c0fa68-4966-2b0b-4592-9a58127f9cd3@10.31.136.111@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.226008:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.226011:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.226013:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:26.0:1616513081.226018:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688880850327424, offset 192 00000800:00000200:20.0:1616513081.226018:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513081.226024:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.111@tcp1 00000800:00000200:7.2:1616513081.226024:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.226026:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513081.226032:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.111@tcp1 routed destination 00000400:00000200:26.0:1616513081.226047:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.111@tcp1(10.31.136.111@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.226054:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.226061:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.226063:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.226066:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.226071:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.226079:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513081.226085:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.226092:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.226095:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513081.226095:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513081.226105:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.226111:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388cc0 00000800:00000200:22.0:1616513081.226114:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513081.226133:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.226140:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.226143:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.226149:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.226153:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aedd0 00000800:00000200:12.0:1616513081.226156:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.226159:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.238894:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.238952:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.238962:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.238971:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.185@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.238981:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.185@tcp1 of length 224 into portal 26 MB=0x5ffe3019e0780 00000400:00000200:24.0:1616513081.238988:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.185@tcp1 of length 224/224 into md 0xf97c9389 [1] + 0 00000400:00000200:24.0:1616513081.238993:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e22a8 00000400:00000200:24.0:1616513081.238995:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.238998:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513081.239003:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e22a8 00000800:00000200:24.0:1616513081.239008:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.239008:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.239011:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.239015:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.239018:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.239019:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513081.239027:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.185@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.239036:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.185@tcp1 of length 224 into portal 12 MB=0x5ffe3019e0880 00000400:00000200:16.0:1616513081.239042:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.185@tcp1 of length 224/224 into md 0x865bbec1 [1] + 39760 00000400:00000200:16.0:1616513081.239052:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.239055:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513081.239063:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.239066:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.239067:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.239080:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688725333346176 00010000:00000200:20.0:1616513081.239092:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688725333346176/t0(0) o400->e8d850cb-a207-c556-4737-4af179f76e65@10.31.141.185@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.239106:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688725333346176, offset 192 00000400:00000200:20.0:1616513081.239112:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.185@tcp1 00000400:00000200:20.0:1616513081.239120:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.185@tcp1 routed destination 00000100:00000200:0.0:1616513081.239128:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688725333346432 00000400:00000200:20.0:1616513081.239137:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.185@tcp1(10.31.141.185@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513081.239140:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5100 x1688725333346432/t0(0) o400->9af28cd9-be1f-bcbf-751c-04e4a322c16a@10.31.141.185@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.239144:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.239150:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.239153:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513081.239154:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688725333346432, offset 192 00000800:00000200:20.0:1616513081.239155:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.239160:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513081.239160:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.185@tcp1 00000800:00000200:7.2:1616513081.239167:0:49575:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.239173:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513081.239174:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.185@tcp1 routed destination 00000400:00000200:0.0:1616513081.239188:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.185@tcp1(10.31.141.185@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.239194:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.239199:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.239202:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.239204:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.239208:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.239214:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513081.239225:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.239232:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.239234:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513081.239237:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513081.239241:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513081.239247:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13882a8 00000800:00000200:18.0:1616513081.239250:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513081.239267:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.239275:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.239278:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513081.239279:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.239284:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.239288:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04330 00000800:00000200:28.0:1616513081.239292:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.247688:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513081.247701:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.247710:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.247719:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.246@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.247728:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.246@tcp1 of length 224 into portal 26 MB=0x5f924afeb2700 00000400:00000200:12.0:1616513081.247735:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.246@tcp1 of length 224/224 into md 0xf97c9399 [1] + 0 00000400:00000200:12.0:1616513081.247740:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e25d8 00000400:00000200:12.0:1616513081.247743:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.247746:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513081.247750:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e25d8 00000800:00000200:12.0:1616513081.247753:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.247754:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.247756:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.247758:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.247811:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681310849115904 00000800:00000200:11.2:1616513081.247812:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:20.0:1616513081.247821:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1681310849115904/t0(0) o400->3d798636-36f9-5e0b-9ce5-6d2fc15a3eea@10.31.130.246@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.247835:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681310849115904, offset 192 00000400:00000200:20.0:1616513081.247840:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.246@tcp1 00000400:00000200:20.0:1616513081.247847:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.246@tcp1 routed destination 00000400:00000200:20.0:1616513081.247861:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.246@tcp1(10.31.130.246@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.247867:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.247873:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.247875:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.247877:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513081.247878:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.247882:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513081.247887:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:20.0:1616513081.247888:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.247889:0:49576:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513081.247895:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.246@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.247904:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.246@tcp1 of length 224 into portal 12 MB=0x5f924afeb2780 00000400:00000200:24.0:1616513081.247910:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.246@tcp1 of length 224/224 into md 0x865bbec1 [1] + 39984 00000400:00000200:24.0:1616513081.247915:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.247918:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.247925:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.247928:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.247930:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.247933:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.247944:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513081.247945:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513081.247950:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681310849116032 00000800:00000200:14.0:1616513081.247951:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513081.247952:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.141@tcp1 : PUT - for me 00000800:00000200:14.0:1616513081.247958:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513081.247961:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.141@tcp1 of length 224 into portal 26 MB=0x6040f971ed300 00000400:00000200:14.0:1616513081.247961:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:0.0:1616513081.247963:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d7980 x1681310849116032/t0(0) o400->82643d43-8679-26b5-66b3-5b54f447eb8c@10.31.130.246@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513081.247967:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.247968:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.141@tcp1 of length 224/224 into md 0xf97c93a9 [1] + 0 00000400:00000200:28.0:1616513081.247973:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3dc0 00000400:00000200:14.0:1616513081.247973:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388330 00000400:00000200:28.0:1616513081.247976:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513081.247976:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513081.247977:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681310849116032, offset 192 00000100:00000200:28.0:1616513081.247979:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513081.247979:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513081.247983:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3dc0 00000400:00000200:0.0:1616513081.247983:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.246@tcp1 00000800:00000200:28.0:1616513081.247986:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.247989:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.247991:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513081.247991:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.246@tcp1 routed destination 00000100:00000200:20.0:1616513081.247995:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693314866664192 00010000:00000200:20.0:1616513081.248000:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1693314866664192/t0(0) o400->2e00d28a-985d-6eac-8245-26231f7584c2@10.31.136.141@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513081.248006:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.246@tcp1(10.31.130.246@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:20.0:1616513081.248010:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693314866664192, offset 192 00000800:00000200:0.0:1616513081.248013:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513081.248015:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.141@tcp1 00000800:00000200:0.0:1616513081.248019:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513081.248021:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.141@tcp1 routed destination 00000800:00000200:0.0:1616513081.248022:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.248026:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.248030:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513081.248033:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.141@tcp1(10.31.136.141@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513081.248035:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.248038:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.248043:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513081.248044:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513081.248047:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.248049:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513081.248050:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513081.248051:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513081.248052:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513081.248055:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513081.248056:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.248056:0:49576:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513081.248058:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.248061:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04ee0 00000800:00000200:12.0:1616513081.248064:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513081.248066:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.248071:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.248073:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513081.248081:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.248087:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388c38 00000800:00000200:2.0:1616513081.248090:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.248092:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513081.248095:0:49576:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513081.248105:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.248111:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513081.248113:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.248114:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.248115:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.248119:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513081.248349:0:49576:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513081.248358:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.248364:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513081.248366:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513081.248371:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.141@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.248379:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.141@tcp1 of length 224 into portal 12 MB=0x6040f971ed380 00000400:00000200:2.0:1616513081.248385:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.141@tcp1 of length 224/224 into md 0xf964d519 [1] + 9408 00000400:00000200:2.0:1616513081.248389:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.248392:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513081.248399:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.248401:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.248403:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.248465:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693314866664320 00010000:00000200:26.0:1616513081.248477:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21f980 x1693314866664320/t0(0) o400->546bcbf2-89e8-ea3b-9a7c-bca4ff154094@10.31.136.141@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.248490:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693314866664320, offset 192 00000400:00000200:26.0:1616513081.248496:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.141@tcp1 00000400:00000200:26.0:1616513081.248502:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.141@tcp1 routed destination 00000400:00000200:26.0:1616513081.248521:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.141@tcp1(10.31.136.141@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.248529:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.248535:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.248537:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.248539:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.248543:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.248548:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.248559:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.248564:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.248567:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513081.248569:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.248573:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.248577:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:16.0:1616513081.248581:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.308612:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.308668:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.308678:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.308689:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.35@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.308700:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.35@tcp1 of length 224 into portal 26 MB=0x60183958d3d40 00000400:00000200:24.0:1616513081.308707:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.35@tcp1 of length 224/224 into md 0xf97c93b9 [1] + 0 00000400:00000200:24.0:1616513081.308712:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e33a8 00000400:00000200:24.0:1616513081.308715:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.308718:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513081.308723:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e33a8 00000800:00000200:12.0:1616513081.308723:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.308727:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513081.308730:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513081.308732:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.308734:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513081.308735:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513081.308744:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.35@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.308753:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.35@tcp1 of length 224 into portal 12 MB=0x60183958d3dc0 00000400:00000200:12.0:1616513081.308760:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.35@tcp1 of length 224/224 into md 0x865bbec1 [1] + 40208 00000400:00000200:12.0:1616513081.308765:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.308768:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513081.308776:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.308780:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.308782:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.308791:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514521668928 00010000:00000200:20.0:1616513081.308803:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1690514521668928/t0(0) o400->ee91c86a-d8a1-a69b-700b-ba6c6cf51b70@10.31.136.35@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.308819:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514521668928, offset 192 00000400:00000200:20.0:1616513081.308825:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.35@tcp1 00000400:00000200:20.0:1616513081.308833:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.35@tcp1 routed destination 00000100:00000200:0.0:1616513081.308843:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514521669056 00000400:00000200:20.0:1616513081.308849:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.35@tcp1(10.31.136.35@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.308856:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:0.0:1616513081.308858:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dec00 x1690514521669056/t0(0) o400->75f0f248-b53c-f1d0-6c6b-088a7b7ef038@10.31.136.35@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.308863:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.308866:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.308868:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513081.308873:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514521669056, offset 192 00000800:00000200:20.0:1616513081.308874:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513081.308879:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.35@tcp1 00000800:00000200:7.2:1616513081.308880:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513081.308887:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.35@tcp1 routed destination 00000100:00000200:20.0:1616513081.308888:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513081.308898:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513081.308903:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.35@tcp1(10.31.136.35@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513081.308906:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.308909:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513081.308910:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513081.308916:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513081.308916:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.308918:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513081.308921:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13883b8 00000800:00000200:0.0:1616513081.308921:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513081.308925:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513081.308926:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513081.308928:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.308930:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.308987:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.308994:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.308997:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513081.308998:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.309004:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.309008:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05dc0 00000800:00000200:16.0:1616513081.309012:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.318385:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513081.318452:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.318461:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.318471:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.74@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.318480:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.74@tcp1 of length 224 into portal 12 MB=0x5ffe9d11b8c80 00000400:00000200:12.0:1616513081.318486:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.74@tcp1 of length 224/224 into md 0x865bbec1 [1] + 40432 00000400:00000200:12.0:1616513081.318492:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.318494:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513081.318501:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.318504:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.318506:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.318508:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.318517:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513081.318566:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688754584259712 00000800:00000200:22.0:1616513081.318577:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513081.318579:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dc380 x1688754584259712/t0(0) o400->f31f65b6-676f-3e45-9a13-a0cdeffcea86@10.31.142.74@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:22.0:1616513081.318586:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:0.0:1616513081.318593:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688754584259712, offset 192 00000400:00000200:22.0:1616513081.318596:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.74@tcp1 : PUT - for me 00000400:00000200:0.0:1616513081.318600:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.74@tcp1 00000400:00000200:22.0:1616513081.318605:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.74@tcp1 of length 224 into portal 26 MB=0x5ffe9d11b8900 00000400:00000200:0.0:1616513081.318608:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.74@tcp1 routed destination 00000400:00000200:22.0:1616513081.318611:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.74@tcp1 of length 224/224 into md 0xf97c93c9 [1] + 0 00000400:00000200:22.0:1616513081.318616:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2aa0 00000400:00000200:22.0:1616513081.318619:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.318622:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513081.318623:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.74@tcp1(10.31.142.74@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513081.318626:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2aa0 00000800:00000200:22.0:1616513081.318629:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513081.318629:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513081.318632:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.318632:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.318636:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513081.318636:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.318638:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513081.318641:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513081.318646:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.318650:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.318691:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688754584258816 00010000:00000200:20.0:1616513081.318702:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688754584258816/t0(0) o400->c7b919d3-4efc-15be-5213-1b9b61116c14@10.31.142.74@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513081.318706:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.318713:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:20.0:1616513081.318716:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688754584258816, offset 192 00000400:00000200:18.0:1616513081.318716:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513081.318722:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.74@tcp1 00000400:00000200:18.0:1616513081.318723:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513081.318727:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05320 00000400:00000200:20.0:1616513081.318730:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.74@tcp1 routed destination 00000800:00000200:18.0:1616513081.318731:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.318734:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513081.318745:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.74@tcp1(10.31.142.74@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.318751:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.318757:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.318760:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.318762:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.318767:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.318771:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.318774:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513081.318825:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.318832:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.318835:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.318842:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.318849:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388dd0 00000800:00000200:24.0:1616513081.318853:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.318856:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:31.0:1616513081.322547:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926096, portal 4 00000100:00000200:31.0:1616513081.322552:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926096, offset 0 00000400:00000200:31.0:1616513081.322558:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:31.0:1616513081.322563:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:31.0:1616513081.322574:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513081.322580:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:31.0:1616513081.322586:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513081.322588:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:31.0:1616513081.322592:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:31.0:1616513081.322597:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513081.322643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513081.322662:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513081.322668:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513081.322671:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513081.322678:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513081.322685:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be40000 x1693067725926096/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513081.322698:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2c38 00000800:00000200:1.0:1616513081.322702:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513081.322705:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513081.322817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513081.322826:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513081.322832:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513081.322835:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513081.322840:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513081.322849:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626ed0 00000400:00000200:1.0:1616513081.322855:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffc85 [1] + 192 00000400:00000200:1.0:1616513081.322861:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513081.322866:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be40000 x1693067725926096/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513081.322879:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513081.322882:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513081.322884:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:31.0:1616513081.322887:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be40000 x1693067725926096/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513081.322900:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be40000 x1693067725926096/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513081.322910:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000800:00000200:11.2:1616513081.326824:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513081.326878:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.326887:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.326896:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.48@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.326905:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.48@tcp1 of length 224 into portal 26 MB=0x602185d4a4fc0 00000400:00000200:12.0:1616513081.326912:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.48@tcp1 of length 224/224 into md 0xf97c93d9 [1] + 0 00000400:00000200:12.0:1616513081.326917:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2a18 00000400:00000200:12.0:1616513081.326919:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.326924:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513081.326928:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2a18 00000800:00000200:28.0:1616513081.326931:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.326931:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.326934:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.326936:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.327004:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691153527885760 00000800:00000200:11.2:1616513081.327012:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:20.0:1616513081.327014:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1691153527885760/t0(0) o400->f380baed-7d3a-f0c1-8c60-a7799ec73b87@10.31.129.48@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.327029:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691153527885760, offset 192 00000400:00000200:20.0:1616513081.327036:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.48@tcp1 00000400:00000200:20.0:1616513081.327043:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.48@tcp1 routed destination 00000400:00000200:20.0:1616513081.327058:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.48@tcp1(10.31.129.48@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.327065:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513081.327070:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.327071:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.327074:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.327076:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513081.327079:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.327081:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.327087:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.327087:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513081.327088:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.48@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.327097:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.48@tcp1 of length 224 into portal 12 MB=0x602185d4a5500 00000400:00000200:24.0:1616513081.327103:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.48@tcp1 of length 224/224 into md 0x865bbec1 [1] + 40656 00000400:00000200:24.0:1616513081.327108:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.327111:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.327118:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.327121:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.327123:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.327125:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513081.327147:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.327154:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.327156:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513081.327163:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.327169:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388a18 00000800:00000200:2.0:1616513081.327172:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.327175:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513081.327186:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691153527887104 00010000:00000200:0.0:1616513081.327198:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d1b00 x1691153527887104/t0(0) o400->77fc04b8-e361-1033-fe5c-165b5cf48524@10.31.129.48@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.327213:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691153527887104, offset 192 00000400:00000200:0.0:1616513081.327221:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.48@tcp1 00000400:00000200:0.0:1616513081.327228:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.48@tcp1 routed destination 00000400:00000200:0.0:1616513081.327243:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.48@tcp1(10.31.129.48@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.327249:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.327255:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.327258:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.327260:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.327265:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.327270:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.327287:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.327292:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.327294:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513081.327295:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.327301:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.327307:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ba0 00000800:00000200:28.0:1616513081.327312:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.333792:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.333846:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.333855:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.333864:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.3@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.333873:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.3@tcp1 of length 224 into portal 12 MB=0x5ffe096cb5b40 00000400:00000200:24.0:1616513081.333879:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.3@tcp1 of length 224/224 into md 0x865bbec1 [1] + 40880 00000400:00000200:24.0:1616513081.333884:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.333887:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.333893:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.333896:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.333898:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.333901:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.333941:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513081.333976:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714951220032 00010000:00000200:0.0:1616513081.333988:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d4800 x1688714951220032/t0(0) o400->f5beb4e3-596f-7203-5677-00f6bd5e5c21@10.31.142.3@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513081.333997:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513081.334001:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688714951220032, offset 192 00000800:00000200:18.0:1616513081.334006:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513081.334008:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.3@tcp1 00000800:00000200:14.0:1616513081.334012:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513081.334015:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.3@tcp1 : PUT - for me 00000400:00000200:0.0:1616513081.334015:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.3@tcp1 routed destination 00000400:00000200:0.0:1616513081.334029:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.3@tcp1(10.31.142.3@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.334035:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513081.334037:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.3@tcp1 of length 224 into portal 26 MB=0x5ffe096cb5940 00000800:00000200:0.0:1616513081.334040:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.334043:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513081.334044:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.3@tcp1 of length 224/224 into md 0xf97c93f9 [1] + 0 00000800:00000200:0.0:1616513081.334045:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513081.334049:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3100 00000800:00000200:0.0:1616513081.334050:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.334052:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.334054:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:7.2:1616513081.334054:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513081.334058:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3100 00000800:00000200:18.0:1616513081.334062:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513081.334064:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513081.334068:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.334071:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.334075:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.334077:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513081.334082:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513081.334085:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:18.0:1616513081.334088:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.334091:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.334137:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714951219520 00010000:00000200:20.0:1616513081.334147:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688714951219520/t0(0) o400->ae6d0669-f3d1-e0ab-54c1-ef6b0fa529f8@10.31.142.3@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.334160:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688714951219520, offset 192 00000400:00000200:20.0:1616513081.334166:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.3@tcp1 00000400:00000200:20.0:1616513081.334173:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.3@tcp1 routed destination 00000400:00000200:20.0:1616513081.334187:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.3@tcp1(10.31.142.3@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.334193:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.334198:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.334200:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.334202:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.334207:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.334211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.334213:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513081.334221:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.334228:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.334230:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513081.334236:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513081.334237:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.334242:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388990 00000800:00000200:12.0:1616513081.334245:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.363165:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513081.363222:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.363232:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513081.363241:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.80@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.363251:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.80@tcp1 of length 224 into portal 26 MB=0x5f922f05dde00 00000400:00000200:14.0:1616513081.363257:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.80@tcp1 of length 224/224 into md 0xf97c9419 [1] + 0 00000400:00000200:14.0:1616513081.363262:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3430 00000400:00000200:14.0:1616513081.363265:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.363268:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513081.363272:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3430 00000800:00000200:18.0:1616513081.363276:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.363276:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.363279:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.363284:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.363287:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513081.363287:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513081.363296:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.80@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.363305:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.80@tcp1 of length 224 into portal 12 MB=0x5f922f05dde80 00000400:00000200:18.0:1616513081.363312:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.80@tcp1 of length 224/224 into md 0xf964d519 [1] + 9632 00000400:00000200:18.0:1616513081.363317:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.363320:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513081.363329:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.363332:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.363334:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.363336:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681303340441088 00010000:00000200:20.0:1616513081.363347:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1681303340441088/t0(0) o400->d1641141-8af8-93c1-2d83-0865400e176e@10.31.131.80@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.363361:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681303340441088, offset 192 00000400:00000200:20.0:1616513081.363368:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.80@tcp1 00000400:00000200:20.0:1616513081.363375:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.80@tcp1 routed destination 00000400:00000200:20.0:1616513081.363389:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.80@tcp1(10.31.131.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.363395:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:26.0:1616513081.363396:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681303340441216 00000800:00000200:20.0:1616513081.363401:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.363404:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.363406:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:26.0:1616513081.363409:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfb180 x1681303340441216/t0(0) o400->316e3142-c519-feff-4b87-fd3f3bf9a425@10.31.131.80@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.363411:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.363415:0:49589:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.363417:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513081.363423:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681303340441216, offset 192 00000400:00000200:26.0:1616513081.363430:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.80@tcp1 00000400:00000200:26.0:1616513081.363452:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.80@tcp1 routed destination 00000400:00000200:26.0:1616513081.363467:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.80@tcp1(10.31.131.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513081.363475:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.363476:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.363482:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513081.363482:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513081.363485:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513081.363485:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513081.363486:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513081.363487:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.363491:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513081.363492:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.363497:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388770 00000800:00000200:11.2:1616513081.363497:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513081.363501:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513081.363550:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.363557:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.363559:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.363565:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.363569:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aebb0 00000800:00000200:24.0:1616513081.363573:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.363576:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.381527:0:49591:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513081.381581:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.381590:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513081.381598:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.91@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.381606:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.91@tcp1 of length 224 into portal 26 MB=0x5ffd4d7187640 00000400:00000200:18.0:1616513081.381612:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.91@tcp1 of length 224/224 into md 0xf97c9439 [1] + 0 00000400:00000200:18.0:1616513081.381617:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3a90 00000400:00000200:18.0:1616513081.381619:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.381622:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513081.381626:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3a90 00000800:00000200:18.0:1616513081.381628:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.381631:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.381633:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.381635:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.381698:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664490407488 00010000:00000200:20.0:1616513081.381715:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688664490407488/t0(0) o400->a2a877ff-bfeb-512d-7f48-1276d8ea4f78@10.31.131.91@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513081.381720:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513081.381729:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664490407488, offset 192 00000400:00000200:20.0:1616513081.381736:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.91@tcp1 00000400:00000200:20.0:1616513081.381743:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.91@tcp1 routed destination 00000400:00000200:20.0:1616513081.381758:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.91@tcp1(10.31.131.91@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.381765:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.381771:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.381774:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513081.381776:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.381776:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.381780:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.381785:0:49591:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513081.381786:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000100:00000200:20.0:1616513081.381787:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513081.381795:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.91@tcp1 : PUT - for me 00000400:00000200:28.0:1616513081.381803:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.91@tcp1 of length 224 into portal 12 MB=0x5ffd4d71876c0 00000400:00000200:28.0:1616513081.381809:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.91@tcp1 of length 224/224 into md 0x865bbec1 [1] + 41104 00000400:00000200:28.0:1616513081.381814:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513081.381817:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513081.381824:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.381827:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.381828:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513081.381832:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513081.381849:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.381856:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.381861:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513081.381868:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.381873:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13887f8 00000800:00000200:14.0:1616513081.381877:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.381880:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513081.381889:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664490407616 00010000:00000200:0.0:1616513081.381901:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9bf00 x1688664490407616/t0(0) o400->e7578088-4e34-9b9e-192d-e6c1781c04c8@10.31.131.91@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.381915:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664490407616, offset 192 00000400:00000200:0.0:1616513081.381920:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.91@tcp1 00000400:00000200:0.0:1616513081.381927:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.91@tcp1 routed destination 00000400:00000200:0.0:1616513081.381941:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.91@tcp1(10.31.131.91@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.381947:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.381953:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.381957:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.381959:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.381963:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.381967:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.382020:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.382027:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.382030:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.382038:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.382042:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04110 00000800:00000200:24.0:1616513081.382046:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.382049:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.392602:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513081.392657:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.392665:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513081.392674:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.224@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.392684:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.224@tcp1 of length 224 into portal 26 MB=0x605893badd680 00000400:00000200:18.0:1616513081.392690:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.224@tcp1 of length 224/224 into md 0xf97c9449 [1] + 0 00000400:00000200:18.0:1616513081.392694:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e27f8 00000400:00000200:18.0:1616513081.392697:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.392699:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513081.392703:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e27f8 00000800:00000200:18.0:1616513081.392706:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.392708:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.392709:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.392715:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513081.392718:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513081.392772:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694936830170752 00000800:00000200:14.0:1616513081.392776:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:20.0:1616513081.392783:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1694936830170752/t0(0) o400->d13101ad-69ad-cbb9-d34a-6a7ff3e65226@10.31.140.224@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513081.392785:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513081.392793:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.224@tcp1 : PUT - for me 00000100:00000200:20.0:1616513081.392797:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694936830170752, offset 192 00000400:00000200:14.0:1616513081.392802:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.224@tcp1 of length 224 into portal 12 MB=0x605893badd7c0 00000400:00000200:20.0:1616513081.392803:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.224@tcp1 00000400:00000200:14.0:1616513081.392809:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.224@tcp1 of length 224/224 into md 0xf964d519 [1] + 9856 00000400:00000200:20.0:1616513081.392810:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.224@tcp1 routed destination 00000400:00000200:14.0:1616513081.392814:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.392817:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513081.392825:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.224@tcp1(10.31.140.224@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513081.392829:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.392830:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513081.392832:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513081.392834:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.392835:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513081.392840:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.392842:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.392844:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.392849:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.392854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.392856:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513081.392863:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.392869:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.392871:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513081.392873:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513081.392878:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.392883:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388d48 00000800:00000200:2.0:1616513081.392887:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.392891:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694936830171072 00010000:00000200:26.0:1616513081.392903:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba1200 x1694936830171072/t0(0) o400->c0d750cd-5a22-247a-c5c3-65598e2bee86@10.31.140.224@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.392918:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694936830171072, offset 192 00000400:00000200:26.0:1616513081.392924:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.224@tcp1 00000400:00000200:26.0:1616513081.392931:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.224@tcp1 routed destination 00000400:00000200:26.0:1616513081.392946:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.224@tcp1(10.31.140.224@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.392953:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.392959:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.392967:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.392970:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.392975:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.392980:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.393033:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.393040:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.393042:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.393049:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.393053:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af430 00000800:00000200:12.0:1616513081.393057:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.393060:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.423048:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513081.423066:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513081.423102:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.423112:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.423121:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.87@tcp1 : PUT - for me 00000800:00000200:22.0:1616513081.423124:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513081.423131:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.87@tcp1 of length 224 into portal 26 MB=0x5ffd4c6a7a740 00000800:00000200:22.0:1616513081.423134:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513081.423138:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.87@tcp1 of length 224/224 into md 0xf97c9459 [1] + 0 00000400:00000200:24.0:1616513081.423142:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2770 00000400:00000200:22.0:1616513081.423143:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.87@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.423145:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.423148:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513081.423153:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2770 00000400:00000200:22.0:1616513081.423153:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.87@tcp1 of length 224 into portal 12 MB=0x5ffd4c6a7a7c0 00000800:00000200:24.0:1616513081.423156:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.423156:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.423159:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513081.423159:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.87@tcp1 of length 224/224 into md 0xf964d519 [1] + 10080 00000800:00000200:24.0:1616513081.423161:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513081.423164:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.423167:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513081.423174:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.423177:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.423181:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.423188:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.423224:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664214579008 00010000:00000200:20.0:1616513081.423235:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688664214579008/t0(0) o400->a9ece953-de9c-5e88-4991-a3b0ea3d1b0f@10.31.138.87@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513081.423246:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664214579136 00000100:00000200:20.0:1616513081.423250:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664214579008, offset 192 00000400:00000200:20.0:1616513081.423257:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.87@tcp1 00010000:00000200:26.0:1616513081.423258:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec201f80 x1688664214579136/t0(0) o400->5cfb3f0d-015a-1984-c72c-e9923e33a93e@10.31.138.87@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513081.423264:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.87@tcp1 routed destination 00000100:00000200:26.0:1616513081.423273:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664214579136, offset 192 00000400:00000200:26.0:1616513081.423279:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.87@tcp1 00000400:00000200:20.0:1616513081.423280:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.87@tcp1(10.31.138.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.423287:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513081.423289:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.87@tcp1 routed destination 00000800:00000200:20.0:1616513081.423293:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.423296:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.423298:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.423303:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513081.423305:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.87@tcp1(10.31.138.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513081.423307:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.423310:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513081.423311:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.423317:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.423320:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.423322:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.423327:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.423330:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513081.423361:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.423368:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.423371:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513081.423378:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513081.423383:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388550 00000800:00000200:12.0:1616513081.423386:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513081.423387:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.423390:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513081.423393:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.423396:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.423403:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.423406:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af540 00000800:00000200:12.0:1616513081.423410:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.423413:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.434459:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.434522:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.434532:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513081.434541:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.185@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.434551:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.185@tcp1 of length 224 into portal 26 MB=0x5ffd4ee8dbc40 00000400:00000200:16.0:1616513081.434558:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.185@tcp1 of length 224/224 into md 0xf97c9469 [1] + 0 00000400:00000200:16.0:1616513081.434562:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e37e8 00000400:00000200:16.0:1616513081.434565:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.434568:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513081.434573:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e37e8 00000800:00000200:24.0:1616513081.434576:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.434577:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.434579:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513081.434582:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.434586:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513081.434587:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513081.434595:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.185@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.434604:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.185@tcp1 of length 224 into portal 12 MB=0x5ffd4ee8dbcc0 00000400:00000200:24.0:1616513081.434610:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.185@tcp1 of length 224/224 into md 0x865bbec1 [1] + 41328 00000400:00000200:24.0:1616513081.434615:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.434618:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.434625:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.434628:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.434630:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.434637:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664883969088 00010000:00000200:20.0:1616513081.434648:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688664883969088/t0(0) o400->3936fc55-568f-9642-2009-4fdca63ef1a6@10.31.132.185@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.434662:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664883969088, offset 192 00000400:00000200:20.0:1616513081.434669:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.185@tcp1 00000400:00000200:20.0:1616513081.434676:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.185@tcp1 routed destination 00000400:00000200:20.0:1616513081.434691:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.185@tcp1(10.31.132.185@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513081.434691:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664883969216 00000800:00000200:20.0:1616513081.434698:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.434704:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513081.434704:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5e80 x1688664883969216/t0(0) o400->282d4b6f-404f-26fc-4065-2c3a3ec0763f@10.31.132.185@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.434707:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.434709:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.434714:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.434719:0:49598:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513081.434719:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664883969216, offset 192 00000100:00000200:20.0:1616513081.434723:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513081.434725:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.185@tcp1 00000400:00000200:0.0:1616513081.434732:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.185@tcp1 routed destination 00000400:00000200:0.0:1616513081.434747:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.185@tcp1(10.31.132.185@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.434754:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.434760:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.434762:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.434764:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.434769:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513081.434772:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513081.434773:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513081.434779:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.434782:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513081.434788:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.434794:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13887f8 00000800:00000200:14.0:1616513081.434797:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.434800:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513081.434826:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.434833:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.434836:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513081.434837:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.434843:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.434847:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb057e8 00000800:00000200:28.0:1616513081.434851:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.439931:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513081.439985:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.439994:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513081.440003:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.29@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.440012:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.29@tcp1 of length 224 into portal 26 MB=0x6041a0e0be640 00000400:00000200:18.0:1616513081.440019:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.29@tcp1 of length 224/224 into md 0xf97c9479 [1] + 0 00000400:00000200:18.0:1616513081.440024:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2c38 00000400:00000200:18.0:1616513081.440026:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.440029:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513081.440033:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2c38 00000800:00000200:18.0:1616513081.440036:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.440039:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.440040:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.440044:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.440096:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359811585600 00010000:00000200:20.0:1616513081.440107:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1693359811585600/t0(0) o400->2f48c656-e3dc-f930-b90c-5782fcbfae11@10.31.129.29@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.440121:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693359811585600, offset 192 00000400:00000200:20.0:1616513081.440128:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.29@tcp1 00000400:00000200:20.0:1616513081.440135:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.29@tcp1 routed destination 00000400:00000200:20.0:1616513081.440150:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.29@tcp1(10.31.129.29@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.440157:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.440162:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.440165:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.440167:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:7.2:1616513081.440184:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513081.440185:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.440191:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513081.440238:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.440245:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.440247:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513081.440254:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.440260:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388990 00000800:00000200:14.0:1616513081.440263:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.440266:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.440272:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513081.440274:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.440280:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.29@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.440288:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.29@tcp1 of length 224 into portal 12 MB=0x6041a0e0bea00 00000400:00000200:14.0:1616513081.440294:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.29@tcp1 of length 224/224 into md 0xf964d519 [1] + 10304 00000400:00000200:14.0:1616513081.440301:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.440303:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513081.440310:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.440312:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.440314:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.440376:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359811586560 00010000:00000200:26.0:1616513081.440389:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba1b00 x1693359811586560/t0(0) o400->5bfdfe90-49bf-d439-99d7-598311244fb9@10.31.129.29@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.440403:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693359811586560, offset 192 00000400:00000200:26.0:1616513081.440409:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.29@tcp1 00000400:00000200:26.0:1616513081.440417:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.29@tcp1 routed destination 00000400:00000200:26.0:1616513081.440431:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.29@tcp1(10.31.129.29@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.440451:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.440457:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.440459:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.440462:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.440466:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.440472:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.440525:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.440532:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.440534:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.440541:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.440545:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afb18 00000800:00000200:24.0:1616513081.440549:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.440552:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.456060:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513081.456113:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.456122:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513081.456131:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.92@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.456139:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.92@tcp1 of length 224 into portal 26 MB=0x604c9d5d50f40 00000400:00000200:18.0:1616513081.456147:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.92@tcp1 of length 224/224 into md 0xf97c9489 [1] + 0 00000400:00000200:18.0:1616513081.456152:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2f68 00000400:00000200:18.0:1616513081.456154:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.456157:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513081.456161:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2f68 00000800:00000200:18.0:1616513081.456164:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.456166:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.456167:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513081.456174:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.456177:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.456178:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.456189:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.92@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.456197:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.92@tcp1 of length 224 into portal 12 MB=0x604c9d5d50fc0 00000400:00000200:2.0:1616513081.456204:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.92@tcp1 of length 224/224 into md 0xf964d519 [1] + 10528 00000400:00000200:2.0:1616513081.456209:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.456212:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513081.456218:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.456222:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.456223:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.456224:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694114782711616 00010000:00000200:20.0:1616513081.456235:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1694114782711616/t0(0) o400->7fdc2359-6ef2-4ec0-f9c1-c9cf8a92f417@10.31.131.92@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.456249:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694114782711616, offset 192 00000400:00000200:20.0:1616513081.456256:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.92@tcp1 00000400:00000200:20.0:1616513081.456263:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.92@tcp1 routed destination 00000400:00000200:20.0:1616513081.456278:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.92@tcp1(10.31.131.92@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513081.456283:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694114782711744 00000800:00000200:20.0:1616513081.456285:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.456291:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.456294:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:26.0:1616513081.456295:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21a880 x1694114782711744/t0(0) o400->7959e900-8533-090f-1a8b-57147a93f950@10.31.131.92@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.456296:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.456301:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.456305:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.456308:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513081.456310:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694114782711744, offset 192 00000400:00000200:26.0:1616513081.456317:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.92@tcp1 00000400:00000200:26.0:1616513081.456324:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.92@tcp1 routed destination 00000400:00000200:26.0:1616513081.456340:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.92@tcp1(10.31.131.92@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.456347:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.456353:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.456355:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.456357:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513081.456360:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.456362:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513081.456367:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513081.456368:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513081.456370:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513081.456371:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.456377:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.456382:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388dd0 00000800:00000200:14.0:1616513081.456386:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513081.456421:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.456428:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.456431:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513081.456452:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.456456:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afc28 00000800:00000200:16.0:1616513081.456460:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.456463:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.458117:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513081.458170:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.458179:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.458188:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.119@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.458197:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.119@tcp1 of length 224 into portal 26 MB=0x604aad78f0540 00000400:00000200:2.0:1616513081.458203:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.119@tcp1 of length 224/224 into md 0xf97c94a9 [1] + 0 00000400:00000200:2.0:1616513081.458208:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e3650 00000400:00000200:2.0:1616513081.458211:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.458213:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513081.458217:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e3650 00000800:00000200:2.0:1616513081.458220:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.458223:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513081.458223:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.458230:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.458233:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513081.458233:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513081.458242:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.119@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.458251:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.119@tcp1 of length 224 into portal 12 MB=0x604aad78f05c0 00000400:00000200:22.0:1616513081.458258:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.119@tcp1 of length 224/224 into md 0xf964d519 [1] + 10752 00000400:00000200:22.0:1616513081.458263:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.458265:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513081.458272:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.458275:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.458277:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.458281:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693981667689792 00010000:00000200:20.0:1616513081.458292:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1693981667689792/t0(0) o400->c628082c-791e-1591-e323-cff45729ef4f@10.31.140.119@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.458306:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693981667689792, offset 192 00000400:00000200:20.0:1616513081.458314:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.119@tcp1 00000400:00000200:20.0:1616513081.458321:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.119@tcp1 routed destination 00000100:00000200:26.0:1616513081.458335:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693981667689920 00000400:00000200:20.0:1616513081.458337:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.119@tcp1(10.31.140.119@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.458343:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:26.0:1616513081.458347:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec201200 x1693981667689920/t0(0) o400->971af0f3-e7eb-3c04-b6bd-190a896e40d2@10.31.140.119@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.458349:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.458352:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.458354:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.458359:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:26.0:1616513081.458361:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693981667689920, offset 192 00000800:00000200:7.2:1616513081.458363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.458365:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513081.458368:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.119@tcp1 00000400:00000200:26.0:1616513081.458376:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.119@tcp1 routed destination 00000400:00000200:26.0:1616513081.458391:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.119@tcp1(10.31.140.119@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.458397:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.458403:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.458406:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.458408:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.458412:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513081.458415:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513081.458416:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513081.458422:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.458425:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513081.458426:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.458432:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.458455:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388c38 00000800:00000200:14.0:1616513081.458459:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513081.458471:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.458478:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.458481:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.458487:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.458491:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afcb0 00000800:00000200:24.0:1616513081.458495:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.458498:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.458503:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513081.458506:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.458508:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.458509:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.458512:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.458618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.458627:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.458632:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513081.458636:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513081.458640:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.193@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.458647:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.193@tcp1 of length 224 into portal 26 MB=0x5ffe873c57340 00000400:00000200:22.0:1616513081.458652:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.193@tcp1 of length 224/224 into md 0xf97c94b9 [1] + 0 00000400:00000200:22.0:1616513081.458656:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e2660 00000400:00000200:22.0:1616513081.458658:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.458661:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513081.458664:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e2660 00000800:00000200:22.0:1616513081.458666:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.458668:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.458670:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.458677:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748723368768 00010000:00000200:20.0:1616513081.458682:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688748723368768/t0(0) o400->62a837ef-d3f2-7725-21e0-3ffadd75d910@10.31.141.193@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.458694:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688748723368768, offset 192 00000400:00000200:20.0:1616513081.458699:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.193@tcp1 00000800:00000200:7.2:1616513081.458704:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513081.458705:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.193@tcp1 routed destination 00000800:00000200:2.0:1616513081.458712:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513081.458718:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.193@tcp1(10.31.141.193@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513081.458718:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513081.458721:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513081.458724:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.458727:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.193@tcp1 : PUT - for me 00000800:00000200:20.0:1616513081.458729:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.458732:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.458733:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513081.458733:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.193@tcp1 of length 224 into portal 12 MB=0x5ffe873c57700 00000800:00000200:20.0:1616513081.458736:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.458739:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.193@tcp1 of length 224/224 into md 0xf964d519 [1] + 10976 00000800:00000200:7.2:1616513081.458740:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:20.0:1616513081.458741:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513081.458743:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.458745:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513081.458749:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513081.458751:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513081.458753:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:18.0:1616513081.458754:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513081.458754:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:18.0:1616513081.458757:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513081.458759:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513081.458763:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:26.0:1616513081.458764:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748723369728 00000400:00000200:18.0:1616513081.458768:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13882a8 00010000:00000200:26.0:1616513081.458770:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21de80 x1688748723369728/t0(0) o400->b6bc5c2d-020a-85e8-0ce1-e2a9fccd26e7@10.31.141.193@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513081.458771:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.458781:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688748723369728, offset 192 00000400:00000200:26.0:1616513081.458786:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.193@tcp1 00000400:00000200:26.0:1616513081.458792:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.193@tcp1 routed destination 00000400:00000200:26.0:1616513081.458805:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.193@tcp1(10.31.141.193@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.458811:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.458815:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.458818:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.458821:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.458825:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.458829:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.458881:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.458888:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.458891:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513081.458892:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.458897:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.458901:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeaa0 00000800:00000200:28.0:1616513081.458905:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.462341:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513081.462394:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.462403:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513081.462412:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.11@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.462421:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.11@tcp1 of length 224 into portal 12 MB=0x60393b6c69540 00000400:00000200:14.0:1616513081.462428:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.11@tcp1 of length 224/224 into md 0xf964d519 [1] + 11200 00000400:00000200:14.0:1616513081.462443:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.462446:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513081.462449:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.462453:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.462456:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.462458:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.462499:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513081.462519:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692782821807424 00010000:00000200:26.0:1616513081.462531:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba2400 x1692782821807424/t0(0) o400->66953bb2-4e67-c9a9-c6e9-03decddb7ab8@10.31.140.11@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.462545:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692782821807424, offset 192 00000400:00000200:26.0:1616513081.462551:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.11@tcp1 00000800:00000200:12.0:1616513081.462551:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513081.462559:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.11@tcp1 routed destination 00000800:00000200:12.0:1616513081.462560:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.462569:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.11@tcp1 : PUT - for me 00000400:00000200:26.0:1616513081.462574:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.11@tcp1(10.31.140.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513081.462578:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.11@tcp1 of length 224 into portal 26 MB=0x60393b6c69200 00000800:00000200:26.0:1616513081.462581:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513081.462585:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.11@tcp1 of length 224/224 into md 0xf97c94c9 [1] + 0 00000800:00000200:26.0:1616513081.462586:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.462589:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513081.462590:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8e5f7e36d8 00000800:00000200:26.0:1616513081.462591:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513081.462593:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513081.462596:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513081.462597:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:7.2:1616513081.462600:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513081.462601:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8e5f7e36d8 00000800:00000200:12.0:1616513081.462604:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.462606:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.462607:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.462610:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513081.462614:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.462620:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.462623:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513081.462630:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.462634:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae5d8 00000800:00000200:2.0:1616513081.462638:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.462640:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.462664:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692782821806592 00010000:00000200:20.0:1616513081.462675:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1692782821806592/t0(0) o400->ff297ec4-65bf-af2f-e9cf-a466c3536794@10.31.140.11@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.462690:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692782821806592, offset 192 00000400:00000200:20.0:1616513081.462696:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.11@tcp1 00000400:00000200:20.0:1616513081.462703:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.11@tcp1 routed destination 00000400:00000200:20.0:1616513081.462718:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.11@tcp1(10.31.140.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.462729:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.462734:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.462737:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.462739:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.462744:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.462748:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.462751:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513081.462800:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.462807:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.462809:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.462816:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.462821:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388110 00000800:00000200:24.0:1616513081.462825:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.462828:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.466747:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.466799:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.466809:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513081.466818:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.23@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.466827:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.23@tcp1 of length 224 into portal 26 MB=0x5ffd3f1f37dc0 00000400:00000200:16.0:1616513081.466834:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.23@tcp1 of length 224/224 into md 0xf97c94f9 [1] + 0 00000400:00000200:16.0:1616513081.466839:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636f68 00000400:00000200:16.0:1616513081.466841:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.466844:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513081.466848:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636f68 00000800:00000200:16.0:1616513081.466851:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.466853:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.466854:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.466857:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.466887:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513081.466911:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660646002112 00010000:00000200:20.0:1616513081.466922:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688660646002112/t0(0) o400->2585ccbd-3e6b-31ba-3edc-2d458ae87cb0@10.31.129.23@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.466936:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660646002112, offset 192 00000800:00000200:24.0:1616513081.466940:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513081.466943:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.23@tcp1 00000800:00000200:24.0:1616513081.466949:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513081.466951:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.23@tcp1 routed destination 00000400:00000200:24.0:1616513081.466961:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.23@tcp1 : PUT - for me 00000400:00000200:20.0:1616513081.466966:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.23@tcp1(10.31.129.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513081.466970:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.23@tcp1 of length 224 into portal 12 MB=0x5ffd3f1f38180 00000800:00000200:20.0:1616513081.466972:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513081.466976:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.23@tcp1 of length 224/224 into md 0x865bbec1 [1] + 41552 00000800:00000200:20.0:1616513081.466978:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513081.466981:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513081.466981:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.466983:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513081.466984:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513081.466988:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513081.466991:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513081.466992:0:49602:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513081.466994:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.466994:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:20.0:1616513081.466994:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513081.466999:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.467054:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660646003072 00000800:00000200:22.0:1616513081.467056:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.467063:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.467066:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:0.0:1616513081.467067:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5a00 x1688660646003072/t0(0) o400->504d2966-7296-2652-d73f-e77e554b1f0c@10.31.129.23@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:22.0:1616513081.467072:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.467078:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388c38 00000800:00000200:22.0:1616513081.467081:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513081.467082:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660646003072, offset 192 00000800:00000200:22.0:1616513081.467084:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513081.467088:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.23@tcp1 00000400:00000200:0.0:1616513081.467096:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.23@tcp1 routed destination 00000400:00000200:0.0:1616513081.467110:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.23@tcp1(10.31.129.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.467117:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.467123:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.467125:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.467128:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.467132:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.467136:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.467146:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.467151:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.467154:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513081.467156:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513081.467160:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.467164:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05e48 00000800:00000200:12.0:1616513081.467168:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.477104:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513081.477158:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.477167:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513081.477176:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.83@tcp1 : PUT - for me 00000400:00000200:28.0:1616513081.477185:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.83@tcp1 of length 224 into portal 26 MB=0x5ffdc4bfda840 00000400:00000200:28.0:1616513081.477191:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.83@tcp1 of length 224/224 into md 0xf97c9509 [1] + 0 00000400:00000200:28.0:1616513081.477196:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c6365d8 00000400:00000200:28.0:1616513081.477199:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513081.477204:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513081.477208:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6365d8 00000800:00000200:28.0:1616513081.477211:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.477212:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.477214:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513081.477219:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513081.477222:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.477223:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.477231:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.83@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.477240:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.83@tcp1 of length 224 into portal 12 MB=0x5ffdc4bfda8c0 00000400:00000200:24.0:1616513081.477248:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.83@tcp1 of length 224/224 into md 0x865bbec1 [1] + 41776 00000400:00000200:24.0:1616513081.477253:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.477255:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.477279:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.477282:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.477283:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.477288:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696516356160 00010000:00000200:20.0:1616513081.477298:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688696516356160/t0(0) o400->d367de93-64e2-e0b8-dba8-6ced5ca780fc@10.31.141.83@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.477312:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696516356160, offset 192 00000400:00000200:20.0:1616513081.477317:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.83@tcp1 00000400:00000200:20.0:1616513081.477324:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.83@tcp1 routed destination 00000100:00000200:0.0:1616513081.477332:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696516356288 00000400:00000200:20.0:1616513081.477338:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.83@tcp1(10.31.141.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.477344:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:0.0:1616513081.477344:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d7080 x1688696516356288/t0(0) o400->ade67784-7431-01cb-c8b6-97a4d3d7fb17@10.31.141.83@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.477349:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.477352:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.477355:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.477360:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.477366:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.477366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513081.477370:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696516356288, offset 192 00000400:00000200:0.0:1616513081.477376:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.83@tcp1 00000400:00000200:0.0:1616513081.477383:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.83@tcp1 routed destination 00000400:00000200:0.0:1616513081.477398:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.83@tcp1(10.31.141.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.477404:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.477410:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.477413:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.477415:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513081.477419:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513081.477420:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.477424:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513081.477426:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.477429:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513081.477433:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.477444:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.477445:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.477448:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.477454:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.477457:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000400:00000200:2.0:1616513081.477458:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513081.477461:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513081.477464:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388990 00000800:00000200:2.0:1616513081.477468:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.477471:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.482067:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.482122:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.482131:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.482140:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.182@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.482149:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.182@tcp1 of length 224 into portal 12 MB=0x602ad3f798a00 00000400:00000200:24.0:1616513081.482155:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.182@tcp1 of length 224/224 into md 0x865bbec1 [1] + 42000 00000400:00000200:24.0:1616513081.482161:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.482163:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.482171:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.482174:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.482175:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.482179:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.482207:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513081.482237:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691792977791488 00010000:00000200:0.0:1616513081.482249:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd705e80 x1691792977791488/t0(0) o400->853f7c5c-c89c-6ed1-73b0-1a463558d006@10.31.141.182@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:22.0:1616513081.482260:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513081.482264:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691792977791488, offset 192 00000800:00000200:22.0:1616513081.482269:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513081.482270:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.182@tcp1 00000400:00000200:0.0:1616513081.482278:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.182@tcp1 routed destination 00000400:00000200:22.0:1616513081.482280:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.182@tcp1 : PUT - for me 00000400:00000200:0.0:1616513081.482309:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.182@tcp1(10.31.141.182@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.482315:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.482318:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.182@tcp1 of length 224 into portal 26 MB=0x602ad3f7987c0 00000800:00000200:0.0:1616513081.482321:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513081.482322:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513081.482323:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513081.482324:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.182@tcp1 of length 224/224 into md 0xf97c9519 [1] + 0 00000800:00000200:0.0:1616513081.482326:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513081.482328:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636660 00000800:00000200:0.0:1616513081.482330:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.482331:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.482333:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:7.2:1616513081.482334:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513081.482337:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636660 00000800:00000200:22.0:1616513081.482340:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513081.482342:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513081.482344:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.482346:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.482350:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.482352:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.482357:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.482360:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:22.0:1616513081.482362:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.482365:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.482400:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691792977790912 00010000:00000200:20.0:1616513081.482410:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1691792977790912/t0(0) o400->836bb962-8074-bb74-2690-bfdbd92c03b2@10.31.141.182@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.482423:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691792977790912, offset 192 00000400:00000200:20.0:1616513081.482429:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.182@tcp1 00000400:00000200:20.0:1616513081.482446:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.182@tcp1 routed destination 00000400:00000200:20.0:1616513081.482460:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.182@tcp1(10.31.141.182@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.482466:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.482471:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.482474:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.482476:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.482480:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.482484:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.482487:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513081.482536:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.482543:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.482546:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513081.482548:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.482552:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.482558:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388550 00000800:00000200:28.0:1616513081.482561:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.492898:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513081.492952:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.492961:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513081.492970:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.51@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.492979:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.51@tcp1 of length 224 into portal 26 MB=0x6048d7dea6140 00000400:00000200:14.0:1616513081.492988:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.51@tcp1 of length 224/224 into md 0xf97c9529 [1] + 0 00000400:00000200:14.0:1616513081.492993:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636a18 00000400:00000200:14.0:1616513081.492995:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.492998:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513081.493002:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636a18 00000800:00000200:14.0:1616513081.493005:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.493006:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.493008:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.493012:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.493015:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.493016:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.493025:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.51@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.493034:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.51@tcp1 of length 224 into portal 12 MB=0x6048d7dea6400 00000400:00000200:22.0:1616513081.493041:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.51@tcp1 of length 224/224 into md 0xf964d519 [1] + 11424 00000400:00000200:22.0:1616513081.493046:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.493048:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513081.493055:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.493058:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.493060:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.493066:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855609676096 00010000:00000200:20.0:1616513081.493077:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1693855609676096/t0(0) o400->02e4bde1-32df-d0ac-1084-023375c29dbf@10.31.141.51@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.493091:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855609676096, offset 192 00000400:00000200:20.0:1616513081.493097:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.51@tcp1 00000400:00000200:20.0:1616513081.493105:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.51@tcp1 routed destination 00000400:00000200:20.0:1616513081.493120:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.51@tcp1(10.31.141.51@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513081.493122:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855609676800 00000800:00000200:20.0:1616513081.493126:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.493132:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:26.0:1616513081.493134:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec202d00 x1693855609676800/t0(0) o400->84a3a187-f7a5-82b4-91b7-788baef30d57@10.31.141.51@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.493135:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.493137:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.493142:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.493146:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:26.0:1616513081.493148:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855609676800, offset 192 00000100:00000200:20.0:1616513081.493149:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513081.493154:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.51@tcp1 00000400:00000200:26.0:1616513081.493162:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.51@tcp1 routed destination 00000400:00000200:26.0:1616513081.493176:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.51@tcp1(10.31.141.51@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.493183:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.493188:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.493191:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.493193:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.493198:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513081.493202:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513081.493205:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513081.493209:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.493212:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513081.493213:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513081.493219:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.493224:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388c38 00000800:00000200:2.0:1616513081.493228:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513081.493260:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.493267:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.493270:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513081.493277:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.493281:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae000 00000800:00000200:16.0:1616513081.493285:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.493288:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.503078:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513081.503087:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513081.503134:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.503143:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.503152:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.44@tcp1 : PUT - for me 00000800:00000200:22.0:1616513081.503153:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513081.503161:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.44@tcp1 of length 224 into portal 26 MB=0x60194a9dc2cc0 00000800:00000200:22.0:1616513081.503162:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513081.503167:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.44@tcp1 of length 224/224 into md 0xf97c9539 [1] + 0 00000400:00000200:22.0:1616513081.503171:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.44@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.503172:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637a08 00000400:00000200:12.0:1616513081.503175:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.503178:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513081.503180:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.44@tcp1 of length 224 into portal 12 MB=0x60194a9dc2d40 00000400:00000200:12.0:1616513081.503182:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637a08 00000800:00000200:12.0:1616513081.503185:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513081.503187:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.44@tcp1 of length 224/224 into md 0xf964d519 [1] + 11648 00000800:00000200:28.0:1616513081.503188:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.503188:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.503192:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513081.503195:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.503197:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513081.503204:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.503207:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.503210:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.503214:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.503252:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690587876830400 00010000:00000200:20.0:1616513081.503263:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1690587876830400/t0(0) o400->81fe08f1-5182-ca88-810d-ad01f5a118e9@10.31.141.44@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513081.503276:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690587876830528 00000100:00000200:20.0:1616513081.503280:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690587876830400, offset 192 00000400:00000200:20.0:1616513081.503286:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.44@tcp1 00010000:00000200:26.0:1616513081.503289:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec202880 x1690587876830528/t0(0) o400->eed37678-2977-d139-191d-a3474c128ec4@10.31.141.44@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513081.503293:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.44@tcp1 routed destination 00000100:00000200:26.0:1616513081.503303:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690587876830528, offset 192 00000400:00000200:20.0:1616513081.503308:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.44@tcp1(10.31.141.44@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513081.503311:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.44@tcp1 00000800:00000200:20.0:1616513081.503315:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513081.503318:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.44@tcp1 routed destination 00000800:00000200:20.0:1616513081.503321:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.503324:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.503326:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.503331:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513081.503333:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.44@tcp1(10.31.141.44@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513081.503335:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.503338:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513081.503340:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.503345:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.503348:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.503350:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.503355:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.503359:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513081.503387:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.503394:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.503397:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513081.503404:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.503414:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388c38 00000800:00000200:14.0:1616513081.503418:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.503420:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513081.503447:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.503454:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.503457:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513081.503463:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.503470:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af760 00000800:00000200:16.0:1616513081.503474:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.503477:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.511844:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513081.511896:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.511906:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513081.511915:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.183@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.511923:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.183@tcp1 of length 224 into portal 26 MB=0x602ad64a21e40 00000400:00000200:18.0:1616513081.511930:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.183@tcp1 of length 224/224 into md 0xf97c9569 [1] + 0 00000400:00000200:18.0:1616513081.511935:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636e58 00000400:00000200:18.0:1616513081.511937:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.511940:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:11.2:1616513081.511942:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513081.511944:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636e58 00000800:00000200:18.0:1616513081.511947:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.511949:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.511951:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.511955:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.512007:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691793601207872 00000800:00000200:28.0:1616513081.512012:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:20.0:1616513081.512018:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1691793601207872/t0(0) o400->64b93aee-fef4-17dc-1353-0a3f43e72e7e@10.31.141.183@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513081.512021:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513081.512030:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.183@tcp1 : PUT - for me 00000100:00000200:20.0:1616513081.512033:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691793601207872, offset 192 00000400:00000200:28.0:1616513081.512038:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.183@tcp1 of length 224 into portal 12 MB=0x602ad64a21f40 00000400:00000200:20.0:1616513081.512039:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.183@tcp1 00000400:00000200:28.0:1616513081.512044:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.183@tcp1 of length 224/224 into md 0x865bbec1 [1] + 42224 00000400:00000200:20.0:1616513081.512046:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.183@tcp1 routed destination 00000400:00000200:28.0:1616513081.512049:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513081.512052:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513081.512059:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513081.512061:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.183@tcp1(10.31.141.183@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513081.512062:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513081.512064:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.512066:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:20.0:1616513081.512068:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.512073:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.512076:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.512078:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.512083:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.512087:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.512090:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513081.512125:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691793601208128 00010000:00000200:0.0:1616513081.512138:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b98900 x1691793601208128/t0(0) o400->e3d21848-3542-b102-d7af-d7f128ec5931@10.31.141.183@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513081.512139:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.512146:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.512149:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1616513081.512152:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691793601208128, offset 192 00000400:00000200:14.0:1616513081.512155:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:0.0:1616513081.512158:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.183@tcp1 00000400:00000200:14.0:1616513081.512163:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388c38 00000400:00000200:0.0:1616513081.512165:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.183@tcp1 routed destination 00000800:00000200:14.0:1616513081.512167:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.512170:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513081.512180:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.183@tcp1(10.31.141.183@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.512191:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.512197:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.512200:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.512202:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.512207:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.512211:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.512263:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.512270:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.512273:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513081.512280:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.512284:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05078 00000800:00000200:16.0:1616513081.512288:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.512291:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.513128:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513081.513139:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.513144:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513081.513147:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.513151:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.60@tcp1 : PUT - for me 00000400:00000200:28.0:1616513081.513159:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.60@tcp1 of length 224 into portal 26 MB=0x5ffd47fec3080 00000400:00000200:28.0:1616513081.513164:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.60@tcp1 of length 224/224 into md 0xf97c9579 [1] + 0 00000400:00000200:28.0:1616513081.513168:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636220 00000400:00000200:28.0:1616513081.513170:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513081.513173:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513081.513177:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636220 00000800:00000200:28.0:1616513081.513182:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.513184:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.513185:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.513190:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663027888256 00010000:00000200:20.0:1616513081.513195:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688663027888256/t0(0) o400->f69540ec-9f8e-a20d-b219-a70953732131@10.31.138.60@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.513207:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663027888256, offset 192 00000400:00000200:20.0:1616513081.513212:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.60@tcp1 00000400:00000200:20.0:1616513081.513218:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.60@tcp1 routed destination 00000400:00000200:20.0:1616513081.513233:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.60@tcp1(10.31.138.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.513239:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.513244:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.513247:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.513249:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.513252:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.513255:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.513256:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513081.513308:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.513315:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.513317:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.513324:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.513329:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79add0 00000800:00000200:22.0:1616513081.513333:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.513336:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513081.513369:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513081.513388:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.513399:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513081.513408:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.60@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.513417:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.60@tcp1 of length 224 into portal 12 MB=0x5ffd47fec3140 00000800:00000200:14.0:1616513081.513424:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513081.513425:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.60@tcp1 of length 224/224 into md 0xf964d519 [1] + 11872 00000400:00000200:18.0:1616513081.513430:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.513448:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513081.513455:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.513458:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.513460:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.513530:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663027888448 00010000:00000200:26.0:1616513081.513542:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bff080 x1688663027888448/t0(0) o400->f50c7dde-10b7-7983-dadf-055f0dbf0e6c@10.31.138.60@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.513555:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663027888448, offset 192 00000400:00000200:26.0:1616513081.513561:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.60@tcp1 00000400:00000200:26.0:1616513081.513568:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.60@tcp1 routed destination 00000400:00000200:26.0:1616513081.513581:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.60@tcp1(10.31.138.60@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.513587:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.513593:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.513595:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.513597:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.513602:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.513607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.513660:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.513666:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.513669:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513081.513676:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.513680:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afd38 00000800:00000200:16.0:1616513081.513684:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.513687:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.547282:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.547339:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.547349:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.547358:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.30@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.547367:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.30@tcp1 of length 224 into portal 26 MB=0x5ffe688096f40 00000400:00000200:22.0:1616513081.547374:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.30@tcp1 of length 224/224 into md 0xf97c9589 [1] + 0 00000400:00000200:22.0:1616513081.547379:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c6367f8 00000400:00000200:22.0:1616513081.547382:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.547385:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513081.547389:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6367f8 00000800:00000200:22.0:1616513081.547393:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.547396:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.547396:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513081.547404:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.547407:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.547408:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513081.547416:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.30@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.547425:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.30@tcp1 of length 224 into portal 12 MB=0x5ffe688097200 00000400:00000200:14.0:1616513081.547432:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.30@tcp1 of length 224/224 into md 0xf964d519 [1] + 12096 00000400:00000200:14.0:1616513081.547452:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.547455:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513081.547464:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.547466:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688740473433920 00000800:00000200:14.0:1616513081.547476:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:20.0:1616513081.547477:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688740473433920/t0(0) o400->0adcd26f-778e-ddcc-882a-7cbe4fa68f14@10.31.142.30@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513081.547478:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.547492:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688740473433920, offset 192 00000400:00000200:20.0:1616513081.547498:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.30@tcp1 00000400:00000200:20.0:1616513081.547506:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.30@tcp1 routed destination 00000400:00000200:20.0:1616513081.547521:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.30@tcp1(10.31.142.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.547527:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:26.0:1616513081.547530:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688740473434624 00000800:00000200:20.0:1616513081.547534:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.547536:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.547539:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:26.0:1616513081.547542:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba0000 x1688740473434624/t0(0) o400->e94a2f45-5f67-1d8a-0712-0ffbf1e636d0@10.31.142.30@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.547544:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.547550:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.547551:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513081.547555:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688740473434624, offset 192 00000400:00000200:26.0:1616513081.547561:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.30@tcp1 00000400:00000200:26.0:1616513081.547568:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.30@tcp1 routed destination 00000400:00000200:26.0:1616513081.547582:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.30@tcp1(10.31.142.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.547588:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.547593:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.547597:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.547599:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.547604:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513081.547608:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513081.547610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513081.547615:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.547618:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513081.547619:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513081.547625:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513081.547631:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a6e8 00000800:00000200:18.0:1616513081.547634:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513081.547665:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.547674:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.547677:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.547683:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.547687:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeee0 00000800:00000200:12.0:1616513081.547692:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.547695:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.548670:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.548724:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.548733:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513081.548742:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.207@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.548751:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.207@tcp1 of length 224 into portal 26 MB=0x5ffdb37591e40 00000400:00000200:16.0:1616513081.548758:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.207@tcp1 of length 224/224 into md 0xf97c95a9 [1] + 0 00000400:00000200:16.0:1616513081.548763:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637650 00000400:00000200:16.0:1616513081.548765:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.548768:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513081.548772:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637650 00000800:00000200:16.0:1616513081.548776:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.548778:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.548780:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.548782:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.548788:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513081.548791:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.548795:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.207@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.548800:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.207@tcp1 of length 224 into portal 12 MB=0x5ffdb37591ec0 00000400:00000200:16.0:1616513081.548805:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.207@tcp1 of length 224/224 into md 0x865bbec1 [1] + 42448 00000400:00000200:16.0:1616513081.548812:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.548815:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513081.548820:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.548822:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.548824:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.548842:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691875061312 00010000:00000200:20.0:1616513081.548853:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688691875061312/t0(0) o400->b8c42e2c-4c92-6a24-96ff-efa2f5bdfd9b@10.31.140.207@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.548867:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688691875061312, offset 192 00000400:00000200:20.0:1616513081.548874:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.207@tcp1 00000400:00000200:20.0:1616513081.548882:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.207@tcp1 routed destination 00000100:00000200:0.0:1616513081.548885:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691875061440 00000400:00000200:20.0:1616513081.548898:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.207@tcp1(10.31.140.207@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513081.548898:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5a00 x1688691875061440/t0(0) o400->705a701e-f4e8-576e-4462-78cf33f18e7c@10.31.140.207@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.548905:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.548910:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513081.548912:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688691875061440, offset 192 00000800:00000200:20.0:1616513081.548915:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.548917:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513081.548919:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.207@tcp1 00000800:00000200:20.0:1616513081.548922:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513081.548926:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.207@tcp1 routed destination 00000800:00000200:7.2:1616513081.548927:0:49611:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.548929:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513081.548942:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.207@tcp1(10.31.140.207@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.548948:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.548954:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.548957:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.548959:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.548963:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.548969:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513081.548980:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.548987:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.548990:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513081.548996:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.549002:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b100 00000800:00000200:2.0:1616513081.549005:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.549008:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513081.549022:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.549029:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.549032:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513081.549032:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.549038:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.549042:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05f58 00000800:00000200:28.0:1616513081.549046:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.550019:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.550075:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.550085:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513081.550091:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513081.550094:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.29@tcp1 : PUT - for me 00000800:00000200:18.0:1616513081.550101:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.550103:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.29@tcp1 of length 224 into portal 26 MB=0x60182c2a9bb80 00000800:00000200:2.0:1616513081.550103:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.550110:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.29@tcp1 of length 224/224 into md 0xf97c95c9 [1] + 0 00000400:00000200:18.0:1616513081.550110:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.29@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.550115:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636ff0 00000400:00000200:22.0:1616513081.550119:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513081.550119:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.29@tcp1 of length 224 into portal 12 MB=0x60182c2a9bc00 00000100:00000200:22.0:1616513081.550122:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513081.550126:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636ff0 00000800:00000200:22.0:1616513081.550129:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513081.550131:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.29@tcp1 of length 224/224 into md 0xf964d519 [1] + 12320 00000800:00000200:22.0:1616513081.550132:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.550134:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513081.550136:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.550139:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513081.550146:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.550149:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.550150:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.550189:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510983543680 00010000:00000200:20.0:1616513081.550200:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1690510983543680/t0(0) o400->a4d14125-01ea-d0ef-6546-01dacb509f17@10.31.141.29@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.550214:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510983543680, offset 192 00000400:00000200:20.0:1616513081.550221:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.29@tcp1 00000100:00000200:26.0:1616513081.550228:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510983543808 00000400:00000200:20.0:1616513081.550228:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.29@tcp1 routed destination 00010000:00000200:26.0:1616513081.550240:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf8900 x1690510983543808/t0(0) o400->0f700d49-c750-fa22-2e0f-02ab7bfd49ce@10.31.141.29@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513081.550243:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.29@tcp1(10.31.141.29@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.550250:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:26.0:1616513081.550255:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510983543808, offset 192 00000800:00000200:20.0:1616513081.550256:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.550259:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:26.0:1616513081.550261:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.29@tcp1 00000800:00000200:20.0:1616513081.550261:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.550266:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513081.550269:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.29@tcp1 routed destination 00000800:00000200:7.2:1616513081.550271:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.550272:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513081.550284:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.29@tcp1(10.31.141.29@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.550290:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.550301:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.550304:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.550306:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.550311:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.550316:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513081.550340:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.550348:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.550350:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513081.550350:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.550357:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.550363:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a990 00000800:00000200:14.0:1616513081.550366:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513081.550372:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.550378:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.550381:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.550388:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.550392:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af8f8 00000800:00000200:24.0:1616513081.550395:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.550398:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:2.0:1616513081.578635:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926112, portal 4 00000100:00000200:2.0:1616513081.578641:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926112, offset 0 00000400:00000200:2.0:1616513081.578648:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:2.0:1616513081.578654:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:2.0:1616513081.578667:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513081.578676:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:2.0:1616513081.578683:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513081.578686:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:2.0:1616513081.578688:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:2.0:1616513081.578694:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513081.578741:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513081.578761:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513081.578768:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513081.578771:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513081.578778:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513081.578785:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dec204800 x1693067725926112/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513081.578798:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed320 00000800:00000200:27.0:1616513081.578803:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513081.578806:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513081.578916:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513081.578926:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513081.578932:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513081.578936:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513081.578940:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513081.578948:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626ee0 00000400:00000200:27.0:1616513081.578956:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166bfd [1] + 192 00000400:00000200:27.0:1616513081.578961:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513081.578968:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dec204800 x1693067725926112/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513081.578982:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513081.578985:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513081.578987:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:2.0:1616513081.578991:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dec204800 x1693067725926112/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513081.579004:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dec204800 x1693067725926112/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513088 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513081.579015:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eeded0 00000800:00000200:7.2:1616513081.588292:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.588346:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.588355:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.588365:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.65@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.588374:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.65@tcp1 of length 224 into portal 26 MB=0x60215d4158ec0 00000400:00000200:22.0:1616513081.588381:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.65@tcp1 of length 224/224 into md 0xf97c95d9 [1] + 0 00000400:00000200:22.0:1616513081.588386:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637320 00000400:00000200:22.0:1616513081.588388:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.588391:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513081.588396:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637320 00000800:00000200:22.0:1616513081.588399:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.588399:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.588401:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.588407:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.588410:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.588410:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513081.588419:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.65@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.588428:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.65@tcp1 of length 224 into portal 12 MB=0x60215d4158f40 00000400:00000200:18.0:1616513081.588448:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.65@tcp1 of length 224/224 into md 0xf964d519 [1] + 12544 00000400:00000200:18.0:1616513081.588453:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.588456:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:20.0:1616513081.588463:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691142636015296 00000800:00000200:18.0:1616513081.588466:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.588469:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.588471:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:20.0:1616513081.588475:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1691142636015296/t0(0) o400->41bff222-9fe2-efd8-1461-d4f21f0c333e@10.31.140.65@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.588489:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691142636015296, offset 192 00000400:00000200:20.0:1616513081.588496:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.65@tcp1 00000400:00000200:20.0:1616513081.588503:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.65@tcp1 routed destination 00000400:00000200:20.0:1616513081.588526:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.65@tcp1(10.31.140.65@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.588533:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.588538:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.588541:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.588543:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.588548:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:26.0:1616513081.588553:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691142636015424 00000800:00000200:7.2:1616513081.588553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.588555:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:26.0:1616513081.588567:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfde80 x1691142636015424/t0(0) o400->ca6c103b-f48d-54b8-f87d-28272d8a19b7@10.31.140.65@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.588581:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691142636015424, offset 192 00000400:00000200:26.0:1616513081.588587:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.65@tcp1 00000400:00000200:26.0:1616513081.588595:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.65@tcp1 routed destination 00000400:00000200:26.0:1616513081.588610:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.65@tcp1(10.31.140.65@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.588616:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513081.588620:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.588622:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.588625:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.588627:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513081.588627:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.588630:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513081.588631:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513081.588632:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513081.588637:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513081.588640:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513081.588642:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bba0 00000800:00000200:14.0:1616513081.588645:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513081.588696:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.588702:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.588705:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.588712:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.588716:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae7f8 00000800:00000200:12.0:1616513081.588720:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.588723:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.595259:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513081.595273:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.595282:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.595291:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.216@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.595300:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.216@tcp1 of length 224 into portal 26 MB=0x600235a1a6c80 00000400:00000200:2.0:1616513081.595307:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.216@tcp1 of length 224/224 into md 0xf97c95e9 [1] + 0 00000400:00000200:2.0:1616513081.595311:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636770 00000400:00000200:2.0:1616513081.595314:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.595317:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513081.595321:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636770 00000800:00000200:2.0:1616513081.595324:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.595327:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.595327:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.595331:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.595386:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689001695800448 00010000:00000200:20.0:1616513081.595397:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1689001695800448/t0(0) o400->8a2e9cbb-7921-78cc-1bf6-bb553907f02e@10.31.128.216@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.595411:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689001695800448, offset 192 00000400:00000200:20.0:1616513081.595417:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.216@tcp1 00000400:00000200:20.0:1616513081.595425:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.216@tcp1 routed destination 00000400:00000200:20.0:1616513081.595452:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.216@tcp1(10.31.128.216@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.595459:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.595465:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.595468:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.595470:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.595475:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.595478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.595482:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513081.595531:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.595538:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.595541:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.595547:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.595553:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b8f8 00000800:00000200:22.0:1616513081.595556:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.595559:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.595693:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.595748:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.595757:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.595766:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.216@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.595775:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.216@tcp1 of length 224 into portal 12 MB=0x600235a1a7180 00000400:00000200:24.0:1616513081.595782:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.216@tcp1 of length 224/224 into md 0x865bbec1 [1] + 42672 00000400:00000200:24.0:1616513081.595787:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.595789:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.595796:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.595799:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.595801:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.595806:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.595862:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689001695801728 00010000:00000200:0.0:1616513081.595874:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd701200 x1689001695801728/t0(0) o400->de6ad1d8-34f5-c73f-f3b7-8cbe86fe4a2f@10.31.128.216@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.595888:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689001695801728, offset 192 00000400:00000200:0.0:1616513081.595895:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.216@tcp1 00000400:00000200:0.0:1616513081.595902:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.216@tcp1 routed destination 00000400:00000200:0.0:1616513081.595917:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.216@tcp1(10.31.128.216@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.595924:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.595929:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.595932:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.595934:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.595939:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.595943:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.595955:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.595961:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.595964:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.595971:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.595975:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05c28 00000800:00000200:12.0:1616513081.595979:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.595982:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.606821:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513081.606821:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513081.606876:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513081.606877:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513081.606886:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:18.0:1616513081.606887:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513081.606890:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513081.606895:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.182@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.606896:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.182@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.606905:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.182@tcp1 of length 224 into portal 12 MB=0x5ffd40cb17740 00000400:00000200:16.0:1616513081.606905:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.182@tcp1 of length 224 into portal 26 MB=0x5ffd40cb17600 00000400:00000200:16.0:1616513081.606911:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.182@tcp1 of length 224/224 into md 0xf97c9609 [1] + 0 00000400:00000200:16.0:1616513081.606916:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c6366e8 00000400:00000200:16.0:1616513081.606919:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513081.606931:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.182@tcp1 of length 224/224 into md 0xf964d519 [1] + 12768 00000100:00000200:16.0:1616513081.606935:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513081.606936:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513081.606936:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513081.606940:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6366e8 00000100:00000200:18.0:1616513081.606942:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513081.606943:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.606946:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.606948:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513081.606949:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.606951:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.606953:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.607008:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661094659584 00010000:00000200:20.0:1616513081.607019:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688661094659584/t0(0) o400->07acad4b-ffd1-014c-7da0-7b45300fdd5c@10.31.131.182@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513081.607026:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661094659904 00000100:00000200:20.0:1616513081.607034:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661094659584, offset 192 00010000:00000200:26.0:1616513081.607038:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfbf00 x1688661094659904/t0(0) o400->2d73376b-6ec8-dc9b-4b52-c579df8a488f@10.31.131.182@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513081.607040:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.182@tcp1 00000400:00000200:20.0:1616513081.607047:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.182@tcp1 routed destination 00000400:00000200:20.0:1616513081.607063:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.182@tcp1(10.31.131.182@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513081.607064:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661094659904, offset 192 00000800:00000200:20.0:1616513081.607069:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513081.607070:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.182@tcp1 00000800:00000200:20.0:1616513081.607075:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:26.0:1616513081.607078:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.182@tcp1 routed destination 00000800:00000200:20.0:1616513081.607078:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.607080:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.607084:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.607090:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.607091:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513081.607093:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.182@tcp1(10.31.131.182@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.607099:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.607105:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.607107:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.607110:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.607114:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.607120:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513081.607143:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.607150:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.607153:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.607159:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.607165:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79ae58 00000800:00000200:22.0:1616513081.607169:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.607171:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513081.607175:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.607181:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.607184:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.607191:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.607195:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af320 00000800:00000200:12.0:1616513081.607199:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.607202:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.619271:0:49619:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513081.619284:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.619293:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.619302:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.121@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.619311:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.121@tcp1 of length 224 into portal 26 MB=0x60183d4982f40 00000400:00000200:2.0:1616513081.619317:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.121@tcp1 of length 224/224 into md 0xf97c9619 [1] + 0 00000400:00000200:2.0:1616513081.619322:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637760 00000400:00000200:2.0:1616513081.619325:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.619327:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513081.619334:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637760 00000800:00000200:18.0:1616513081.619337:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.619337:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.619340:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.619342:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.619407:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515579350848 00010000:00000200:20.0:1616513081.619417:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1690515579350848/t0(0) o400->c464c1a2-b4cc-4db9-aab1-8d5119a1fb5f@10.31.142.121@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.619430:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690515579350848, offset 192 00000400:00000200:20.0:1616513081.619454:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.121@tcp1 00000400:00000200:20.0:1616513081.619461:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.121@tcp1 routed destination 00000400:00000200:20.0:1616513081.619475:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.121@tcp1(10.31.142.121@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.619481:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.619487:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.619490:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.619492:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.619496:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.619500:0:49619:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.619503:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513081.619553:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.619560:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.619563:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.619570:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.619575:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b7e8 00000800:00000200:22.0:1616513081.619579:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.619582:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513081.619849:0:49619:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513081.619860:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.619868:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513081.619870:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513081.619877:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.121@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.619885:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.121@tcp1 of length 224 into portal 12 MB=0x60183d4983240 00000400:00000200:18.0:1616513081.619891:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.121@tcp1 of length 224/224 into md 0xf964d519 [1] + 12992 00000400:00000200:18.0:1616513081.619897:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.619899:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513081.619906:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.619909:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.619911:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.619972:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515579351616 00010000:00000200:26.0:1616513081.619984:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfe300 x1690515579351616/t0(0) o400->7553f75b-eea9-46cf-5b3e-f0f608c8dde4@10.31.142.121@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.619999:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690515579351616, offset 192 00000400:00000200:26.0:1616513081.620005:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.121@tcp1 00000400:00000200:26.0:1616513081.620012:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.121@tcp1 routed destination 00000400:00000200:26.0:1616513081.620027:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.121@tcp1(10.31.142.121@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.620033:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.620039:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.620042:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.620044:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.620048:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.620055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.620110:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.620117:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.620120:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.620127:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.620130:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af980 00000800:00000200:24.0:1616513081.620135:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.620138:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.646370:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513081.646424:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.646447:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.646457:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.88@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.646466:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.88@tcp1 of length 224 into portal 26 MB=0x601ab34603380 00000400:00000200:12.0:1616513081.646473:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.88@tcp1 of length 224/224 into md 0xf97c9639 [1] + 0 00000800:00000200:28.0:1616513081.646479:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513081.646482:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637cb0 00000400:00000200:12.0:1616513081.646485:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.646488:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513081.646493:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637cb0 00000800:00000200:12.0:1616513081.646496:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.646499:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.646501:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.646515:0:49622:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513081.646557:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690684690019200 00010000:00000200:20.0:1616513081.646568:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1690684690019200/t0(0) o400->3d905677-5c14-0c90-570f-b7ed37ccb992@10.31.136.88@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513081.646571:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.646580:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:20.0:1616513081.646582:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690684690019200, offset 192 00000400:00000200:22.0:1616513081.646589:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.88@tcp1 : PUT - for me 00000400:00000200:20.0:1616513081.646589:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.88@tcp1 00000400:00000200:20.0:1616513081.646597:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.88@tcp1 routed destination 00000400:00000200:22.0:1616513081.646598:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.88@tcp1 of length 224 into portal 12 MB=0x601ab34603400 00000400:00000200:22.0:1616513081.646605:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.88@tcp1 of length 224/224 into md 0xf964d519 [1] + 13216 00000400:00000200:22.0:1616513081.646610:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513081.646612:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.88@tcp1(10.31.136.88@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513081.646613:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513081.646619:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513081.646625:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513081.646625:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.646628:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513081.646629:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.646629:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513081.646630:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513081.646634:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513081.646635:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:7.2:1616513081.646639:0:49622:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.646641:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513081.646649:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.646656:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.646659:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513081.646666:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.646671:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a908 00000800:00000200:2.0:1616513081.646675:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.646677:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.646689:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690684690019328 00010000:00000200:26.0:1616513081.646701:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec203f00 x1690684690019328/t0(0) o400->761cf07d-cfdc-9959-7a0d-c2d25d1550d2@10.31.136.88@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.646716:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690684690019328, offset 192 00000400:00000200:26.0:1616513081.646722:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.88@tcp1 00000400:00000200:26.0:1616513081.646730:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.88@tcp1 routed destination 00000400:00000200:26.0:1616513081.646745:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.88@tcp1(10.31.136.88@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.646753:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.646759:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.646762:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.646764:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.646769:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.646773:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.646829:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.646836:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.646839:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.646845:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.646849:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae6e8 00000800:00000200:24.0:1616513081.646854:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.646856:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.664444:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513081.664502:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.664511:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513081.664520:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.66@tcp1 : PUT - for me 00000400:00000200:28.0:1616513081.664529:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.66@tcp1 of length 224 into portal 26 MB=0x5ffd48e3a8180 00000400:00000200:28.0:1616513081.664536:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.66@tcp1 of length 224/224 into md 0xf97c9649 [1] + 0 00000400:00000200:28.0:1616513081.664540:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637d38 00000400:00000200:28.0:1616513081.664543:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513081.664546:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513081.664551:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637d38 00000800:00000200:28.0:1616513081.664554:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.664554:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.664556:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513081.664561:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513081.664564:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.664565:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.664574:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.66@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.664582:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.66@tcp1 of length 224 into portal 12 MB=0x5ffd48e3a8200 00000400:00000200:12.0:1616513081.664589:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.66@tcp1 of length 224/224 into md 0x865bbec1 [1] + 42896 00000400:00000200:12.0:1616513081.664594:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.664596:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513081.664604:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.664607:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.664609:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.664615:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663267901824 00010000:00000200:20.0:1616513081.664626:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688663267901824/t0(0) o400->a839ceed-a177-90fd-4019-bb31b87a3eaa@10.31.138.66@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.664641:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663267901824, offset 192 00000400:00000200:20.0:1616513081.664647:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.66@tcp1 00000400:00000200:20.0:1616513081.664655:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.66@tcp1 routed destination 00000100:00000200:0.0:1616513081.664667:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663267901952 00000400:00000200:20.0:1616513081.664670:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.66@tcp1(10.31.138.66@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.664676:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:0.0:1616513081.664680:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dcc80 x1688663267901952/t0(0) o400->0534229c-81c8-afe3-3dbe-ad9f7fef3c4b@10.31.138.66@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.664684:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.664687:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.664689:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.664694:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513081.664694:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663267901952, offset 192 00000800:00000200:7.2:1616513081.664699:0:49624:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.664701:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513081.664701:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.66@tcp1 00000400:00000200:0.0:1616513081.664708:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.66@tcp1 routed destination 00000400:00000200:0.0:1616513081.664723:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.66@tcp1(10.31.138.66@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.664730:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.664735:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.664738:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.664740:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.664745:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.664750:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513081.664753:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.664760:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.664763:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513081.664770:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.664775:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aee0 00000800:00000200:14.0:1616513081.664779:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.664782:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513081.664802:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.664810:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513081.664813:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.664813:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513081.664819:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.664823:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:16.0:1616513081.664827:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.688114:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513081.688169:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.688179:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.688188:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.94@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.688197:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.94@tcp1 of length 224 into portal 26 MB=0x5ffd4eadf5b40 00000400:00000200:12.0:1616513081.688204:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.94@tcp1 of length 224/224 into md 0xf97c9669 [1] + 0 00000400:00000200:12.0:1616513081.688208:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636990 00000400:00000200:12.0:1616513081.688211:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.688213:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513081.688218:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636990 00000800:00000200:12.0:1616513081.688220:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.688223:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.688223:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.688227:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.688282:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664822209344 00010000:00000200:20.0:1616513081.688293:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688664822209344/t0(0) o400->2520637e-4b50-f9fe-53ce-0666ed759870@10.31.131.94@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.688307:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664822209344, offset 192 00000400:00000200:20.0:1616513081.688313:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.94@tcp1 00000800:00000200:7.2:1616513081.688318:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513081.688321:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.94@tcp1 routed destination 00000400:00000200:20.0:1616513081.688336:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.94@tcp1(10.31.131.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.688343:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.688349:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.688352:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.688355:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513081.688359:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:20.0:1616513081.688366:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513081.688373:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.688382:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.688390:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.94@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.688398:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.94@tcp1 of length 224 into portal 12 MB=0x5ffd4eadf5bc0 00000400:00000200:2.0:1616513081.688404:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.94@tcp1 of length 224/224 into md 0xf964d519 [1] + 13440 00000400:00000200:2.0:1616513081.688409:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.688412:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513081.688419:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513081.688422:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513081.688423:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513081.688426:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.688447:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.688450:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513081.688456:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513081.688462:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a880 00000800:00000200:18.0:1616513081.688466:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.688469:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.688485:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664822209472 00010000:00000200:26.0:1616513081.688498:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21c800 x1688664822209472/t0(0) o400->3a8f4ed4-d9fa-4d17-75b2-95691fd64659@10.31.131.94@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.688513:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664822209472, offset 192 00000400:00000200:26.0:1616513081.688519:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.94@tcp1 00000400:00000200:26.0:1616513081.688526:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.94@tcp1 routed destination 00000400:00000200:26.0:1616513081.688541:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.94@tcp1(10.31.131.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.688548:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.688553:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.688556:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.688558:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.688563:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.688569:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.688623:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.688630:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.688633:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513081.688640:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.688644:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae220 00000800:00000200:28.0:1616513081.688648:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.688651:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.739622:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.739680:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.739690:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.739700:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.29@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.739710:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.29@tcp1 of length 224 into portal 12 MB=0x5ffe972b303c0 00000400:00000200:22.0:1616513081.739717:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.29@tcp1 of length 224/224 into md 0xf964d519 [1] + 13664 00000400:00000200:22.0:1616513081.739723:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.739726:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513081.739734:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.739734:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.739737:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.739739:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.739772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513081.739801:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753000350656 00010000:00000200:26.0:1616513081.739813:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec200900 x1688753000350656/t0(0) o400->9451769b-a203-08f1-cc0e-f51372b7e8f1@10.31.140.29@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.739828:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688753000350656, offset 192 00000800:00000200:24.0:1616513081.739828:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513081.739835:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.29@tcp1 00000800:00000200:24.0:1616513081.739837:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513081.739843:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.29@tcp1 routed destination 00000400:00000200:24.0:1616513081.739846:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.29@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.739855:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.29@tcp1 of length 224 into portal 26 MB=0x5ffe972b30100 00000400:00000200:26.0:1616513081.739858:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.29@tcp1(10.31.140.29@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513081.739862:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.29@tcp1 of length 224/224 into md 0xf97c9679 [1] + 0 00000800:00000200:26.0:1616513081.739865:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513081.739867:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c6378f8 00000400:00000200:24.0:1616513081.739870:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513081.739871:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:24.0:1616513081.739872:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513081.739874:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513081.739876:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513081.739877:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6378f8 00000800:00000200:24.0:1616513081.739880:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513081.739881:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513081.739881:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.739883:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.739885:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.739887:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.739949:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513081.739956:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.739959:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:20.0:1616513081.739964:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753000349952 00000400:00000200:18.0:1616513081.739966:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513081.739972:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae770 00010000:00000200:20.0:1616513081.739975:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688753000349952/t0(0) o400->05b21e22-dbee-65be-dd2d-1e196f0d44a2@10.31.140.29@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513081.739976:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.739979:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.739990:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688753000349952, offset 192 00000400:00000200:20.0:1616513081.739996:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.29@tcp1 00000400:00000200:20.0:1616513081.740003:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.29@tcp1 routed destination 00000400:00000200:20.0:1616513081.740018:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.29@tcp1(10.31.140.29@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.740025:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.740031:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.740034:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.740036:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.740041:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.740046:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.740049:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513081.740101:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.740108:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.740111:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513081.740117:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.740125:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aaa0 00000800:00000200:28.0:1616513081.740128:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.740131:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.749169:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513081.749226:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.749235:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.749244:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.28@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.749253:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.28@tcp1 of length 224 into portal 26 MB=0x5ffd3c366bd00 00000400:00000200:2.0:1616513081.749260:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.28@tcp1 of length 224/224 into md 0xf97c9689 [1] + 0 00000400:00000200:2.0:1616513081.749265:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637e48 00000400:00000200:2.0:1616513081.749267:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.749270:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513081.749274:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637e48 00000800:00000200:2.0:1616513081.749277:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.749279:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.749282:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.749284:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.749344:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659865025792 00010000:00000200:20.0:1616513081.749355:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688659865025792/t0(0) o400->2baf57aa-1e01-d797-aecc-df9db7fcfdd0@10.31.138.28@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.749369:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688659865025792, offset 192 00000800:00000200:11.2:1616513081.749374:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:20.0:1616513081.749375:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.28@tcp1 00000400:00000200:20.0:1616513081.749383:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.28@tcp1 routed destination 00000400:00000200:20.0:1616513081.749399:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.28@tcp1(10.31.138.28@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.749405:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.749411:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.749414:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.749416:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.749421:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.749425:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.749428:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513081.749428:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.749449:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.749458:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.28@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.749468:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.28@tcp1 of length 224 into portal 12 MB=0x5ffd3c366bd80 00000400:00000200:12.0:1616513081.749474:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.28@tcp1 of length 224/224 into md 0x865bbec1 [1] + 43120 00000800:00000200:18.0:1616513081.749478:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513081.749479:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.749482:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513081.749485:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513081.749488:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513081.749489:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.749492:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.749494:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513081.749495:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513081.749498:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513081.749500:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a5d8 00000800:00000200:18.0:1616513081.749504:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.749507:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513081.749569:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659865025920 00010000:00000200:0.0:1616513081.749582:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d8480 x1688659865025920/t0(0) o400->a9ab2556-5fc1-c4f0-d9ba-af5f2ccc9433@10.31.138.28@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.749597:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688659865025920, offset 192 00000400:00000200:0.0:1616513081.749603:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.28@tcp1 00000400:00000200:0.0:1616513081.749610:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.28@tcp1 routed destination 00000400:00000200:0.0:1616513081.749625:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.28@tcp1(10.31.138.28@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.749632:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.749638:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.749640:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.749642:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.749647:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.749652:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.749705:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.749711:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.749714:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513081.749721:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.749725:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04088 00000800:00000200:28.0:1616513081.749729:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.749732:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.751445:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.751499:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.751508:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.751517:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.146@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.751527:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.146@tcp1 of length 224 into portal 26 MB=0x5ffe06639ec00 00000400:00000200:24.0:1616513081.751535:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.146@tcp1 of length 224/224 into md 0xf97c9699 [1] + 0 00000400:00000200:24.0:1616513081.751540:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637078 00000400:00000200:24.0:1616513081.751543:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.751546:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513081.751550:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637078 00000800:00000200:12.0:1616513081.751552:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.751553:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.751556:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513081.751559:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.751562:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.751562:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.751571:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.146@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.751580:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.146@tcp1 of length 224 into portal 12 MB=0x5ffe06639ee00 00000400:00000200:12.0:1616513081.751586:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.146@tcp1 of length 224/224 into md 0x865bbec1 [1] + 43344 00000400:00000200:12.0:1616513081.751592:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.751595:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513081.751601:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.751604:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.751606:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.751613:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714136382464 00010000:00000200:20.0:1616513081.751624:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688714136382464/t0(0) o400->950b8941-443f-4fdf-70c7-636b01ce6565@10.31.140.146@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.751639:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688714136382464, offset 192 00000400:00000200:20.0:1616513081.751645:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.146@tcp1 00000400:00000200:20.0:1616513081.751652:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.146@tcp1 routed destination 00000100:00000200:0.0:1616513081.751666:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714136382976 00000400:00000200:20.0:1616513081.751667:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.146@tcp1(10.31.140.146@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.751673:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:0.0:1616513081.751678:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0df500 x1688714136382976/t0(0) o400->e9842d71-9680-897d-ae66-25c0104c8b57@10.31.140.146@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513081.751679:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.751682:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.751684:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.751689:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513081.751692:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688714136382976, offset 192 00000800:00000200:7.2:1616513081.751694:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.751696:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513081.751698:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.146@tcp1 00000400:00000200:0.0:1616513081.751706:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.146@tcp1 routed destination 00000400:00000200:0.0:1616513081.751723:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.146@tcp1(10.31.140.146@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.751729:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.751742:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.751744:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.751746:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.751751:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513081.751752:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513081.751757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513081.751758:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.751761:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.751768:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.751773:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b760 00000800:00000200:22.0:1616513081.751779:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.751782:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513081.751828:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.751835:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.751838:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.751838:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513081.751844:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.751848:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04b28 00000800:00000200:16.0:1616513081.751852:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.759383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513081.759396:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.759405:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513081.759414:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.167@tcp1 : PUT - for me 00000400:00000200:18.0:1616513081.759423:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.167@tcp1 of length 224 into portal 26 MB=0x5ffd833055d00 00000400:00000200:18.0:1616513081.759430:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.167@tcp1 of length 224/224 into md 0xf97c96a9 [1] + 0 00000400:00000200:18.0:1616513081.759441:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636550 00000400:00000200:18.0:1616513081.759444:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513081.759447:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513081.759450:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513081.759455:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636550 00000800:00000200:18.0:1616513081.759459:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513081.759461:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513081.759463:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.759519:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678917561600 00010000:00000200:20.0:1616513081.759529:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688678917561600/t0(0) o400->462f3373-6735-9794-418e-c69fdabf188c@10.31.129.167@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.759542:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688678917561600, offset 192 00000400:00000200:20.0:1616513081.759549:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.167@tcp1 00000400:00000200:20.0:1616513081.759556:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.167@tcp1 routed destination 00000400:00000200:20.0:1616513081.759570:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.167@tcp1(10.31.129.167@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.759576:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.759581:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.759584:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.759586:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.759590:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.759595:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.759596:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513081.759646:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.759653:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.759656:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.759662:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.759668:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79ad48 00000800:00000200:22.0:1616513081.759672:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.759675:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513081.760266:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513081.760277:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.760286:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513081.760287:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.760295:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.167@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.760304:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.167@tcp1 of length 224 into portal 12 MB=0x5ffd833056140 00000400:00000200:14.0:1616513081.760311:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.167@tcp1 of length 224/224 into md 0xf964d519 [1] + 13888 00000400:00000200:14.0:1616513081.760316:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.760319:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513081.760326:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.760329:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.760331:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.760392:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678917562688 00010000:00000200:26.0:1616513081.760405:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba6300 x1688678917562688/t0(0) o400->307970f6-ed59-3c71-6bfc-67acabc6008b@10.31.129.167@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.760419:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688678917562688, offset 192 00000400:00000200:26.0:1616513081.760425:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.167@tcp1 00000400:00000200:26.0:1616513081.760432:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.167@tcp1 routed destination 00000400:00000200:26.0:1616513081.760462:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.167@tcp1(10.31.129.167@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.760469:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.760474:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.760477:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.760479:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.760484:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.760499:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.760553:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.760559:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.760562:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.760568:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.760571:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aec38 00000800:00000200:12.0:1616513081.760575:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.760578:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.761492:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.761544:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.761552:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.761560:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.202@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.761569:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.202@tcp1 of length 224 into portal 12 MB=0x5ffd51f2d8f40 00000400:00000200:24.0:1616513081.761575:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.202@tcp1 of length 224/224 into md 0x865bbec1 [1] + 43568 00000400:00000200:24.0:1616513081.761580:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.761582:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.761588:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.761591:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.761593:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.761595:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.761613:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665699749696 00010000:00000200:0.0:1616513081.761625:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd706300 x1688665699749696/t0(0) o400->36562f5e-592f-86a5-3647-06dc25c32316@10.31.132.202@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.761640:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665699749696, offset 192 00000400:00000200:0.0:1616513081.761646:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.202@tcp1 00000400:00000200:0.0:1616513081.761654:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.202@tcp1 routed destination 00000400:00000200:0.0:1616513081.761668:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.202@tcp1(10.31.132.202@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.761675:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513081.761681:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.761684:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513081.761686:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513081.761691:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.761725:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.761779:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.761788:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.761796:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.202@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.761805:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.202@tcp1 of length 224 into portal 26 MB=0x5ffd51f2d8ec0 00000400:00000200:22.0:1616513081.761811:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.202@tcp1 of length 224/224 into md 0xf97c96c9 [1] + 0 00000400:00000200:22.0:1616513081.761816:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637210 00000400:00000200:22.0:1616513081.761819:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.761821:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513081.761825:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637210 00000800:00000200:22.0:1616513081.761828:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513081.761831:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513081.761831:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:22.0:1616513081.761839:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513081.761841:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513081.761842:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.761843:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513081.761850:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.761854:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05430 00000800:00000200:2.0:1616513081.761858:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.761886:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665699749568 00010000:00000200:20.0:1616513081.761896:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688665699749568/t0(0) o400->daeea0af-2470-0bcd-2464-ec819977b6cb@10.31.132.202@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.761910:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665699749568, offset 192 00000400:00000200:20.0:1616513081.761915:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.202@tcp1 00000400:00000200:20.0:1616513081.761922:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.202@tcp1 routed destination 00000400:00000200:20.0:1616513081.761936:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.202@tcp1(10.31.132.202@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.761942:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.761947:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.761950:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.761952:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.761956:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.761961:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.761962:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513081.761981:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.761984:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.761986:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.761991:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.761994:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79ba08 00000800:00000200:12.0:1616513081.761997:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.761999:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.764157:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.764210:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.764219:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513081.764228:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.80@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.764238:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.80@tcp1 of length 224 into portal 12 MB=0x5ffd30f94c600 00000400:00000200:16.0:1616513081.764246:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.80@tcp1 of length 224/224 into md 0x865bbec1 [1] + 43792 00000400:00000200:16.0:1616513081.764251:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.764254:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513081.764260:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.764263:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.764263:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.764267:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.764321:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656848143872 00010000:00000200:0.0:1616513081.764332:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd6300 x1688656848143872/t0(0) o400->5d16ad60-3250-10dc-0815-66894be3d66f@10.31.129.80@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.764346:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688656848143872, offset 192 00000400:00000200:0.0:1616513081.764352:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.80@tcp1 00000400:00000200:0.0:1616513081.764359:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.80@tcp1 routed destination 00000400:00000200:0.0:1616513081.764373:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.80@tcp1(10.31.129.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.764379:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513081.764384:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.764387:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513081.764391:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513081.764395:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.764401:0:49635:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.764456:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.764464:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.764467:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513081.764474:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.764478:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:14.0:1616513081.764482:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.764484:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.764490:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513081.764498:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.80@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.764506:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.80@tcp1 of length 224 into portal 26 MB=0x5ffd30f94c240 00000800:00000200:22.0:1616513081.764509:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.764513:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.80@tcp1 of length 224/224 into md 0xf97c96e9 [1] + 0 00000400:00000200:14.0:1616513081.764517:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636cc0 00000400:00000200:14.0:1616513081.764519:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.764522:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513081.764526:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636cc0 00000800:00000200:14.0:1616513081.764529:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.764531:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.764533:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.764587:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656848142912 00010000:00000200:20.0:1616513081.764596:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688656848142912/t0(0) o400->2befd152-d3fb-7030-47f0-abd31ae09804@10.31.129.80@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.764609:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688656848142912, offset 192 00000400:00000200:20.0:1616513081.764614:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.80@tcp1 00000400:00000200:20.0:1616513081.764621:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.80@tcp1 routed destination 00000400:00000200:20.0:1616513081.764634:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.80@tcp1(10.31.129.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.764644:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.764648:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.764651:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.764653:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.764657:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.764662:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.764663:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513081.764672:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.764678:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.764682:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.764689:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.764694:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b540 00000800:00000200:12.0:1616513081.764697:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.764700:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.767668:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.767722:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.767731:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.767740:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.160@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.767749:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.160@tcp1 of length 224 into portal 26 MB=0x5ffe1270d4dc0 00000400:00000200:24.0:1616513081.767756:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.160@tcp1 of length 224/224 into md 0xf97c9709 [1] + 0 00000400:00000200:24.0:1616513081.767761:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c6364c8 00000400:00000200:24.0:1616513081.767763:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.767766:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513081.767770:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6364c8 00000800:00000200:24.0:1616513081.767773:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.767778:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.767780:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.767787:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.767790:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.767790:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513081.767799:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.160@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.767808:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.160@tcp1 of length 224 into portal 12 MB=0x5ffe1270d4ec0 00000400:00000200:16.0:1616513081.767814:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.160@tcp1 of length 224/224 into md 0x865bbec1 [1] + 44016 00000400:00000200:16.0:1616513081.767819:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.767822:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513081.767828:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.767831:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717371461056 00000800:00000200:16.0:1616513081.767832:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.767833:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:20.0:1616513081.767841:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688717371461056/t0(0) o400->084ac13c-cce9-3241-90e3-a7d93242b56a@10.31.141.160@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.767853:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688717371461056, offset 192 00000400:00000200:20.0:1616513081.767858:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.160@tcp1 00000400:00000200:20.0:1616513081.767865:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.160@tcp1 routed destination 00000400:00000200:20.0:1616513081.767878:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.160@tcp1(10.31.141.160@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.767883:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.767888:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513081.767890:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717371461312 00000800:00000200:20.0:1616513081.767891:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.767892:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.767897:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.767902:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.767902:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513081.767902:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd3600 x1688717371461312/t0(0) o400->91855f42-844b-1fce-3f17-8df10510dfec@10.31.141.160@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.767915:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688717371461312, offset 192 00000400:00000200:0.0:1616513081.767921:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.160@tcp1 00000400:00000200:0.0:1616513081.767928:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.160@tcp1 routed destination 00000400:00000200:0.0:1616513081.767942:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.160@tcp1(10.31.141.160@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.767948:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.767953:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.767955:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.767956:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513081.767957:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.767962:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513081.767962:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513081.767965:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513081.767966:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513081.767972:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513081.767977:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b5c8 00000800:00000200:26.0:1616513081.767981:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513081.767983:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513081.768023:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.768030:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.768033:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513081.768034:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.768039:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.768043:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb047f8 00000800:00000200:28.0:1616513081.768047:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.775195:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.775250:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.775260:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513081.775269:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.96@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.775278:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.96@tcp1 of length 224 into portal 26 MB=0x5ffd4fd0f4740 00000400:00000200:16.0:1616513081.775285:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.96@tcp1 of length 224/224 into md 0xf97c9739 [1] + 0 00000400:00000200:16.0:1616513081.775289:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636c38 00000400:00000200:16.0:1616513081.775292:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.775295:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513081.775300:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636c38 00000800:00000200:16.0:1616513081.775303:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.775306:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.775308:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.775310:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.775320:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513081.775328:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.96@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.775336:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.96@tcp1 of length 224 into portal 12 MB=0x5ffd4fd0f4800 00000400:00000200:12.0:1616513081.775342:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.96@tcp1 of length 224/224 into md 0x865bbec1 [1] + 44240 00000400:00000200:12.0:1616513081.775347:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.775349:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513081.775355:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.775358:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:20.0:1616513081.775360:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665127339840 00000800:00000200:12.0:1616513081.775360:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.775362:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:20.0:1616513081.775369:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688665127339840/t0(0) o400->c514a8c9-d433-d75e-d1de-a7991063e365@10.31.131.96@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.775381:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665127339840, offset 192 00000400:00000200:20.0:1616513081.775386:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.96@tcp1 00000400:00000200:20.0:1616513081.775393:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.96@tcp1 routed destination 00000400:00000200:20.0:1616513081.775406:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.96@tcp1(10.31.131.96@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.775411:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.775416:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513081.775417:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665127340032 00000800:00000200:20.0:1616513081.775418:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.775422:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.775426:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513081.775429:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0db600 x1688665127340032/t0(0) o400->1ada1d7a-e80b-6524-de1a-5de085233c6e@10.31.131.96@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513081.775432:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513081.775461:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665127340032, offset 192 00000400:00000200:0.0:1616513081.775466:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.96@tcp1 00000800:00000200:7.2:1616513081.775473:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513081.775473:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.96@tcp1 routed destination 00000400:00000200:0.0:1616513081.775487:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.96@tcp1(10.31.131.96@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.775493:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.775498:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.775501:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.775503:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.775507:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.775512:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513081.775530:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.775537:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.775540:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513081.775547:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.775552:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b980 00000800:00000200:14.0:1616513081.775556:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.775559:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513081.775565:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.775572:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.775575:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513081.775576:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.775581:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.775585:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:28.0:1616513081.775589:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.799386:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513081.799455:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513081.799464:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513081.799473:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.17@tcp1 : PUT - for me 00000400:00000200:26.0:1616513081.799484:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.17@tcp1 of length 224 into portal 26 MB=0x5f9110a521800 00000400:00000200:26.0:1616513081.799491:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.17@tcp1 of length 224/224 into md 0xf97c9769 [1] + 0 00000400:00000200:26.0:1616513081.799496:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637870 00000400:00000200:26.0:1616513081.799498:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513081.799501:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513081.799505:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637870 00000800:00000200:26.0:1616513081.799509:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.799512:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.799514:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513081.799520:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513081.799523:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.799524:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.799532:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.17@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.799541:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.17@tcp1 of length 224 into portal 12 MB=0x5f9110a521880 00000400:00000200:22.0:1616513081.799547:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.17@tcp1 of length 224/224 into md 0xf964d519 [1] + 14112 00000400:00000200:22.0:1616513081.799551:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.799554:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513081.799561:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.799564:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.799564:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681226466465792 00000800:00000200:22.0:1616513081.799568:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:20.0:1616513081.799573:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1681226466465792/t0(0) o400->d803b9b1-6735-0004-aa85-f813e2d3916c@10.31.132.17@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513081.799583:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681226466465920 00000100:00000200:20.0:1616513081.799585:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681226466465792, offset 192 00000400:00000200:20.0:1616513081.799590:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.17@tcp1 00010000:00000200:26.0:1616513081.799594:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec205a00 x1681226466465920/t0(0) o400->cd67ab6e-ced4-792b-f657-7ef220d4c5ba@10.31.132.17@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513081.799596:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.17@tcp1 routed destination 00000100:00000200:26.0:1616513081.799607:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681226466465920, offset 192 00000400:00000200:20.0:1616513081.799608:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.17@tcp1(10.31.132.17@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513081.799613:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.17@tcp1 00000800:00000200:20.0:1616513081.799613:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.799618:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:26.0:1616513081.799620:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.17@tcp1 routed destination 00000800:00000200:20.0:1616513081.799621:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.799622:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.799626:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.799632:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.799632:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:26.0:1616513081.799634:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.17@tcp1(10.31.132.17@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.799641:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.799646:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.799649:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.799651:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.799655:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513081.799685:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.799692:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.799695:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513081.799696:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513081.799701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513081.799701:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.799707:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bc28 00000800:00000200:2.0:1616513081.799711:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513081.799753:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.799763:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.799765:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.799771:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.799775:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae110 00000800:00000200:12.0:1616513081.799778:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.799781:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.824036:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.824092:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.824101:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.824111:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.52@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.824120:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.52@tcp1 of length 224 into portal 26 MB=0x60185b99fa500 00000400:00000200:22.0:1616513081.824126:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.52@tcp1 of length 224/224 into md 0xf97c9779 [1] + 0 00000400:00000200:22.0:1616513081.824139:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636bb0 00000400:00000200:22.0:1616513081.824141:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.824145:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513081.824147:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513081.824149:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636bb0 00000800:00000200:22.0:1616513081.824152:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.824154:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.824156:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.824216:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690523716789504 00010000:00000200:20.0:1616513081.824227:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1690523716789504/t0(0) o400->b9c76072-a58d-2b4d-bb88-3855d55e29af@10.31.133.52@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.824241:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690523716789504, offset 192 00000400:00000200:20.0:1616513081.824247:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.52@tcp1 00000400:00000200:20.0:1616513081.824254:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.52@tcp1 routed destination 00000400:00000200:20.0:1616513081.824270:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.52@tcp1(10.31.133.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.824276:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.824282:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.824285:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.824287:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.824292:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.824297:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.824298:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513081.824349:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.824357:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513081.824360:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513081.824366:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513081.824372:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b298 00000800:00000200:26.0:1616513081.824375:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513081.824378:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.824609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.824663:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.824673:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513081.824679:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.824681:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.52@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.824691:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.52@tcp1 of length 224 into portal 12 MB=0x60185b99fa680 00000400:00000200:16.0:1616513081.824697:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.52@tcp1 of length 224/224 into md 0x865bbec1 [1] + 44464 00000400:00000200:16.0:1616513081.824702:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.824705:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513081.824711:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.824714:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.824716:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.824735:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690523716789888 00010000:00000200:0.0:1616513081.824746:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd4c80 x1690523716789888/t0(0) o400->f38fefd3-f58d-aa10-e02e-345ee7b74646@10.31.133.52@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.824759:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690523716789888, offset 192 00000400:00000200:0.0:1616513081.824767:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.52@tcp1 00000400:00000200:0.0:1616513081.824774:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.52@tcp1 routed destination 00000400:00000200:0.0:1616513081.824787:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.52@tcp1(10.31.133.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.824794:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.824799:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.824802:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.824804:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.824808:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.824813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.824825:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.824831:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.824834:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513081.824841:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.824845:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04000 00000800:00000200:12.0:1616513081.824848:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513081.824851:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.825357:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513081.825410:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.825423:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513081.825432:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.22@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.825468:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.22@tcp1 of length 224 into portal 26 MB=0x5ffdfbb6f9180 00000400:00000200:14.0:1616513081.825474:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.22@tcp1 of length 224/224 into md 0xf97c9799 [1] + 0 00000400:00000200:14.0:1616513081.825479:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637c28 00000400:00000200:14.0:1616513081.825482:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513081.825484:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:14.0:1616513081.825485:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513081.825491:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637c28 00000800:00000200:14.0:1616513081.825494:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513081.825495:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513081.825497:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513081.825497:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513081.825499:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513081.825504:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.22@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.825513:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.22@tcp1 of length 224 into portal 12 MB=0x5ffdfbb6f93c0 00000400:00000200:22.0:1616513081.825520:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.22@tcp1 of length 224/224 into md 0xf964d519 [1] + 14336 00000400:00000200:22.0:1616513081.825525:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.825528:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513081.825534:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.825537:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.825539:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.825553:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711270994880 00000100:00000200:20.0:1616513081.825554:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711270994304 00010000:00000200:26.0:1616513081.825565:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec207500 x1688711270994880/t0(0) o400->f4314f2e-b53c-2e9e-2e56-c50cb7d8fe28@10.31.140.22@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:20.0:1616513081.825565:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688711270994304/t0(0) o400->3c20680b-400d-b054-c31e-871aa1abd6d2@10.31.140.22@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513081.825578:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688711270994880, offset 192 00000100:00000200:20.0:1616513081.825581:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688711270994304, offset 192 00000400:00000200:26.0:1616513081.825585:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.22@tcp1 00000400:00000200:20.0:1616513081.825588:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.22@tcp1 00000400:00000200:26.0:1616513081.825592:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.22@tcp1 routed destination 00000400:00000200:20.0:1616513081.825595:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.22@tcp1 routed destination 00000400:00000200:26.0:1616513081.825607:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.22@tcp1(10.31.140.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513081.825611:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.22@tcp1(10.31.140.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.825614:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.825618:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.825619:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.825622:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.825624:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.825626:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.825626:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.825628:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.825630:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513081.825633:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513081.825636:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513081.825639:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.825640:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513081.825650:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.825655:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.825658:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513081.825660:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.825664:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.825669:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aff0 00000800:00000200:28.0:1616513081.825673:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513081.825690:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.825697:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513081.825700:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513081.825700:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513081.825706:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.825710:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af7e8 00000800:00000200:2.0:1616513081.825714:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513081.842506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513081.842563:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.842573:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513081.842582:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.72@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.842593:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.72@tcp1 of length 224 into portal 26 MB=0x5ffd4c437c600 00000400:00000200:14.0:1616513081.842599:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.72@tcp1 of length 224/224 into md 0xf97c97a9 [1] + 0 00000400:00000200:14.0:1616513081.842604:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c6377e8 00000400:00000200:14.0:1616513081.842607:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.842610:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513081.842614:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6377e8 00000800:00000200:14.0:1616513081.842617:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.842620:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.842621:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.842624:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.842669:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513081.842679:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664173692416 00010000:00000200:20.0:1616513081.842690:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688664173692416/t0(0) o400->5357c089-2ebf-58b5-39e8-970bb4bf7c8b@10.31.131.72@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.842704:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664173692416, offset 192 00000400:00000200:20.0:1616513081.842710:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.72@tcp1 00000400:00000200:20.0:1616513081.842718:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.72@tcp1 routed destination 00000800:00000200:12.0:1616513081.842725:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513081.842733:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.72@tcp1(10.31.131.72@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513081.842735:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.842740:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513081.842744:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.72@tcp1 : PUT - for me 00000800:00000200:20.0:1616513081.842746:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.842749:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.842751:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513081.842753:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.72@tcp1 of length 224 into portal 12 MB=0x5ffd4c437c680 00000800:00000200:20.0:1616513081.842756:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513081.842761:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.72@tcp1 of length 224/224 into md 0x865bbec1 [1] + 44688 00000800:00000200:7.2:1616513081.842761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.842762:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513081.842766:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.842768:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513081.842775:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.842778:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.842779:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.842784:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513081.842814:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.842821:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513081.842824:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513081.842831:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513081.842836:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79ba90 00000800:00000200:26.0:1616513081.842840:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513081.842843:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513081.842847:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664173692544 00010000:00000200:0.0:1616513081.842860:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dd580 x1688664173692544/t0(0) o400->060f7b63-4e40-58fe-0742-de082dfca465@10.31.131.72@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.842874:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664173692544, offset 192 00000400:00000200:0.0:1616513081.842881:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.72@tcp1 00000400:00000200:0.0:1616513081.842888:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.72@tcp1 routed destination 00000400:00000200:0.0:1616513081.842905:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.72@tcp1(10.31.131.72@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.842911:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.842917:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.842920:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.842922:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.842927:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.842932:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.842986:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.842993:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.842995:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513081.843002:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.843006:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04198 00000800:00000200:16.0:1616513081.843010:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.843013:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.843802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513081.843853:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.843862:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513081.843870:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.33@tcp1 : PUT - for me 00000400:00000200:14.0:1616513081.843879:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.33@tcp1 of length 224 into portal 12 MB=0x60393e326a840 00000400:00000200:14.0:1616513081.843884:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.33@tcp1 of length 224/224 into md 0xf964d519 [1] + 14560 00000400:00000200:14.0:1616513081.843889:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513081.843892:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513081.843898:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.843901:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.843903:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.843905:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513081.843916:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783566301248 00010000:00000200:26.0:1616513081.843926:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba4380 x1692783566301248/t0(0) o400->00a3308f-6c26-c731-4f2a-1883345d3fcb@10.31.140.33@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513081.843938:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783566301248, offset 192 00000400:00000200:26.0:1616513081.843944:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.33@tcp1 00000400:00000200:26.0:1616513081.843950:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.33@tcp1 routed destination 00000400:00000200:26.0:1616513081.843963:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.33@tcp1(10.31.140.33@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513081.843969:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513081.843970:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.843974:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513081.843977:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513081.843979:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513081.843982:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.843999:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513081.844009:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.844012:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513081.844014:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513081.844019:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513081.844021:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa08 00000800:00000200:26.0:1616513081.844023:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513081.844024:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513081.844025:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513081.844033:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.844042:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.33@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.844050:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.33@tcp1 of length 224 into portal 26 MB=0x60393e326a5c0 00000400:00000200:24.0:1616513081.844057:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.33@tcp1 of length 224/224 into md 0xf97c97c9 [1] + 0 00000400:00000200:24.0:1616513081.844061:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636d48 00000400:00000200:24.0:1616513081.844064:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.844067:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513081.844071:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636d48 00000800:00000200:24.0:1616513081.844074:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.844076:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.844078:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.844081:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.844136:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783566300608 00010000:00000200:20.0:1616513081.844146:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1692783566300608/t0(0) o400->ce6be2b8-8994-5c88-65eb-e16235f25bf3@10.31.140.33@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.844159:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783566300608, offset 192 00000400:00000200:20.0:1616513081.844165:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.33@tcp1 00000400:00000200:20.0:1616513081.844171:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.33@tcp1 routed destination 00000400:00000200:20.0:1616513081.844187:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.33@tcp1(10.31.140.33@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.844193:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.844198:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.844200:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.844202:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.844207:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.844212:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.844213:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513081.844266:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.844273:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.844276:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513081.844282:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.844288:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bdc0 00000800:00000200:16.0:1616513081.844292:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.844295:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:27.0:1616513081.871229:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:27.0:1616513081.871237:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:7.2:1616513081.879354:0:49647:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513081.879410:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.879420:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.879429:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.172@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.879453:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.172@tcp1 of length 224 into portal 26 MB=0x5ffd4d2a89440 00000400:00000200:22.0:1616513081.879460:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.172@tcp1 of length 224/224 into md 0xf97c97e9 [1] + 0 00000400:00000200:22.0:1616513081.879464:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637298 00000800:00000200:14.0:1616513081.879464:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513081.879467:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.879470:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513081.879474:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637298 00000800:00000200:22.0:1616513081.879479:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.879482:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.879484:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513081.879513:0:49647:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:20.0:1616513081.879540:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664415966272 00010000:00000200:20.0:1616513081.879550:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688664415966272/t0(0) o400->6d0a9a43-f77f-9b40-1edd-54ff24d244d9@10.31.136.172@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.879565:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664415966272, offset 192 00000800:00000200:26.0:1616513081.879567:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513081.879571:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.172@tcp1 00000800:00000200:26.0:1616513081.879577:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513081.879578:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.172@tcp1 routed destination 00000400:00000200:26.0:1616513081.879586:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.172@tcp1 : PUT - for me 00000400:00000200:20.0:1616513081.879594:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.172@tcp1(10.31.136.172@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.879601:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513081.879603:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.172@tcp1 of length 224 into portal 12 MB=0x5ffd4d2a89540 00000800:00000200:20.0:1616513081.879607:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.879609:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513081.879611:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.172@tcp1 of length 224/224 into md 0xf964d519 [1] + 14784 00000800:00000200:20.0:1616513081.879611:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:26.0:1616513081.879616:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513081.879616:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:26.0:1616513081.879619:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:20.0:1616513081.879623:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513081.879623:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513081.879627:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513081.879630:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:2.0:1616513081.879631:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:26.0:1616513081.879632:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513081.879634:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513081.879635:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513081.879641:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.879650:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bdc0 00000800:00000200:2.0:1616513081.879653:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513081.879654:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664415966528 00010000:00000200:30.0:1616513081.879665:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc579680 x1688664415966528/t0(0) o400->9afaeb02-127d-5425-10da-e77dde04fe99@10.31.136.172@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513081.879680:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664415966528, offset 192 00000400:00000200:30.0:1616513081.879686:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.172@tcp1 00000400:00000200:30.0:1616513081.879694:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.172@tcp1 routed destination 00000400:00000200:30.0:1616513081.879709:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.172@tcp1(10.31.136.172@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513081.879716:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513081.879721:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513081.879724:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513081.879726:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513081.879731:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.879777:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.879834:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.879841:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.879843:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.879850:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.879854:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:24.0:1616513081.879858:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.879861:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.887793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.887849:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.887858:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513081.887867:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.232@tcp1 : PUT - for me 00000800:00000200:7.2:1616513081.887874:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513081.887877:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.232@tcp1 of length 224 into portal 26 MB=0x5ffd451bf86c0 00000400:00000200:16.0:1616513081.887884:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.232@tcp1 of length 224/224 into md 0xf97c97f9 [1] + 0 00000400:00000200:16.0:1616513081.887888:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c6362a8 00000400:00000200:16.0:1616513081.887891:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.887894:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513081.887900:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6362a8 00000800:00000200:28.0:1616513081.887902:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.887903:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.887905:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.887909:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513081.887932:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.887941:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513081.887951:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.232@tcp1 : PUT - for me 00000400:00000200:22.0:1616513081.887960:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.232@tcp1 of length 224 into portal 12 MB=0x5ffd451bf8840 00000400:00000200:22.0:1616513081.887966:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.232@tcp1 of length 224/224 into md 0xf964d519 [1] + 15008 00000100:00000200:20.0:1616513081.887966:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662253209280 00000400:00000200:22.0:1616513081.887971:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513081.887973:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:20.0:1616513081.887977:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688662253209280/t0(0) o400->6e6d09f2-b8ea-8237-f285-e603d497fda8@10.31.132.232@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513081.887980:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.887983:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.887984:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513081.887987:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.887991:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662253209280, offset 192 00000400:00000200:20.0:1616513081.887997:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.232@tcp1 00000400:00000200:20.0:1616513081.888005:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.232@tcp1 routed destination 00000400:00000200:20.0:1616513081.888020:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.232@tcp1(10.31.132.232@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.888026:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.888032:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.888036:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.888039:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:30.0:1616513081.888040:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662253209664 00000800:00000200:20.0:1616513081.888044:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.888049:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.888050:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:30.0:1616513081.888051:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec200d80 x1688662253209664/t0(0) o400->ec6535da-fd46-fe76-450f-5d560287ca12@10.31.132.232@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513081.888065:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662253209664, offset 192 00000400:00000200:30.0:1616513081.888070:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.232@tcp1 00000400:00000200:30.0:1616513081.888077:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.232@tcp1 routed destination 00000400:00000200:30.0:1616513081.888092:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.232@tcp1(10.31.132.232@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513081.888098:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513081.888103:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513081.888106:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.888107:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513081.888109:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513081.888114:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513081.888114:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513081.888116:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513081.888119:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513081.888123:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513081.888129:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aff0 00000800:00000200:26.0:1616513081.888132:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513081.888135:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513081.888180:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.888187:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.888189:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.888196:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.888200:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eea18 00000800:00000200:24.0:1616513081.888204:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.888207:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.922638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513081.922706:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.922715:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513081.922724:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.176@tcp1 : PUT - for me 00000400:00000200:28.0:1616513081.922734:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.176@tcp1 of length 224 into portal 26 MB=0x5ffd4b8d4af40 00000400:00000200:28.0:1616513081.922741:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.176@tcp1 of length 224/224 into md 0xf97c9809 [1] + 0 00000400:00000200:28.0:1616513081.922756:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c6373a8 00000400:00000200:28.0:1616513081.922759:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513081.922762:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513081.922766:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6373a8 00000800:00000200:28.0:1616513081.922769:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.922771:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.922772:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513081.922775:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.922842:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663982649152 00010000:00000200:20.0:1616513081.922853:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688663982649152/t0(0) o400->11626a65-6a74-8cac-8e0a-bda2724283af@10.31.138.176@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.922867:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663982649152, offset 192 00000400:00000200:20.0:1616513081.922874:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.176@tcp1 00000400:00000200:20.0:1616513081.922881:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.176@tcp1 routed destination 00000400:00000200:20.0:1616513081.922896:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.176@tcp1(10.31.138.176@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.922903:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.922909:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.922912:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513081.922912:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.922914:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.922919:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.922925:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.922927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513081.922966:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.922976:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.922984:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.176@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.922995:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.176@tcp1 of length 224 into portal 12 MB=0x5ffd4b8d4b0c0 00000800:00000200:22.0:1616513081.922995:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513081.923001:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.176@tcp1 of length 224/224 into md 0x865bbec1 [1] + 44912 00000800:00000200:22.0:1616513081.923001:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.923004:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.923007:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.923009:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:22.0:1616513081.923011:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513081.923016:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513081.923017:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bc28 00000800:00000200:24.0:1616513081.923019:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.923021:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513081.923021:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.923024:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513081.923024:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.923043:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663982649536 00010000:00000200:0.0:1616513081.923055:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd702d00 x1688663982649536/t0(0) o400->a715283a-ea25-6ee2-4768-501dd365313e@10.31.138.176@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.923069:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663982649536, offset 192 00000400:00000200:0.0:1616513081.923075:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.176@tcp1 00000400:00000200:0.0:1616513081.923083:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.176@tcp1 routed destination 00000400:00000200:0.0:1616513081.923098:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.176@tcp1(10.31.138.176@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.923105:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.923111:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.923115:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.923118:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.923123:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.923127:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.923138:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.923143:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.923146:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513081.923148:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.923152:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.923156:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05298 00000800:00000200:16.0:1616513081.923160:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.940775:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513081.940842:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513081.940852:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513081.940861:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.98@tcp1 : PUT - for me 00000400:00000200:26.0:1616513081.940871:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.98@tcp1 of length 224 into portal 26 MB=0x5ffd4530e1640 00000400:00000200:26.0:1616513081.940878:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.98@tcp1 of length 224/224 into md 0xf97c9819 [1] + 0 00000400:00000200:26.0:1616513081.940882:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c6376d8 00000400:00000200:26.0:1616513081.940885:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513081.940888:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513081.940892:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c6376d8 00000800:00000200:26.0:1616513081.940895:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.940896:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.940898:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513081.940904:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513081.940907:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.940907:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.940916:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.98@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.940925:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.98@tcp1 of length 224 into portal 12 MB=0x5ffd4530e16c0 00000400:00000200:2.0:1616513081.940931:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.98@tcp1 of length 224/224 into md 0xf964d519 [1] + 15232 00000400:00000200:2.0:1616513081.940937:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.940940:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513081.940947:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.940952:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.940954:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.940955:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662275135040 00000100:00000200:30.0:1616513081.940967:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662275135168 00010000:00000200:20.0:1616513081.940967:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688662275135040/t0(0) o400->e2f07646-647d-8679-b6d9-343029a7f242@10.31.138.98@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513081.940979:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21f500 x1688662275135168/t0(0) o400->7212ee63-2820-56d4-1e0b-601a39fe8d9d@10.31.138.98@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513081.940981:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662275135040, offset 192 00000400:00000200:20.0:1616513081.940987:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.98@tcp1 00000100:00000200:30.0:1616513081.940994:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662275135168, offset 192 00000400:00000200:20.0:1616513081.940995:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.98@tcp1 routed destination 00000400:00000200:30.0:1616513081.941001:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.98@tcp1 00000400:00000200:20.0:1616513081.941010:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.98@tcp1(10.31.138.98@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.941016:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:30.0:1616513081.941019:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.98@tcp1 routed destination 00000800:00000200:20.0:1616513081.941022:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.941025:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.941027:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.941032:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513081.941035:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.98@tcp1(10.31.138.98@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513081.941037:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.941038:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513081.941041:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513081.941047:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513081.941050:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513081.941052:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513081.941056:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513081.941091:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.941099:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.941101:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513081.941103:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513081.941105:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513081.941108:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.941115:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b5c8 00000800:00000200:14.0:1616513081.941119:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513081.941162:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.941169:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.941172:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.941178:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.941182:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef5c8 00000800:00000200:24.0:1616513081.941187:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.941190:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.961690:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513081.961746:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.961755:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.961764:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.145@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.961774:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.145@tcp1 of length 224 into portal 26 MB=0x60183150b6740 00000400:00000200:2.0:1616513081.961780:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.145@tcp1 of length 224/224 into md 0xf97c9829 [1] + 0 00000400:00000200:2.0:1616513081.961785:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637430 00000400:00000200:2.0:1616513081.961788:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.961790:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513081.961794:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637430 00000800:00000200:2.0:1616513081.961797:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.961800:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.961800:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.961804:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.961858:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512365676352 00000800:00000200:7.2:1616513081.961868:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:20.0:1616513081.961869:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1690512365676352/t0(0) o400->55922baf-9906-abcc-b0d3-2db519e50f92@10.31.136.145@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.961884:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512365676352, offset 192 00000400:00000200:20.0:1616513081.961890:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.145@tcp1 00000400:00000200:20.0:1616513081.961898:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.145@tcp1 routed destination 00000400:00000200:20.0:1616513081.961913:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.145@tcp1(10.31.136.145@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.961920:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513081.961921:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.961925:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.961928:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.961930:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.961930:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:20.0:1616513081.961935:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513081.961938:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.145@tcp1 : PUT - for me 00000100:00000200:20.0:1616513081.961942:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513081.961946:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.145@tcp1 of length 224 into portal 12 MB=0x60183150b6800 00000400:00000200:26.0:1616513081.961952:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.145@tcp1 of length 224/224 into md 0xf964d519 [1] + 15456 00000400:00000200:26.0:1616513081.961959:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513081.961961:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513081.961967:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513081.961970:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513081.961972:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513081.961975:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.961983:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:30.0:1616513081.961985:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512365676544 00000400:00000200:14.0:1616513081.961986:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513081.961987:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.961993:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:30.0:1616513081.961995:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc578d80 x1690512365676544/t0(0) o400->1272a2a6-d324-cbc0-fad1-f520fb1da045@10.31.136.145@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513081.961998:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79ba08 00000800:00000200:14.0:1616513081.962002:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513081.962008:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512365676544, offset 192 00000400:00000200:30.0:1616513081.962014:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.145@tcp1 00000400:00000200:30.0:1616513081.962020:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.145@tcp1 routed destination 00000400:00000200:30.0:1616513081.962034:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.145@tcp1(10.31.136.145@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513081.962040:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513081.962045:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513081.962047:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513081.962049:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513081.962054:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.962099:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.962156:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.962163:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.962166:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513081.962172:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.962176:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeb28 00000800:00000200:28.0:1616513081.962180:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.962183:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.964855:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513081.964909:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.964918:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.964927:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.175@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.964936:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.175@tcp1 of length 224 into portal 26 MB=0x5ffde313e9a40 00000400:00000200:2.0:1616513081.964942:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.175@tcp1 of length 224/224 into md 0xf97c9839 [1] + 0 00000400:00000200:2.0:1616513081.964947:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636b28 00000400:00000200:2.0:1616513081.964950:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513081.964952:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513081.964956:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636b28 00000800:00000200:2.0:1616513081.964959:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.964962:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.964962:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.964968:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.964971:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513081.964972:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513081.964981:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.175@tcp1 : PUT - for me 00000400:00000200:26.0:1616513081.964989:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.175@tcp1 of length 224 into portal 12 MB=0x5ffde313e9ac0 00000400:00000200:26.0:1616513081.964995:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.175@tcp1 of length 224/224 into md 0xf964d519 [1] + 15680 00000400:00000200:26.0:1616513081.965001:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513081.965003:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513081.965010:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513081.965013:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513081.965015:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513081.965019:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704657562304 00000100:00000200:20.0:1616513081.965020:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704657562176 00010000:00000200:30.0:1616513081.965025:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57e300 x1688704657562304/t0(0) o400->7074f470-b056-0732-54a5-15ced36acccd@10.31.136.175@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:20.0:1616513081.965031:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688704657562176/t0(0) o400->5233f495-2558-f805-241d-a91fc0f6cf4d@10.31.136.175@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513081.965036:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704657562304, offset 192 00000400:00000200:30.0:1616513081.965042:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.175@tcp1 00000100:00000200:20.0:1616513081.965045:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704657562176, offset 192 00000400:00000200:30.0:1616513081.965048:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.175@tcp1 routed destination 00000400:00000200:20.0:1616513081.965051:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.175@tcp1 00000400:00000200:20.0:1616513081.965059:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.175@tcp1 routed destination 00000400:00000200:30.0:1616513081.965061:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.175@tcp1(10.31.136.175@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513081.965067:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513081.965071:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513081.965074:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513081.965074:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.175@tcp1(10.31.136.175@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513081.965075:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513081.965078:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513081.965080:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513081.965085:0:49657:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.965086:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.965089:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.965091:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513081.965096:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513081.965096:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:20.0:1616513081.965102:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513081.965103:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.965106:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513081.965107:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.965112:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.965116:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee2a8 00000800:00000200:14.0:1616513081.965120:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.965141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.965197:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.965204:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.965207:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513081.965213:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513081.965219:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b760 00000800:00000200:24.0:1616513081.965222:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.965225:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.965231:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513081.965233:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.965235:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.965236:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.965239:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.975826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.975884:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.975893:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513081.975902:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.185@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.975911:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.185@tcp1 of length 224 into portal 26 MB=0x5ffd4480ee900 00000400:00000200:16.0:1616513081.975918:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.185@tcp1 of length 224/224 into md 0xf97c9869 [1] + 0 00000400:00000200:16.0:1616513081.975923:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636440 00000400:00000200:16.0:1616513081.975925:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.975928:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513081.975933:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636440 00000800:00000200:16.0:1616513081.975935:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.975938:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.975939:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.975942:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.975952:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662090639616 00010000:00000200:20.0:1616513081.975963:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688662090639616/t0(0) o400->39ba76b2-85ab-8d91-f742-ffcf1172417f@10.31.131.185@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.975976:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662090639616, offset 192 00000400:00000200:20.0:1616513081.975983:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.185@tcp1 00000400:00000200:20.0:1616513081.975990:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.185@tcp1 routed destination 00000400:00000200:20.0:1616513081.976004:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.185@tcp1(10.31.131.185@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.976010:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.976015:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.976018:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.976020:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.976024:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.976030:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.976033:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513081.976046:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513081.976047:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.976054:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513081.976057:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513081.976059:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513081.976060:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513081.976063:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513081.976069:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aaa0 00000800:00000200:12.0:1616513081.976069:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513081.976073:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513081.976077:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.185@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.976086:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.185@tcp1 of length 224 into portal 12 MB=0x5ffd4480eea00 00000400:00000200:12.0:1616513081.976092:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.185@tcp1 of length 224/224 into md 0x865bbec1 [1] + 45136 00000400:00000200:12.0:1616513081.976097:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.976100:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513081.976107:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.976110:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513081.976112:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.976115:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.976175:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662090639872 00010000:00000200:0.0:1616513081.976187:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dd100 x1688662090639872/t0(0) o400->1ec6fe03-417c-28ac-57be-381a2fda56c7@10.31.131.185@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513081.976188:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.976199:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513081.976202:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662090639872, offset 192 00000800:00000200:24.0:1616513081.976207:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513081.976208:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513081.976208:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.185@tcp1 00000400:00000200:24.0:1616513081.976216:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.1@tcp1 : PUT - for me 00000400:00000200:0.0:1616513081.976216:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.185@tcp1 routed destination 00000400:00000200:0.0:1616513081.976231:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.185@tcp1(10.31.131.185@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.976237:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.976240:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.1@tcp1 of length 224 into portal 26 MB=0x600117fc62400 00000800:00000200:0.0:1616513081.976243:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513081.976246:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.1@tcp1 of length 224/224 into md 0xf97c9879 [1] + 0 00000800:00000200:0.0:1616513081.976246:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.976248:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513081.976251:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637540 00000800:00000200:0.0:1616513081.976253:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513081.976256:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:11.2:1616513081.976258:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513081.976259:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513081.976263:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637540 00000800:00000200:24.0:1616513081.976266:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513081.976267:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513081.976269:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000100:00000200:20.0:1616513081.976279:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688925018399744 00000800:00000200:24.0:1616513081.976280:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513081.976281:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513081.976283:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513081.976283:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:20.0:1616513081.976284:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688925018399744/t0(0) o400->6044c7a1-3930-feec-4573-4b84494c74e2@10.31.142.1@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513081.976288:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.976291:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05980 00000100:00000200:20.0:1616513081.976294:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688925018399744, offset 192 00000800:00000200:12.0:1616513081.976295:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:20.0:1616513081.976299:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.1@tcp1 00000400:00000200:20.0:1616513081.976304:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.1@tcp1 routed destination 00000400:00000200:20.0:1616513081.976316:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.1@tcp1(10.31.142.1@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.976321:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.976325:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.976327:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.976329:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.976332:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.976336:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.976337:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.976392:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.976399:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513081.976402:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513081.976409:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513081.976414:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aee0 00000800:00000200:2.0:1616513081.976420:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513081.976423:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.976481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.976490:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.976495:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513081.976499:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.976503:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.1@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.976509:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.1@tcp1 of length 224 into portal 12 MB=0x600117fc62500 00000400:00000200:16.0:1616513081.976515:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.1@tcp1 of length 224/224 into md 0x865bbec1 [1] + 45360 00000400:00000200:16.0:1616513081.976519:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.976521:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513081.976527:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.976529:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513081.976531:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.976539:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688925018400000 00010000:00000200:0.0:1616513081.976545:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd0480 x1688925018400000/t0(0) o400->4342488e-d5f0-d2d6-b10a-c84760c4d008@10.31.142.1@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.976557:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688925018400000, offset 192 00000400:00000200:0.0:1616513081.976562:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.1@tcp1 00000400:00000200:0.0:1616513081.976567:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.1@tcp1 routed destination 00000400:00000200:0.0:1616513081.976580:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.1@tcp1(10.31.142.1@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.976586:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.976591:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.976593:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.976595:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.976598:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.976603:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.976613:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.976618:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.976621:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513081.976622:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.976628:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.976632:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb043b8 00000800:00000200:28.0:1616513081.976636:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.977396:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.977406:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.977412:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513081.977414:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513081.977419:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.25@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.977426:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.25@tcp1 of length 224 into portal 26 MB=0x602bd3e33c3c0 00000400:00000200:24.0:1616513081.977432:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.25@tcp1 of length 224/224 into md 0xf97c9899 [1] + 0 00000400:00000200:24.0:1616513081.977448:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637dc0 00000400:00000200:24.0:1616513081.977450:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.977452:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513081.977457:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637dc0 00000800:00000200:24.0:1616513081.977459:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.977461:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.977462:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.977541:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:20.0:1616513081.977549:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691861675918272 00000800:00000200:12.0:1616513081.977549:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.977555:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513081.977557:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:20.0:1616513081.977560:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1691861675918272/t0(0) o400->15ed0207-350a-52f9-1ee5-2d69cfc4a52a@10.31.129.25@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513081.977562:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.25@tcp1 : PUT - for me 00000400:00000200:12.0:1616513081.977569:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.25@tcp1 of length 224 into portal 12 MB=0x602bd3e33c900 00000400:00000200:12.0:1616513081.977574:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.25@tcp1 of length 224/224 into md 0x865bbec1 [1] + 45584 00000100:00000200:20.0:1616513081.977578:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691861675918272, offset 192 00000400:00000200:12.0:1616513081.977578:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513081.977580:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513081.977584:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513081.977587:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.25@tcp1 00000800:00000200:12.0:1616513081.977592:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513081.977594:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.25@tcp1 routed destination 00000800:00000200:12.0:1616513081.977594:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.977596:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691861675919616 00010000:00000200:0.0:1616513081.977602:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d8900 x1691861675919616/t0(0) o400->d0f17b10-d83e-92aa-2aed-e9504a603d5b@10.31.129.25@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513081.977609:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.25@tcp1(10.31.129.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.977615:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:0.0:1616513081.977622:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691861675919616, offset 192 00000800:00000200:20.0:1616513081.977623:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.977626:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.977628:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513081.977628:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.25@tcp1 00000800:00000200:20.0:1616513081.977632:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513081.977633:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.25@tcp1 routed destination 00000100:00000200:20.0:1616513081.977639:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.977639:0:49658:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513081.977646:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.25@tcp1(10.31.129.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.977651:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.977661:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.977663:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.977665:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.977667:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.977673:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.977681:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.977687:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513081.977689:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513081.977691:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513081.977694:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513081.977698:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb042a8 00000800:00000200:16.0:1616513081.977700:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513081.977702:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.977709:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.977712:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.977718:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.977724:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b7e8 00000800:00000200:22.0:1616513081.977728:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.977730:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513081.977736:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513081.977739:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.977741:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513081.977742:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.977747:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.978236:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513081.978245:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.978251:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513081.978253:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513081.978259:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.207@tcp1 : PUT - for me 00000400:00000200:28.0:1616513081.978268:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.207@tcp1 of length 224 into portal 26 MB=0x5ffde03df0980 00000400:00000200:28.0:1616513081.978275:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.207@tcp1 of length 224/224 into md 0xf97c98a9 [1] + 0 00000400:00000200:28.0:1616513081.978279:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637a90 00000400:00000200:28.0:1616513081.978282:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513081.978284:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513081.978289:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637a90 00000800:00000200:28.0:1616513081.978291:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.978294:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.978298:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513081.978301:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703896324480 00010000:00000200:20.0:1616513081.978307:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688703896324480/t0(0) o400->2fce34cb-2f78-27c3-b532-0a375604d04e@10.31.128.207@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.978318:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703896324480, offset 192 00000400:00000200:20.0:1616513081.978324:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.207@tcp1 00000400:00000200:20.0:1616513081.978330:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.207@tcp1 routed destination 00000400:00000200:20.0:1616513081.978343:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.207@tcp1(10.31.128.207@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.978349:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.978354:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.978356:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.978358:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.978361:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513081.978365:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513081.978366:0:49658:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513081.978377:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513081.978384:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513081.978387:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513081.978388:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513081.978393:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513081.978399:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b8f8 00000800:00000200:14.0:1616513081.978402:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.978865:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.978918:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.978927:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.978935:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.207@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.978944:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.207@tcp1 of length 224 into portal 12 MB=0x5ffde03df0dc0 00000400:00000200:24.0:1616513081.978950:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.207@tcp1 of length 224/224 into md 0x865bbec1 [1] + 45808 00000400:00000200:24.0:1616513081.978956:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.978958:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.978965:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.978968:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.978970:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.978972:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.979031:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703896325568 00010000:00000200:0.0:1616513081.979043:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd700900 x1688703896325568/t0(0) o400->4654ee5f-2d87-2ee1-9496-efaeb36ae57a@10.31.128.207@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.979058:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703896325568, offset 192 00000400:00000200:0.0:1616513081.979064:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.207@tcp1 00000400:00000200:0.0:1616513081.979071:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.207@tcp1 routed destination 00000400:00000200:0.0:1616513081.979086:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.207@tcp1(10.31.128.207@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.979092:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.979098:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.979101:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.979103:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.979108:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.979113:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513081.979122:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513081.979125:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513081.979127:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513081.979130:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513081.979132:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513081.979135:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04550 00000800:00000200:12.0:1616513081.979137:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513081.980151:0:49658:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513081.980205:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513081.980214:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513081.980223:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.149@tcp1 : PUT - for me 00000400:00000200:26.0:1616513081.980232:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.149@tcp1 of length 224 into portal 26 MB=0x601969131f500 00000400:00000200:26.0:1616513081.980239:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.149@tcp1 of length 224/224 into md 0xf97c98b9 [1] + 0 00000400:00000200:26.0:1616513081.980244:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637100 00000400:00000200:26.0:1616513081.980246:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513081.980249:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513081.980253:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637100 00000800:00000200:26.0:1616513081.980256:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513081.980259:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.980259:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513081.980263:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513081.980358:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690596052956416 00010000:00000200:20.0:1616513081.980368:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1690596052956416/t0(0) o400->36cf1d67-76c7-9f5a-da88-e6a35031050e@10.31.140.149@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513081.980381:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690596052956416, offset 192 00000400:00000200:20.0:1616513081.980387:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.149@tcp1 00000400:00000200:20.0:1616513081.980394:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.149@tcp1 routed destination 00000400:00000200:20.0:1616513081.980408:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.149@tcp1(10.31.140.149@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.980414:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513081.980420:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.980422:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513081.980424:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513081.980428:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.980435:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.980449:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513081.980488:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513081.980495:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.980497:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513081.980504:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.980509:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a990 00000800:00000200:22.0:1616513081.980513:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513081.980516:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513081.980582:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513081.980636:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513081.980647:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513081.980656:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.149@tcp1 : PUT - for me 00000400:00000200:24.0:1616513081.980665:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.149@tcp1 of length 224 into portal 12 MB=0x601969131f900 00000400:00000200:24.0:1616513081.980671:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.149@tcp1 of length 224/224 into md 0x865bbec1 [1] + 46032 00000400:00000200:24.0:1616513081.980676:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513081.980678:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513081.980685:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513081.980688:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513081.980689:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.980692:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513081.980748:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690596052957440 00010000:00000200:0.0:1616513081.980760:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd701680 x1690596052957440/t0(0) o400->3c66cd13-f5d3-b930-241f-d52b4a2a3411@10.31.140.149@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513081.980774:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690596052957440, offset 192 00000400:00000200:0.0:1616513081.980781:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.149@tcp1 00000400:00000200:0.0:1616513081.980788:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.149@tcp1 routed destination 00000400:00000200:0.0:1616513081.980803:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.149@tcp1(10.31.140.149@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.980815:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513081.980821:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.980824:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513081.980826:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513081.980831:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.980836:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513081.980888:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.980895:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.980898:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513081.980904:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.980908:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb056d8 00000800:00000200:28.0:1616513081.980912:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.980915:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513081.985448:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513081.985511:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513081.985520:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513081.985521:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513081.985529:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.168@tcp1 : PUT - for me 00000400:00000200:16.0:1616513081.985537:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.168@tcp1 of length 224 into portal 12 MB=0x5fba1ad0d0e40 00000400:00000200:16.0:1616513081.985543:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.168@tcp1 of length 224/224 into md 0x865bbec1 [1] + 46256 00000400:00000200:16.0:1616513081.985548:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513081.985551:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513081.985558:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513081.985561:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513081.985563:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513081.985565:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513081.985585:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513081.985593:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513081.985602:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.168@tcp1 : PUT - for me 00000400:00000200:2.0:1616513081.985609:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.168@tcp1 of length 224 into portal 26 MB=0x5fba1ad0d0dc0 00000400:00000200:2.0:1616513081.985615:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.168@tcp1 of length 224/224 into md 0xf97c98f9 [1] + 0 00000400:00000200:2.0:1616513081.985620:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636dd0 00000400:00000200:2.0:1616513081.985622:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513081.985624:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1684046695173696 00000100:00000200:2.0:1616513081.985625:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513081.985629:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636dd0 00000800:00000200:2.0:1616513081.985632:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513081.985634:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513081.985636:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513081.985636:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd7080 x1684046695173696/t0(0) o400->71f27c5b-95f1-60a9-0f41-13630c8e684d@10.31.138.168@tcp1:707/0 lens 224/224 e 0 to 0 dl 1616513087 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513081.985638:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513081.985651:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1684046695173696, offset 192 00000400:00000200:0.0:1616513081.985657:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.168@tcp1 00000400:00000200:0.0:1616513081.985664:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.168@tcp1 routed destination 00000400:00000200:0.0:1616513081.985679:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.168@tcp1(10.31.138.168@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513081.985686:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:20.0:1616513081.985691:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1684046695173568 00000800:00000200:0.0:1616513081.985692:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513081.985695:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513081.985697:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:20.0:1616513081.985701:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1684046695173568/t0(0) o400->71317d7a-e509-d187-bfa3-942733a09e49@10.31.138.168@tcp1:38/0 lens 224/224 e 0 to 0 dl 1616513173 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513081.985702:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513081.985707:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513081.985715:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1684046695173568, offset 192 00000400:00000200:20.0:1616513081.985720:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.168@tcp1 00000400:00000200:20.0:1616513081.985727:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.168@tcp1 routed destination 00000400:00000200:20.0:1616513081.985741:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.168@tcp1(10.31.138.168@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513081.985747:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513081.985752:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513081.985755:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513081.985757:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513081.985761:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513081.985761:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513081.985766:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513081.985767:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513081.985769:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513081.985771:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513081.985774:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513081.985778:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513081.985782:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05188 00000800:00000200:22.0:1616513081.985786:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513081.985820:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513081.985827:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513081.985829:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513081.985836:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513081.985841:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a6e8 00000800:00000200:28.0:1616513081.985845:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513081.985847:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.008110:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513082.008138:0:49661:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.008165:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.008174:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513082.008184:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.79@tcp1 : PUT - for me 00000400:00000200:16.0:1616513082.008193:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.79@tcp1 of length 224 into portal 12 MB=0x5ffdd31f41bc0 00000800:00000200:26.0:1616513082.008196:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513082.008200:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.79@tcp1 of length 224/224 into md 0x865bbec1 [1] + 46480 00000400:00000200:16.0:1616513082.008205:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513082.008206:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:16.0:1616513082.008207:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513082.008214:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513082.008216:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.79@tcp1 : PUT - for me 00000800:00000200:16.0:1616513082.008217:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513082.008219:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513082.008222:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513082.008224:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.79@tcp1 of length 224 into portal 26 MB=0x5ffdd31f41900 00000400:00000200:26.0:1616513082.008231:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.79@tcp1 of length 224/224 into md 0xf97c9919 [1] + 0 00000400:00000200:26.0:1616513082.008236:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c636330 00000400:00000200:26.0:1616513082.008239:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.008241:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513082.008246:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c636330 00000800:00000200:26.0:1616513082.008249:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.008252:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.008252:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.008256:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513082.008288:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700374490048 00010000:00000200:0.0:1616513082.008301:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd7980 x1688700374490048/t0(0) o400->516125e8-bd4e-fcfc-e8db-93a6b3276f07@10.31.142.79@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513082.008314:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700374489344 00000100:00000200:0.0:1616513082.008315:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700374490048, offset 192 00000400:00000200:0.0:1616513082.008321:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.79@tcp1 00010000:00000200:20.0:1616513082.008325:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688700374489344/t0(0) o400->f869b130-474d-84ec-e7c9-abc15445e8fb@10.31.142.79@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513082.008329:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.79@tcp1 routed destination 00000100:00000200:20.0:1616513082.008340:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700374489344, offset 192 00000400:00000200:0.0:1616513082.008344:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.79@tcp1(10.31.142.79@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513082.008347:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.79@tcp1 00000800:00000200:0.0:1616513082.008351:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513082.008354:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.79@tcp1 routed destination 00000400:00000200:20.0:1616513082.008369:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.79@tcp1(10.31.142.79@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513082.008376:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.008376:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.008379:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513082.008381:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.008381:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513082.008384:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.008386:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.008386:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.008390:0:49661:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513082.008391:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.008396:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513082.008398:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513082.008452:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.008460:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.008463:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.008464:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513082.008469:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513082.008471:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513082.008473:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a08 00000400:00000200:28.0:1616513082.008474:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513082.008477:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513082.008480:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513082.008480:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513082.008486:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a6e8 00000800:00000200:28.0:1616513082.008489:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.008492:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.015005:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.015060:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.015069:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.015078:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.8@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.015087:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.8@tcp1 of length 224 into portal 26 MB=0x5ffd5c0b68080 00000400:00000200:24.0:1616513082.015094:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.8@tcp1 of length 224/224 into md 0xf97c9929 [1] + 0 00000400:00000200:24.0:1616513082.015099:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d575c637ed0 00000400:00000200:24.0:1616513082.015101:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.015104:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.015108:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d575c637ed0 00000800:00000200:24.0:1616513082.015111:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.015114:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513082.015114:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513082.015121:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.015124:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513082.015125:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513082.015133:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.8@tcp1 : PUT - for me 00000400:00000200:16.0:1616513082.015142:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.8@tcp1 of length 224 into portal 12 MB=0x5ffd5c0b68100 00000400:00000200:16.0:1616513082.015148:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.8@tcp1 of length 224/224 into md 0x865bbec1 [1] + 46704 00000400:00000200:16.0:1616513082.015153:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513082.015158:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513082.015164:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.015167:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513082.015169:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513082.015182:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668409856128 00010000:00000200:20.0:1616513082.015193:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688668409856128/t0(0) o400->05122a2e-50de-ffe6-bcb7-bdd852b67f95@10.31.130.8@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513082.015207:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668409856128, offset 192 00000400:00000200:20.0:1616513082.015215:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.8@tcp1 00000400:00000200:20.0:1616513082.015222:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.8@tcp1 routed destination 00000100:00000200:0.0:1616513082.015228:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668409856256 00000400:00000200:20.0:1616513082.015237:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.8@tcp1(10.31.130.8@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513082.015240:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd3180 x1688668409856256/t0(0) o400->1ff147af-727e-4d92-c610-73eb75a25891@10.31.130.8@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513082.015243:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513082.015249:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.015252:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513082.015254:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513082.015254:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668409856256, offset 192 00000800:00000200:20.0:1616513082.015259:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513082.015261:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.8@tcp1 00000100:00000200:20.0:1616513082.015266:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.015267:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513082.015268:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.8@tcp1 routed destination 00000400:00000200:0.0:1616513082.015283:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.8@tcp1(10.31.130.8@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.015290:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.015296:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.015298:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.015301:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.015306:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.015311:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513082.015323:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.015329:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.015332:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.015339:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.015344:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b8f8 00000800:00000200:2.0:1616513082.015348:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.015351:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513082.015363:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.015371:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.015373:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513082.015375:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.015380:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.015384:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb054b8 00000800:00000200:12.0:1616513082.015388:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.023468:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.023527:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.023537:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.023546:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.84@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.023556:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.84@tcp1 of length 224 into portal 26 MB=0x6048d6a1acf00 00000400:00000200:14.0:1616513082.023563:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.84@tcp1 of length 224/224 into md 0xf97c9939 [1] + 0 00000400:00000200:14.0:1616513082.023568:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc13886e8 00000400:00000200:14.0:1616513082.023572:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.023575:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513082.023579:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13886e8 00000800:00000200:22.0:1616513082.023581:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.023583:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.023585:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513082.023587:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.023590:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.023591:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.023600:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.84@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.023609:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.84@tcp1 of length 224 into portal 12 MB=0x6048d6a1acf80 00000400:00000200:22.0:1616513082.023615:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.84@tcp1 of length 224/224 into md 0xf964d519 [1] + 15904 00000400:00000200:22.0:1616513082.023620:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.023623:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513082.023630:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.023633:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.023635:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513082.023643:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855277305600 00000100:00000200:30.0:1616513082.023649:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855277305728 00010000:00000200:20.0:1616513082.023654:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1693855277305600/t0(0) o400->3d7136c2-33ff-14e9-52a2-c7b7eef31425@10.31.140.84@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513082.023659:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec203600 x1693855277305728/t0(0) o400->772a3219-1522-9862-3b4c-b9e1f5912fa6@10.31.140.84@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513082.023669:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855277305600, offset 192 00000100:00000200:30.0:1616513082.023672:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855277305728, offset 192 00000400:00000200:20.0:1616513082.023675:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.84@tcp1 00000400:00000200:30.0:1616513082.023678:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.84@tcp1 00000400:00000200:20.0:1616513082.023683:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.84@tcp1 routed destination 00000400:00000200:30.0:1616513082.023684:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.84@tcp1 routed destination 00000400:00000200:30.0:1616513082.023699:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.84@tcp1(10.31.140.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513082.023700:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.84@tcp1(10.31.140.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.023706:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.023706:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513082.023711:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.023712:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.023713:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513082.023715:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.023715:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513082.023717:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513082.023720:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.023722:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513082.023727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513082.023727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513082.023728:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513082.023781:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.023789:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.023791:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513082.023793:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.023798:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513082.023800:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513082.023804:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aee0 00000800:00000200:26.0:1616513082.023807:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513082.023807:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513082.023809:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513082.023816:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513082.023820:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef298 00000800:00000200:16.0:1616513082.023824:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513082.023827:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.028526:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513082.028547:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513082.028581:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.028590:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.028599:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.70@tcp1 : PUT - for me 00000800:00000200:24.0:1616513082.028603:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513082.028608:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.70@tcp1 of length 224 into portal 26 MB=0x5ffe3859ec280 00000800:00000200:24.0:1616513082.028612:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513082.028615:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.70@tcp1 of length 224/224 into md 0xf97c9949 [1] + 0 00000400:00000200:22.0:1616513082.028619:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388198 00000400:00000200:24.0:1616513082.028621:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.70@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.028622:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.028624:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.028629:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.70@tcp1 of length 224 into portal 12 MB=0x5ffe3859ec680 00000400:00000200:22.0:1616513082.028630:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388198 00000800:00000200:22.0:1616513082.028633:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.028634:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513082.028636:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.70@tcp1 of length 224/224 into md 0x865bbec1 [1] + 46928 00000800:00000200:22.0:1616513082.028636:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.028638:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513082.028641:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.028644:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.028652:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.028655:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.028657:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513082.028659:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513082.028693:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727547986560 00010000:00000200:20.0:1616513082.028704:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688727547986560/t0(0) o400->1d30c471-4e1a-ba1b-b50c-4b64ec76578d@10.31.141.70@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513082.028717:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688727547986560, offset 192 00000100:00000200:0.0:1616513082.028719:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727547987584 00000400:00000200:20.0:1616513082.028723:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.70@tcp1 00000400:00000200:20.0:1616513082.028729:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.70@tcp1 routed destination 00010000:00000200:0.0:1616513082.028732:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd705a00 x1688727547987584/t0(0) o400->006ad115-9427-0186-2d90-eb2c309122fc@10.31.141.70@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513082.028743:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.70@tcp1(10.31.141.70@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513082.028746:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688727547987584, offset 192 00000800:00000200:20.0:1616513082.028750:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513082.028752:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.70@tcp1 00000800:00000200:20.0:1616513082.028756:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.028759:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513082.028759:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.70@tcp1 routed destination 00000800:00000200:20.0:1616513082.028761:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513082.028766:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.028771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513082.028772:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513082.028774:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.70@tcp1(10.31.141.70@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.028781:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.028786:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.028789:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.028791:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.028796:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.028801:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.028812:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.028818:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513082.028821:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513082.028822:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513082.028827:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513082.028827:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513082.028828:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:16.0:1616513082.028831:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb045d8 00000400:00000200:14.0:1616513082.028831:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513082.028835:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513082.028837:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.028842:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b760 00000800:00000200:14.0:1616513082.028845:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.028848:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.032313:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.032365:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.032374:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.032383:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.84@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.032392:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.84@tcp1 of length 224 into portal 12 MB=0x5ffd5dd266640 00000400:00000200:2.0:1616513082.032399:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.84@tcp1 of length 224/224 into md 0xf964d519 [1] + 16128 00000400:00000200:2.0:1616513082.032404:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.032406:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513082.032414:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.032418:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.032418:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.032422:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.032431:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668886951488 00010000:00000200:30.0:1616513082.032446:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21d100 x1688668886951488/t0(0) o400->540f92c2-abe0-7385-bd9f-4cafcdaa1632@10.31.136.84@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.032458:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668886951488, offset 192 00000400:00000200:30.0:1616513082.032464:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.84@tcp1 00000400:00000200:30.0:1616513082.032470:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.84@tcp1 routed destination 00000400:00000200:30.0:1616513082.032483:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.84@tcp1(10.31.136.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.032489:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513082.032493:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.032497:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513082.032498:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513082.032503:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.032508:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513082.032531:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513082.032558:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.032565:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.032568:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.032574:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.032577:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eedd0 00000800:00000200:14.0:1616513082.032581:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.032583:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513082.032590:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.032599:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.032608:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.84@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.032617:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.84@tcp1 of length 224 into portal 26 MB=0x5ffd5dd2665c0 00000400:00000200:24.0:1616513082.032623:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.84@tcp1 of length 224/224 into md 0xf97c9979 [1] + 0 00000400:00000200:24.0:1616513082.032628:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388880 00000400:00000200:24.0:1616513082.032631:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.032633:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.032638:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388880 00000800:00000200:24.0:1616513082.032641:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.032643:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513082.032645:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.032649:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513082.032700:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668886951360 00010000:00000200:20.0:1616513082.032710:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688668886951360/t0(0) o400->067f9d69-62bf-e60b-6547-9392a26b5d0b@10.31.136.84@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513082.032723:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668886951360, offset 192 00000400:00000200:20.0:1616513082.032729:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.84@tcp1 00000400:00000200:20.0:1616513082.032736:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.84@tcp1 routed destination 00000400:00000200:20.0:1616513082.032750:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.84@tcp1(10.31.136.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513082.032756:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.032762:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.032764:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.032766:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.032771:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.032776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513082.032777:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513082.032829:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.032836:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.032838:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.032845:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.032851:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b5c8 00000800:00000200:28.0:1616513082.032854:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.032857:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.042327:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513082.042381:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.042391:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513082.042399:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.192@tcp1 : PUT - for me 00000400:00000200:12.0:1616513082.042410:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.192@tcp1 of length 224 into portal 26 MB=0x5ffd45c594a80 00000400:00000200:12.0:1616513082.042416:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.192@tcp1 of length 224/224 into md 0xf97c9989 [1] + 0 00000400:00000200:12.0:1616513082.042421:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388f68 00000400:00000200:12.0:1616513082.042424:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513082.042426:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513082.042430:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388f68 00000800:00000200:12.0:1616513082.042446:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.042449:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513082.042449:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513082.042453:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513082.042491:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662431058560 00010000:00000200:20.0:1616513082.042501:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688662431058560/t0(0) o400->8d4f087c-81fb-9f3f-2772-0c91dc4336e3@10.31.136.192@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513082.042514:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662431058560, offset 192 00000400:00000200:20.0:1616513082.042520:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.192@tcp1 00000400:00000200:20.0:1616513082.042528:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.192@tcp1 routed destination 00000400:00000200:20.0:1616513082.042542:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.192@tcp1(10.31.136.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513082.042548:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513082.042553:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.042555:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513082.042557:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513082.042562:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513082.042570:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.042570:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.042624:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.042631:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.042634:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.042641:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.042646:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aff0 00000800:00000200:22.0:1616513082.042649:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.042652:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.043086:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.043140:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.043149:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.043157:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.192@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.043166:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.192@tcp1 of length 224 into portal 12 MB=0x5ffd45c594b00 00000400:00000200:28.0:1616513082.043175:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.192@tcp1 of length 224/224 into md 0x865bbec1 [1] + 47152 00000400:00000200:28.0:1616513082.043180:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.043182:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.043190:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.043192:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.043193:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.043196:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.043213:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662431058688 00010000:00000200:0.0:1616513082.043225:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedde80 x1688662431058688/t0(0) o400->7e8c109c-4ea4-45bb-e2ef-5f535eb58eba@10.31.136.192@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.043238:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662431058688, offset 192 00000400:00000200:0.0:1616513082.043244:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.192@tcp1 00000400:00000200:0.0:1616513082.043251:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.192@tcp1 routed destination 00000400:00000200:0.0:1616513082.043267:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.192@tcp1(10.31.136.192@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.043273:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.043278:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.043281:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.043283:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.043287:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.043293:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.043304:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.043308:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.043311:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513082.043313:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.043317:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.043321:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a90 00000800:00000200:24.0:1616513082.043324:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.046998:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.047014:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.047022:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.047031:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.84@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.047039:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.84@tcp1 of length 224 into portal 12 MB=0x5ffd82f136880 00000400:00000200:14.0:1616513082.047045:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.84@tcp1 of length 224/224 into md 0xf964d519 [1] + 16352 00000400:00000200:14.0:1616513082.047050:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.047052:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513082.047059:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.047061:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.047063:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.047066:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.047075:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678851373184 00010000:00000200:30.0:1616513082.047084:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba6c00 x1688678851373184/t0(0) o400->cafecc29-79dd-673b-f3be-efcf55d240b3@10.31.129.84@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.047096:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688678851373184, offset 192 00000400:00000200:30.0:1616513082.047101:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.84@tcp1 00000400:00000200:30.0:1616513082.047108:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.84@tcp1 routed destination 00000400:00000200:30.0:1616513082.047121:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.84@tcp1(10.31.129.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.047126:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513082.047131:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.047134:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513082.047135:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513082.047139:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.047144:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.047213:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.047219:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.047222:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.047229:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.047233:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee330 00000800:00000200:22.0:1616513082.047237:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.047240:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.047288:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.047342:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.047351:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513082.047360:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.84@tcp1 : PUT - for me 00000400:00000200:16.0:1616513082.047368:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.84@tcp1 of length 224 into portal 26 MB=0x5ffd82f136440 00000400:00000200:16.0:1616513082.047375:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.84@tcp1 of length 224/224 into md 0xf97c9999 [1] + 0 00000400:00000200:16.0:1616513082.047380:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc13884c8 00000400:00000200:16.0:1616513082.047382:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513082.047385:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513082.047389:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13884c8 00000800:00000200:16.0:1616513082.047392:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.047395:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513082.047396:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513082.047399:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513082.047459:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678851372096 00010000:00000200:20.0:1616513082.047468:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688678851372096/t0(0) o400->b719e47a-cf57-ad2b-ce58-953f819e1970@10.31.129.84@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513082.047481:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688678851372096, offset 192 00000400:00000200:20.0:1616513082.047486:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.84@tcp1 00000400:00000200:20.0:1616513082.047492:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.84@tcp1 routed destination 00000400:00000200:20.0:1616513082.047505:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.84@tcp1(10.31.129.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513082.047511:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.047516:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.047518:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.047520:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.047524:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513082.047530:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.047530:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.047582:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.047589:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.047591:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513082.047598:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.047605:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aff0 00000800:00000200:12.0:1616513082.047608:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513082.047611:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.048101:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.048111:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.048117:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513082.048121:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513082.048124:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.8.174@tcp1 : PUT - for me 00000400:00000200:16.0:1616513082.048134:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.8.174@tcp1 of length 224 into portal 12 MB=0x5ffa3b4529340 00000400:00000200:16.0:1616513082.048139:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.8.174@tcp1 of length 224/224 into md 0x865bbec1 [1] + 47376 00000400:00000200:16.0:1616513082.048143:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513082.048145:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513082.048150:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.048153:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513082.048154:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.048211:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688453453615936 00010000:00000200:0.0:1616513082.048222:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd6c00 x1688453453615936/t0(0) o400->80b12b36-77fb-9d06-ac8d-e569d0830eb4@10.31.8.174@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.048234:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688453453615936, offset 192 00000400:00000200:0.0:1616513082.048240:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.8.174@tcp1 00000400:00000200:0.0:1616513082.048247:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.8.174@tcp1 routed destination 00000800:00000200:7.2:1616513082.048253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513082.048260:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.8.174@tcp1(10.31.8.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.048265:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.048271:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.048273:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.048275:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513082.048279:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.048320:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.048330:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.048339:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.8.174@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.048348:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.8.174@tcp1 of length 224 into portal 26 MB=0x5ffa3b4529140 00000400:00000200:26.0:1616513082.048354:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.8.174@tcp1 of length 224/224 into md 0xf97c99a9 [1] + 0 00000400:00000200:26.0:1616513082.048359:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388088 00000400:00000200:26.0:1616513082.048361:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.048364:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513082.048369:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388088 00000800:00000200:26.0:1616513082.048372:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513082.048372:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513082.048374:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:20.0:1616513082.048380:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688453453615424 00000800:00000200:26.0:1616513082.048381:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513082.048382:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513082.048384:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00010000:00000200:20.0:1616513082.048385:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688453453615424/t0(0) o400->7a612242-bddb-18d1-ef07-4c8e0b4a8d9e@10.31.8.174@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513082.048385:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.048391:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.048395:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000100:00000200:20.0:1616513082.048398:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688453453615424, offset 192 00000800:00000200:14.0:1616513082.048399:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513082.048402:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.8.174@tcp1 00000400:00000200:20.0:1616513082.048408:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.8.174@tcp1 routed destination 00000400:00000200:20.0:1616513082.048419:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.8.174@tcp1(10.31.8.174@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513082.048424:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.048428:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.048430:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.048431:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.048440:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513082.048444:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.048444:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.048453:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.048457:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.048458:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513082.048462:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.048464:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.048467:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b760 00000800:00000200:12.0:1616513082.048469:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.074270:0:49668:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.074325:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.074334:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.074344:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.38@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.074352:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.38@tcp1 of length 224 into portal 12 MB=0x60182ecaf2f40 00000400:00000200:2.0:1616513082.074359:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.38@tcp1 of length 224/224 into md 0xf964d519 [1] + 16576 00000400:00000200:2.0:1616513082.074364:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.074367:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513082.074374:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.074377:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.074379:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.074385:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.074393:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511688544064 00010000:00000200:30.0:1616513082.074404:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21ad00 x1690511688544064/t0(0) o400->161ba4c4-46b4-d34e-042a-8b78260a1055@10.31.133.38@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.074419:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511688544064, offset 192 00000400:00000200:30.0:1616513082.074425:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.38@tcp1 00000400:00000200:30.0:1616513082.074432:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.38@tcp1 routed destination 00000800:00000200:11.2:1616513082.074440:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:30.0:1616513082.074452:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.38@tcp1(10.31.133.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.074459:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513082.074465:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.074467:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513082.074470:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513082.074474:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.074479:0:49668:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513082.074494:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.074507:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.074516:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.38@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.074525:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.38@tcp1 of length 224 into portal 26 MB=0x60182ecaf2ec0 00000400:00000200:28.0:1616513082.074531:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.38@tcp1 of length 224/224 into md 0xf97c99c9 [1] + 0 00000400:00000200:28.0:1616513082.074536:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc13885d8 00000400:00000200:28.0:1616513082.074539:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.074542:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.074546:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13885d8 00000800:00000200:28.0:1616513082.074549:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.074552:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.074554:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.074556:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513082.074561:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.074568:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.074571:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.074577:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.074581:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee330 00000800:00000200:14.0:1616513082.074585:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.074588:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513082.074606:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511688543936 00010000:00000200:20.0:1616513082.074615:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1690511688543936/t0(0) o400->8e6d94e5-e3dc-8952-67bd-68e484b1a087@10.31.133.38@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513082.074628:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511688543936, offset 192 00000400:00000200:20.0:1616513082.074635:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.38@tcp1 00000400:00000200:20.0:1616513082.074641:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.38@tcp1 routed destination 00000400:00000200:20.0:1616513082.074654:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.38@tcp1(10.31.133.38@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513082.074660:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.074665:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.074667:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.074669:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.074673:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:20.0:1616513082.074679:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.074679:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.074732:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.074739:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.074741:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513082.074747:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.074752:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b8f8 00000800:00000200:24.0:1616513082.074756:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.074758:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.083376:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.083431:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.083455:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513082.083464:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.158@tcp1 : PUT - for me 00000400:00000200:16.0:1616513082.083474:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.158@tcp1 of length 224 into portal 26 MB=0x5ffd572eafb80 00000400:00000200:16.0:1616513082.083481:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.158@tcp1 of length 224/224 into md 0xf97c99d9 [1] + 0 00000800:00000200:28.0:1616513082.083483:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513082.083485:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388e58 00000400:00000200:16.0:1616513082.083488:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513082.083491:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513082.083495:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388e58 00000800:00000200:16.0:1616513082.083498:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.083501:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513082.083503:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513082.083559:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667104672640 00010000:00000200:20.0:1616513082.083570:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688667104672640/t0(0) o400->817a977b-263f-2537-77ef-511aa7b08960@10.31.132.158@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513082.083585:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667104672640, offset 192 00000400:00000200:20.0:1616513082.083591:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.158@tcp1 00000400:00000200:20.0:1616513082.083599:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.158@tcp1 routed destination 00000400:00000200:20.0:1616513082.083614:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.158@tcp1(10.31.132.158@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513082.083621:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513082.083626:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.083629:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513082.083631:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513082.083636:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.083642:0:49669:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513082.083643:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513082.083697:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.083703:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.083706:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.083713:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.083718:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b8f8 00000800:00000200:26.0:1616513082.083722:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.083725:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.083730:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.083738:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.158@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.083746:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.158@tcp1 of length 224 into portal 12 MB=0x5ffd572eafbc0 00000800:00000200:2.0:1616513082.083749:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.083753:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.158@tcp1 of length 224/224 into md 0xf964d519 [1] + 16800 00000400:00000200:26.0:1616513082.083758:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.083760:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.083766:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.083769:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.083771:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.083782:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667104672704 00010000:00000200:30.0:1616513082.083791:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc579f80 x1688667104672704/t0(0) o400->a9dd001a-3bd9-f123-526b-0590a0fdd5f2@10.31.132.158@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.083813:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667104672704, offset 192 00000400:00000200:30.0:1616513082.083819:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.158@tcp1 00000400:00000200:30.0:1616513082.083825:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.158@tcp1 routed destination 00000400:00000200:30.0:1616513082.083839:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.158@tcp1(10.31.132.158@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.083845:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.083850:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.083853:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513082.083855:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513082.083859:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.083864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.083919:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.083926:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.083928:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513082.083932:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.083935:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.083941:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eedd0 00000800:00000200:24.0:1616513082.083945:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.092199:0:49670:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513082.092255:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.092264:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.092274:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.214@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.092283:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.214@tcp1 of length 224 into portal 26 MB=0x5fff28aa30640 00000400:00000200:22.0:1616513082.092299:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.214@tcp1 of length 224/224 into md 0xf97c99e9 [1] + 0 00000400:00000200:22.0:1616513082.092304:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388440 00000400:00000200:22.0:1616513082.092307:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.092310:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513082.092314:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388440 00000800:00000200:22.0:1616513082.092317:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.092317:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.092319:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.092324:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.092327:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.092327:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.092336:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.52@tcp1 : PUT - for me 00000800:00000200:7.2:1616513082.092337:0:49670:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:2.0:1616513082.092345:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.52@tcp1 of length 224 into portal 26 MB=0x603e2acb3de00 00000400:00000200:2.0:1616513082.092351:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.52@tcp1 of length 224/224 into md 0xf97c99f9 [1] + 0 00000400:00000200:2.0:1616513082.092356:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388ee0 00000400:00000200:2.0:1616513082.092359:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.092361:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.092366:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388ee0 00000800:00000200:2.0:1616513082.092369:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.092372:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.092374:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.092376:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513082.092378:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688792056661568 00000800:00000200:2.0:1616513082.092383:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513082.092386:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513082.092387:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693121955225088 00010000:00000200:20.0:1616513082.092389:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688792056661568/t0(0) o400->c1da89ad-86f6-608e-59b0-316fdff0f27d@10.31.136.214@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513082.092390:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.214@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.092396:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.214@tcp1 of length 224 into portal 12 MB=0x5fff28aa30740 00010000:00000200:8.0:1616513082.092397:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1693121955225088/t0(0) o400->78bd30e3-95da-0c16-581d-b417d3beca39@10.31.136.52@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513082.092402:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.214@tcp1 of length 224/224 into md 0xf964d519 [1] + 17024 00000100:00000200:20.0:1616513082.092403:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688792056661568, offset 192 00000400:00000200:2.0:1616513082.092406:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.092408:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513082.092410:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.214@tcp1 00000100:00000200:8.0:1616513082.092413:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693121955225088, offset 192 00000400:00000200:20.0:1616513082.092417:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.214@tcp1 routed destination 00000400:00000200:8.0:1616513082.092419:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.52@tcp1 00000800:00000200:2.0:1616513082.092421:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513082.092427:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.52@tcp1 routed destination 00000400:00000200:20.0:1616513082.092447:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.214@tcp1(10.31.136.214@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513082.092456:0:6472:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513082.092458:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513082.092460:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.092462:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.092465:0:49670:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513082.092473:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.52@tcp1(10.31.136.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513082.092474:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.092478:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:8.0:1616513082.092479:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.092480:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513082.092482:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513082.092483:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:30.0:1616513082.092484:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688792056661824 00000800:00000200:8.0:1616513082.092484:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513082.092485:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513082.092485:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513082.092487:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513082.092489:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.092490:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513082.092494:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513082.092494:0:49670:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:30.0:1616513082.092496:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21cc80 x1688792056661824/t0(0) o400->d8145951-ea47-d813-9519-b4821680cb07@10.31.136.214@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513082.092498:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.092500:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513082.092500:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513082.092504:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:30.0:1616513082.092511:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688792056661824, offset 192 00000800:00000200:14.0:1616513082.092511:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.092513:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513082.092515:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513082.092517:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.214@tcp1 00000400:00000200:14.0:1616513082.092520:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513082.092524:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.214@tcp1 routed destination 00000400:00000200:14.0:1616513082.092526:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aff0 00000400:00000200:30.0:1616513082.092539:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.214@tcp1(10.31.136.214@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513082.092544:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:30.0:1616513082.092545:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513082.092551:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.092554:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513082.092556:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513082.092556:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513082.092561:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513082.092563:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.092566:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513082.092566:0:49670:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.092569:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513082.092573:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513082.092574:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513082.092578:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f718f8 00000800:00000200:22.0:1616513082.092578:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.092579:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513082.092581:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513082.092582:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513082.092585:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.092588:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef298 00000800:00000200:22.0:1616513082.092591:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.093071:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513082.093124:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.093133:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513082.093143:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.52@tcp1 : PUT - for me 00000400:00000200:12.0:1616513082.093151:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.52@tcp1 of length 224 into portal 12 MB=0x603e2acb3dfc0 00000400:00000200:12.0:1616513082.093157:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.52@tcp1 of length 224/224 into md 0x865bbec1 [1] + 47600 00000400:00000200:12.0:1616513082.093163:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513082.093165:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513082.093172:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.093175:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.093177:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513082.093182:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.093197:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693121955225536 00010000:00000200:0.0:1616513082.093210:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dde80 x1693121955225536/t0(0) o400->41475bb5-13c4-9634-7e61-276635034d0e@10.31.136.52@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.093224:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693121955225536, offset 192 00000400:00000200:0.0:1616513082.093231:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.52@tcp1 00000400:00000200:0.0:1616513082.093238:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.52@tcp1 routed destination 00000400:00000200:0.0:1616513082.093253:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.52@tcp1(10.31.136.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.093260:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.093266:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.093269:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.093271:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.093278:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.093282:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.093293:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.093298:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513082.093301:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.093302:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513082.093307:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513082.093311:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04c38 00000800:00000200:16.0:1616513082.093315:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.106817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.106872:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513082.106878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513082.106881:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.106890:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.142@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.106900:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.142@tcp1 of length 224 into portal 26 MB=0x603fcd9cbed00 00000400:00000200:24.0:1616513082.106907:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.142@tcp1 of length 224/224 into md 0xf97c9a09 [1] + 0 00000400:00000200:24.0:1616513082.106912:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388220 00000400:00000200:24.0:1616513082.106914:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.106917:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.106922:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388220 00000800:00000200:24.0:1616513082.106925:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.106925:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.106927:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.106929:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513082.106939:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234380926208 00000800:00000200:2.0:1616513082.106947:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:8.0:1616513082.106948:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1693234380926208/t0(0) o400->2d5fc50d-4587-a11d-bec0-2ad9fb67a394@10.31.138.142@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513082.106956:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513082.106961:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234380926208, offset 192 00000400:00000200:2.0:1616513082.106964:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.142@tcp1 : PUT - for me 00000400:00000200:8.0:1616513082.106967:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.142@tcp1 00000400:00000200:2.0:1616513082.106973:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.142@tcp1 of length 224 into portal 12 MB=0x603fcd9cbed80 00000400:00000200:8.0:1616513082.106974:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.142@tcp1 routed destination 00000400:00000200:2.0:1616513082.106979:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.142@tcp1 of length 224/224 into md 0xf964d519 [1] + 17248 00000400:00000200:2.0:1616513082.106983:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.106986:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513082.106988:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.142@tcp1(10.31.138.142@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513082.106992:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513082.106994:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513082.106995:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.106997:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513082.106999:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513082.106999:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513082.107001:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513082.107004:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513082.107008:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:30.0:1616513082.107011:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234380926336 00000800:00000200:7.2:1616513082.107013:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513082.107014:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:30.0:1616513082.107022:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21ec00 x1693234380926336/t0(0) o400->fa279d97-ede4-0c89-1cc6-ed2fbe866c19@10.31.138.142@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.107036:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234380926336, offset 192 00000400:00000200:30.0:1616513082.107042:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.142@tcp1 00000400:00000200:30.0:1616513082.107049:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.142@tcp1 routed destination 00000400:00000200:30.0:1616513082.107063:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.142@tcp1(10.31.138.142@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.107069:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.107075:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513082.107076:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513082.107077:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513082.107079:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513082.107083:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513082.107084:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513082.107086:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513082.107089:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513082.107093:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.107098:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71e48 00000800:00000200:26.0:1616513082.107102:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.107105:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513082.107152:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.107159:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.107162:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.107168:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.107172:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee2a8 00000800:00000200:28.0:1616513082.107178:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.107181:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.116837:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.116901:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.116911:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513082.116915:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513082.116920:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.106@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.116930:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.106@tcp1 of length 224 into portal 26 MB=0x60466f0018980 00000400:00000200:2.0:1616513082.116936:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.106@tcp1 of length 224/224 into md 0xf97c9a29 [1] + 0 00000400:00000200:2.0:1616513082.116941:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388908 00000400:00000200:2.0:1616513082.116944:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.116947:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.116951:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388908 00000800:00000200:2.0:1616513082.116954:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.116956:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.116958:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513082.116968:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693690020071808 00010000:00000200:8.0:1616513082.116977:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1693690020071808/t0(0) o400->7aabed37-da9e-0d95-4ae6-737dbc3f6cac@10.31.141.106@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513082.116989:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693690020071808, offset 192 00000400:00000200:8.0:1616513082.116994:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.106@tcp1 00000400:00000200:8.0:1616513082.117001:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.106@tcp1 routed destination 00000400:00000200:8.0:1616513082.117014:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.106@tcp1(10.31.141.106@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513082.117020:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513082.117025:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513082.117028:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513082.117030:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513082.117034:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513082.117039:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.117039:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.117092:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.117099:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.117101:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.117108:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.117114:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70440 00000800:00000200:26.0:1616513082.117117:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.117120:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.117126:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513082.117128:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.117134:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.106@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.117144:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.106@tcp1 of length 224 into portal 12 MB=0x60466f0018bc0 00000400:00000200:26.0:1616513082.117150:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.106@tcp1 of length 224/224 into md 0xf964d519 [1] + 17472 00000400:00000200:26.0:1616513082.117155:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.117158:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.117164:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.117166:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.117168:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.117177:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693690020072384 00010000:00000200:30.0:1616513082.117185:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57b600 x1693690020072384/t0(0) o400->6bb1ad97-67f3-cc8a-c85f-d583d041edfd@10.31.141.106@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.117196:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693690020072384, offset 192 00000400:00000200:30.0:1616513082.117201:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.106@tcp1 00000400:00000200:30.0:1616513082.117208:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.106@tcp1 routed destination 00000400:00000200:30.0:1616513082.117220:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.106@tcp1(10.31.141.106@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.117226:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.117230:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.117232:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513082.117234:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513082.117237:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.117245:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.117300:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.117307:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.117309:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513082.117316:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.117320:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeb28 00000800:00000200:12.0:1616513082.117324:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513082.117327:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.150905:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.150961:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.150971:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.150980:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.217@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.150991:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.217@tcp1 of length 224 into portal 12 MB=0x604a46b6ad7c0 00000400:00000200:28.0:1616513082.150997:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.217@tcp1 of length 224/224 into md 0x865bbec1 [1] + 47824 00000400:00000200:28.0:1616513082.151002:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.151005:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.151012:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.151015:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513082.151015:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.151020:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.151027:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513082.151076:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954083575744 00000800:00000200:2.0:1616513082.151087:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513082.151089:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedad00 x1693954083575744/t0(0) o400->14c681f0-25e6-d2a8-6c02-27a632d336f8@10.31.140.217@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513082.151096:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:0.0:1616513082.151102:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954083575744, offset 192 00000400:00000200:2.0:1616513082.151105:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.217@tcp1 : PUT - for me 00000400:00000200:0.0:1616513082.151108:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.217@tcp1 00000400:00000200:2.0:1616513082.151115:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.217@tcp1 of length 224 into portal 26 MB=0x604a46b6ad580 00000400:00000200:0.0:1616513082.151116:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.217@tcp1 routed destination 00000400:00000200:2.0:1616513082.151122:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.217@tcp1 of length 224/224 into md 0xf97c9a39 [1] + 0 00000400:00000200:2.0:1616513082.151128:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388aa0 00000400:00000200:0.0:1616513082.151130:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.217@tcp1(10.31.140.217@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513082.151132:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.151134:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1616513082.151136:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.151139:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388aa0 00000800:00000200:14.0:1616513082.151141:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.151142:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513082.151142:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513082.151145:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513082.151145:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.151147:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513082.151147:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513082.151152:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.151156:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513082.151159:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954083575168 00010000:00000200:8.0:1616513082.151169:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1693954083575168/t0(0) o400->22597046-0239-56dd-8528-7fe58e91433e@10.31.140.217@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513082.151183:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954083575168, offset 192 00000400:00000200:8.0:1616513082.151189:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.217@tcp1 00000400:00000200:8.0:1616513082.151196:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.217@tcp1 routed destination 00000400:00000200:8.0:1616513082.151212:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.217@tcp1(10.31.140.217@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513082.151218:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513082.151223:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513082.151226:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513082.151226:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513082.151228:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513082.151233:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:8.0:1616513082.151233:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513082.151236:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513082.151237:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513082.151239:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513082.151242:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.151247:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb044c8 00000800:00000200:22.0:1616513082.151251:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.151254:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513082.151301:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.151308:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.151310:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513082.151317:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.151323:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71c28 00000800:00000200:12.0:1616513082.151326:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513082.151331:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.152482:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.152551:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.152560:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513082.152569:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.183@tcp1 : PUT - for me 00000400:00000200:16.0:1616513082.152579:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.183@tcp1 of length 224 into portal 26 MB=0x6058f1186edc0 00000400:00000200:16.0:1616513082.152586:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.183@tcp1 of length 224/224 into md 0xf97c9a69 [1] + 0 00000400:00000200:16.0:1616513082.152593:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388d48 00000400:00000200:16.0:1616513082.152596:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513082.152599:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513082.152604:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513082.152604:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388d48 00000800:00000200:16.0:1616513082.152607:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.152610:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513082.152612:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.152614:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:8.0:1616513082.152614:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694961892781504 00000800:00000200:16.0:1616513082.152615:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:8.0:1616513082.152619:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1694961892781504/t0(0) o400->5300488a-c702-fd81-f90b-b6874012f0cf@10.31.129.183@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513082.152623:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.183@tcp1 : PUT - for me 00000100:00000200:8.0:1616513082.152629:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694961892781504, offset 192 00000400:00000200:28.0:1616513082.152633:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.183@tcp1 of length 224 into portal 12 MB=0x6058f1186ee40 00000400:00000200:8.0:1616513082.152634:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.183@tcp1 00000400:00000200:28.0:1616513082.152639:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.183@tcp1 of length 224/224 into md 0x865bbec1 [1] + 48048 00000400:00000200:8.0:1616513082.152639:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.183@tcp1 routed destination 00000400:00000200:28.0:1616513082.152644:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.152647:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513082.152652:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.183@tcp1(10.31.129.183@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513082.152654:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.152657:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513082.152657:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513082.152659:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513082.152661:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513082.152664:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513082.152665:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513082.152668:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513082.152672:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.152686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513082.152723:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694961892781632 00010000:00000200:0.0:1616513082.152735:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedba80 x1694961892781632/t0(0) o400->a10eca1b-4674-3fd3-5fa3-fca4ea906731@10.31.129.183@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513082.152740:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.152746:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.152749:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1616513082.152750:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694961892781632, offset 192 00000400:00000200:14.0:1616513082.152756:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:0.0:1616513082.152756:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.183@tcp1 00000400:00000200:14.0:1616513082.152761:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70dd0 00000400:00000200:0.0:1616513082.152764:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.183@tcp1 routed destination 00000800:00000200:14.0:1616513082.152765:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.152768:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513082.152779:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.183@tcp1(10.31.129.183@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.152785:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.152791:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.152794:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.152796:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.152801:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.152805:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.152858:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.152865:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.152868:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513082.152869:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.152875:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.152879:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb058f8 00000800:00000200:24.0:1616513082.152883:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.158490:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513082.158545:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.158554:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.158563:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.117@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.158572:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.117@tcp1 of length 224 into portal 26 MB=0x5ffd74bcbca00 00000400:00000200:22.0:1616513082.158579:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.117@tcp1 of length 224/224 into md 0xf97c9a79 [1] + 0 00000400:00000200:22.0:1616513082.158584:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388770 00000400:00000200:22.0:1616513082.158587:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.158589:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513082.158595:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388770 00000800:00000200:26.0:1616513082.158598:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.158598:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.158600:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.158605:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.158608:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.158609:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.158617:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.117@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.158626:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.117@tcp1 of length 224 into portal 12 MB=0x5ffd74bcbcb00 00000400:00000200:26.0:1616513082.158633:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.117@tcp1 of length 224/224 into md 0xf964d519 [1] + 17696 00000400:00000200:26.0:1616513082.158638:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.158640:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.158648:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.158651:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.158653:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.158665:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675038251776 00000100:00000200:12.0:1616513082.158670:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675038251520 00010000:00000200:30.0:1616513082.158675:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57d580 x1688675038251776/t0(0) o400->fcb828d3-425f-2a73-20fb-2931904bd369@10.31.132.117@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:12.0:1616513082.158681:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688675038251520/t0(0) o400->56571a7f-04b1-38e7-4b7a-f365bd650038@10.31.132.117@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513082.158689:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688675038251776, offset 192 00000400:00000200:30.0:1616513082.158695:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.117@tcp1 00000100:00000200:12.0:1616513082.158695:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688675038251520, offset 192 00000400:00000200:12.0:1616513082.158701:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.117@tcp1 00000400:00000200:30.0:1616513082.158702:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.117@tcp1 routed destination 00000400:00000200:12.0:1616513082.158708:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.117@tcp1 routed destination 00000400:00000200:30.0:1616513082.158715:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.117@tcp1(10.31.132.117@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.158721:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513082.158723:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.117@tcp1(10.31.132.117@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.158726:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.158729:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513082.158730:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.158731:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513082.158735:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513082.158735:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513082.158738:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513082.158740:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513082.158740:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513082.158745:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513082.158751:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.158751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.158770:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.158774:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.158776:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.158781:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.158782:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.158785:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:12.0:1616513082.158788:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513082.158793:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.158801:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.158803:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513082.158805:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513082.158810:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.158814:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef5c8 00000800:00000200:2.0:1616513082.158818:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.167825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513082.167877:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.167887:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.167896:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.244@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.167905:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.244@tcp1 of length 224 into portal 26 MB=0x60419321cf3c0 00000400:00000200:26.0:1616513082.167912:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.244@tcp1 of length 224/224 into md 0xf97c9a89 [1] + 0 00000400:00000200:26.0:1616513082.167917:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388a18 00000400:00000200:26.0:1616513082.167919:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.167922:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513082.167926:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388a18 00000800:00000200:26.0:1616513082.167929:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.167930:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.167932:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.167934:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513082.167988:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356121715648 00010000:00000200:12.0:1616513082.167999:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1693356121715648/t0(0) o400->57cb70b7-76ae-81be-d948-ecd7fee1591e@10.31.128.244@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513082.168013:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693356121715648, offset 192 00000400:00000200:12.0:1616513082.168019:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.244@tcp1 00000400:00000200:12.0:1616513082.168027:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.244@tcp1 routed destination 00000400:00000200:12.0:1616513082.168042:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.244@tcp1(10.31.128.244@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513082.168048:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513082.168054:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513082.168056:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513082.168060:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513082.168065:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513082.168071:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.168074:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.168127:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.168134:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.168137:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.168143:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.168149:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:22.0:1616513082.168153:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.168158:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.168164:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513082.168166:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.168171:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.244@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.168179:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.244@tcp1 of length 224 into portal 12 MB=0x60419321cf700 00000400:00000200:22.0:1616513082.168185:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.244@tcp1 of length 224/224 into md 0xf964d519 [1] + 17920 00000400:00000200:22.0:1616513082.168190:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.168192:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513082.168199:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.168201:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.168203:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.168209:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356121716480 00010000:00000200:30.0:1616513082.168215:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec205e80 x1693356121716480/t0(0) o400->8a80f4b6-fc65-bc92-e297-e9131690efa6@10.31.128.244@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.168226:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693356121716480, offset 192 00000400:00000200:30.0:1616513082.168230:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.244@tcp1 00000400:00000200:30.0:1616513082.168236:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.244@tcp1 routed destination 00000400:00000200:30.0:1616513082.168248:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.244@tcp1(10.31.128.244@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.168254:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.168258:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.168260:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513082.168262:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513082.168265:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.168271:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.168333:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.168340:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513082.168342:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513082.168349:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513082.168353:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eea18 00000800:00000200:16.0:1616513082.168357:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513082.168360:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.177686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.177742:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.177752:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.177761:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.85@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.177772:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.85@tcp1 of length 224 into portal 12 MB=0x603e502886180 00000400:00000200:28.0:1616513082.177778:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.85@tcp1 of length 224/224 into md 0x865bbec1 [1] + 48272 00000400:00000200:28.0:1616513082.177783:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.177786:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.177792:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.177795:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513082.177795:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.177799:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.177857:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693131985150336 00010000:00000200:0.0:1616513082.177869:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedd100 x1693131985150336/t0(0) o400->af3d48f8-af9c-0ec7-ec2c-80abe79c272e@10.31.136.85@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.177884:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693131985150336, offset 192 00000400:00000200:0.0:1616513082.177891:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.85@tcp1 00000800:00000200:7.2:1616513082.177896:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513082.177901:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.85@tcp1 routed destination 00000400:00000200:0.0:1616513082.177916:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.85@tcp1(10.31.136.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.177922:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.177928:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.177931:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.177933:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513082.177938:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.177949:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.177957:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.177965:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.85@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.177976:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.85@tcp1 of length 224 into portal 26 MB=0x603e502886100 00000400:00000200:26.0:1616513082.177982:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.85@tcp1 of length 224/224 into md 0xf97c9ab9 [1] + 0 00000400:00000200:26.0:1616513082.177986:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc13883b8 00000400:00000200:26.0:1616513082.177988:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.177991:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513082.177995:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13883b8 00000800:00000200:26.0:1616513082.177998:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513082.178000:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513082.178002:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.178004:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:12.0:1616513082.178019:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693131985150208 00000800:00000200:14.0:1616513082.178020:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.178022:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:12.0:1616513082.178028:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1693131985150208/t0(0) o400->2c6e8166-9b57-749e-25ab-15f1dc8c77ce@10.31.136.85@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513082.178029:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.178033:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05cb0 00000800:00000200:14.0:1616513082.178039:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.178042:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513082.178042:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693131985150208, offset 192 00000400:00000200:12.0:1616513082.178047:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.85@tcp1 00000400:00000200:12.0:1616513082.178055:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.85@tcp1 routed destination 00000400:00000200:12.0:1616513082.178068:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.85@tcp1(10.31.136.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513082.178075:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513082.178080:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513082.178082:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513082.178084:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.178088:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.178093:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513082.178094:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513082.178104:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.178110:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513082.178113:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513082.178119:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513082.178125:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fed0 00000800:00000200:16.0:1616513082.178128:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513082.178131:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.191853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513082.191907:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.191916:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513082.191924:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.139@tcp1 : PUT - for me 00000400:00000200:12.0:1616513082.191932:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.139@tcp1 of length 224 into portal 26 MB=0x6040f98f40840 00000400:00000200:12.0:1616513082.191938:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.139@tcp1 of length 224/224 into md 0xf97c9ad9 [1] + 0 00000400:00000200:12.0:1616513082.191943:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388330 00000400:00000200:12.0:1616513082.191945:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513082.191948:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513082.191953:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388330 00000800:00000200:12.0:1616513082.191956:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.191959:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.191961:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.191968:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513082.191971:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.191972:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.191981:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.139@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.191990:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.139@tcp1 of length 224 into portal 12 MB=0x6040f98f408c0 00000400:00000200:28.0:1616513082.191996:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.139@tcp1 of length 224/224 into md 0x865bbec1 [1] + 48496 00000400:00000200:28.0:1616513082.192001:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.192004:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.192011:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.192014:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.192016:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.192016:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693314897414208 00010000:00000200:16.0:1616513082.192026:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1693314897414208/t0(0) o400->e4a8d512-7957-4039-d9ab-f0bce2700956@10.31.136.139@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.192040:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693314897414208, offset 192 00000400:00000200:16.0:1616513082.192046:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.139@tcp1 00000400:00000200:16.0:1616513082.192053:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.139@tcp1 routed destination 00000400:00000200:16.0:1616513082.192067:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.139@tcp1(10.31.136.139@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.192073:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:0.0:1616513082.192074:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693314897414336 00000800:00000200:16.0:1616513082.192078:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.192081:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.192084:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513082.192085:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedda00 x1693314897414336/t0(0) o400->aabc3b9e-c5f7-c1bf-e9e7-77d456e1d51b@10.31.136.139@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.192089:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513082.192096:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.192096:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513082.192099:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693314897414336, offset 192 00000400:00000200:0.0:1616513082.192105:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.139@tcp1 00000400:00000200:0.0:1616513082.192112:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.139@tcp1 routed destination 00000400:00000200:0.0:1616513082.192126:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.139@tcp1(10.31.136.139@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.192132:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.192137:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.192140:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.192147:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513082.192151:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513082.192152:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513082.192157:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513082.192157:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513082.192160:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.192166:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513082.192169:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513082.192171:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42dd0 00000800:00000200:22.0:1616513082.192175:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513082.192176:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513082.192178:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513082.192179:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513082.192180:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.192186:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.192190:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05540 00000800:00000200:24.0:1616513082.192194:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.212129:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.212186:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.212195:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.212205:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.43@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.212214:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.43@tcp1 of length 224 into portal 26 MB=0x5ffdc63421400 00000400:00000200:28.0:1616513082.212221:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.43@tcp1 of length 224/224 into md 0xf97c9ae9 [1] + 0 00000400:00000200:28.0:1616513082.212225:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388cc0 00000400:00000200:28.0:1616513082.212228:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.212231:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.212235:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388cc0 00000800:00000200:28.0:1616513082.212238:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.212241:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.212241:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.212245:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.212299:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696906716160 00010000:00000200:16.0:1616513082.212310:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688696906716160/t0(0) o400->e4abbd9f-00a9-da15-69a9-8cf283272064@10.31.142.43@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.212324:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696906716160, offset 192 00000400:00000200:16.0:1616513082.212331:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.43@tcp1 00000400:00000200:16.0:1616513082.212338:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.43@tcp1 routed destination 00000400:00000200:16.0:1616513082.212353:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.43@tcp1(10.31.142.43@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.212360:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.212366:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:7.2:1616513082.212366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.212369:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.212371:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513082.212376:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:16.0:1616513082.212382:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513082.212421:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.212430:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.212449:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.43@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.212457:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.43@tcp1 of length 224 into portal 12 MB=0x5ffdc634216c0 00000400:00000200:14.0:1616513082.212463:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.43@tcp1 of length 224/224 into md 0xf964d519 [1] + 18144 00000400:00000200:14.0:1616513082.212468:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.212471:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513082.212474:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513082.212478:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:14.0:1616513082.212480:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:14.0:1616513082.212482:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513082.212482:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513082.212485:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513082.212485:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.212491:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:30.0:1616513082.212495:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696906716864 00000400:00000200:2.0:1616513082.212497:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42198 00000800:00000200:2.0:1616513082.212500:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:30.0:1616513082.212505:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba3a80 x1688696906716864/t0(0) o400->34076e80-a738-9430-a86f-f6f0612de47c@10.31.142.43@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.212518:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696906716864, offset 192 00000400:00000200:30.0:1616513082.212524:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.43@tcp1 00000400:00000200:30.0:1616513082.212530:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.43@tcp1 routed destination 00000400:00000200:30.0:1616513082.212544:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.43@tcp1(10.31.142.43@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.212550:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.212555:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.212558:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513082.212560:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513082.212564:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.212570:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.212622:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.212630:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.212633:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513082.212636:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.212640:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.212644:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:12.0:1616513082.212648:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.219005:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513082.219022:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.219031:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.219040:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.245@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.219049:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.245@tcp1 of length 224 into portal 12 MB=0x60216995b6a00 00000400:00000200:26.0:1616513082.219056:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.245@tcp1 of length 224/224 into md 0xf964d519 [1] + 18368 00000400:00000200:26.0:1616513082.219061:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.219064:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.219071:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.219073:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.219075:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.219078:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.219082:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145945704960 00010000:00000200:30.0:1616513082.219090:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57de80 x1691145945704960/t0(0) o400->87cbe2b6-c089-cddf-ad88-4e0ff0863a08@10.31.140.245@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.219101:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691145945704960, offset 192 00000400:00000200:30.0:1616513082.219106:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.245@tcp1 00000400:00000200:30.0:1616513082.219112:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.245@tcp1 routed destination 00000400:00000200:30.0:1616513082.219125:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.245@tcp1(10.31.140.245@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.219133:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513082.219138:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.219140:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513082.219142:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513082.219145:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.219151:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513082.219168:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513082.219204:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.219211:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.219214:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.219220:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.219225:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef540 00000800:00000200:20.0:1616513082.219228:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513082.219228:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.219231:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513082.219237:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513082.219243:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.219246:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.245@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.219255:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.245@tcp1 of length 224 into portal 26 MB=0x60216995b6740 00000400:00000200:20.0:1616513082.219262:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.245@tcp1 of length 224/224 into md 0xf97c9b19 [1] + 0 00000400:00000200:20.0:1616513082.219266:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc13882a8 00000400:00000200:20.0:1616513082.219269:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.219272:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.219276:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13882a8 00000800:00000200:20.0:1616513082.219279:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.219282:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.219284:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.219341:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145945704256 00010000:00000200:16.0:1616513082.219352:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1691145945704256/t0(0) o400->f7b554f1-22d8-492a-5ca8-520e91b19dab@10.31.140.245@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.219366:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691145945704256, offset 192 00000400:00000200:16.0:1616513082.219372:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.245@tcp1 00000400:00000200:16.0:1616513082.219380:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.245@tcp1 routed destination 00000400:00000200:16.0:1616513082.219395:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.245@tcp1(10.31.140.245@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.219401:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.219407:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.219410:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.219412:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.219417:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.219422:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.219424:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513082.219450:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.219457:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.219460:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513082.219467:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.219472:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ed0 00000800:00000200:24.0:1616513082.219476:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.219484:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.224892:0:49685:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.224947:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.224957:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.224966:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.36@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.224975:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.36@tcp1 of length 224 into portal 12 MB=0x60195956c11c0 00000400:00000200:14.0:1616513082.224981:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.36@tcp1 of length 224/224 into md 0xf964d519 [1] + 18592 00000400:00000200:14.0:1616513082.224987:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.224989:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513082.224996:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.224999:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.225001:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.225005:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.225012:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690591828906432 00000800:00000200:11.2:1616513082.225017:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:30.0:1616513082.225022:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba2880 x1690591828906432/t0(0) o400->27650c50-6a66-7062-808e-495a2d0d0393@10.31.136.36@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.225034:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690591828906432, offset 192 00000400:00000200:30.0:1616513082.225040:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.36@tcp1 00000400:00000200:30.0:1616513082.225047:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.36@tcp1 routed destination 00000400:00000200:30.0:1616513082.225061:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.36@tcp1(10.31.136.36@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.225067:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513082.225072:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.225074:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513082.225076:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513082.225078:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513082.225081:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513082.225087:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513082.225087:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513082.225097:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.36@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.225105:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.36@tcp1 of length 224 into portal 26 MB=0x60195956c1140 00000400:00000200:28.0:1616513082.225112:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.36@tcp1 of length 224/224 into md 0xf97c9b29 [1] + 0 00000400:00000200:28.0:1616513082.225117:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388dd0 00000400:00000200:28.0:1616513082.225119:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.225122:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.225129:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388dd0 00000800:00000200:28.0:1616513082.225132:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.225132:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.225135:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.225137:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513082.225155:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.225162:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.225178:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.225184:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.225188:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee198 00000800:00000200:2.0:1616513082.225192:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.225195:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.225213:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690591828906304 00010000:00000200:16.0:1616513082.225224:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1690591828906304/t0(0) o400->56c35e2a-2eb7-f20b-00b3-104cf795c726@10.31.136.36@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.225240:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690591828906304, offset 192 00000400:00000200:16.0:1616513082.225247:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.36@tcp1 00000400:00000200:16.0:1616513082.225254:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.36@tcp1 routed destination 00000400:00000200:16.0:1616513082.225269:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.36@tcp1(10.31.136.36@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.225275:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.225281:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.225284:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.225286:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.225291:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.225296:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.225297:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513082.225307:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.225314:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.225316:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513082.225323:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.225328:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42330 00000800:00000200:24.0:1616513082.225331:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.225334:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.230594:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513082.230647:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.230657:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513082.230658:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513082.230666:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.229@tcp1 : PUT - for me 00000800:00000200:14.0:1616513082.230668:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.230676:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.229@tcp1 of length 224 into portal 26 MB=0x5ffde64c5ac40 00000400:00000200:14.0:1616513082.230677:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.229@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.230682:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.229@tcp1 of length 224/224 into md 0xf97c9b39 [1] + 0 00000400:00000200:14.0:1616513082.230686:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.229@tcp1 of length 224 into portal 12 MB=0x5ffde64c5acc0 00000400:00000200:26.0:1616513082.230687:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc13887f8 00000400:00000200:26.0:1616513082.230690:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.230692:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.229@tcp1 of length 224/224 into md 0xf964d519 [1] + 18816 00000100:00000200:26.0:1616513082.230697:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513082.230697:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.230701:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc13887f8 00000100:00000200:14.0:1616513082.230703:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.230704:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513082.230706:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513082.230709:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513082.230710:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.230712:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.230713:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.230715:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.230718:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705522052288 00010000:00000200:30.0:1616513082.230724:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba4c80 x1688705522052288/t0(0) o400->eb9a4d3e-18ba-6a59-a9df-687a611a698d@10.31.141.229@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.230734:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705522052288, offset 192 00000400:00000200:30.0:1616513082.230738:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.229@tcp1 00000400:00000200:30.0:1616513082.230744:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.229@tcp1 routed destination 00000400:00000200:30.0:1616513082.230755:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.229@tcp1(10.31.141.229@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.230760:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:16.0:1616513082.230763:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705522052160 00000800:00000200:30.0:1616513082.230764:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.230766:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513082.230768:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513082.230771:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:16.0:1616513082.230773:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688705522052160/t0(0) o400->a71443e9-c70b-41ce-7420-9b8114f024aa@10.31.141.229@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513082.230776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.230786:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705522052160, offset 192 00000400:00000200:16.0:1616513082.230792:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.229@tcp1 00000400:00000200:16.0:1616513082.230799:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.229@tcp1 routed destination 00000400:00000200:16.0:1616513082.230812:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.229@tcp1(10.31.141.229@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.230818:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.230824:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.230826:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.230828:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513082.230829:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513082.230832:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.230839:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513082.230841:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.230844:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513082.230846:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.230850:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.230854:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee770 00000800:00000200:22.0:1616513082.230858:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.230882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.230938:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.230944:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.230947:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.230954:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.230959:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43188 00000800:00000200:20.0:1616513082.230963:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.230966:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.235838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513082.235840:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513082.235891:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.235911:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513082.235912:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.235919:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.156@tcp1 : PUT - for me 00000800:00000200:24.0:1616513082.235920:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513082.235928:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.156@tcp1 of length 224 into portal 26 MB=0x5ffd472c99600 00000800:00000200:24.0:1616513082.235928:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513082.235934:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.156@tcp1 of length 224/224 into md 0xf97c9b49 [1] + 0 00000400:00000200:24.0:1616513082.235937:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.156@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.235938:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388990 00000400:00000200:26.0:1616513082.235941:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.235943:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.235945:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.156@tcp1 of length 224 into portal 12 MB=0x5ffd472c996c0 00000400:00000200:26.0:1616513082.235947:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388990 00000800:00000200:26.0:1616513082.235950:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513082.235951:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.156@tcp1 of length 224/224 into md 0x865bbec1 [1] + 48720 00000800:00000200:26.0:1616513082.235952:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.235954:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513082.235956:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.235958:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.235964:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.235967:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.235969:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.235971:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.236027:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662807516672 00000100:00000200:0.0:1616513082.236031:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662807516864 00010000:00000200:16.0:1616513082.236037:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688662807516672/t0(0) o400->d0617d2f-4181-c28a-6f73-1595cb1123d9@10.31.138.156@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:0.0:1616513082.236044:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd700000 x1688662807516864/t0(0) o400->b7860891-3b7a-8477-27f5-64cde13caf1a@10.31.138.156@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513082.236050:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662807516672, offset 192 00000400:00000200:16.0:1616513082.236056:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.156@tcp1 00000100:00000200:0.0:1616513082.236059:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662807516864, offset 192 00000400:00000200:16.0:1616513082.236064:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.156@tcp1 routed destination 00000400:00000200:0.0:1616513082.236066:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.156@tcp1 00000400:00000200:0.0:1616513082.236075:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.156@tcp1 routed destination 00000400:00000200:16.0:1616513082.236080:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.156@tcp1(10.31.138.156@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.236086:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513082.236090:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.156@tcp1(10.31.138.156@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.236097:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.236097:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.236100:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.236102:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.236102:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.236105:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.236106:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513082.236109:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513082.236111:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.236113:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513082.236114:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.236118:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.236130:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.236136:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.236139:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.236146:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.236150:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04aa0 00000800:00000200:20.0:1616513082.236153:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.236158:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513082.236164:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.236173:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.236176:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.236182:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.236188:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a434b8 00000800:00000200:2.0:1616513082.236191:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.236194:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.258295:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513082.258352:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.258362:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513082.258372:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.193@tcp1 : PUT - for me 00000400:00000200:12.0:1616513082.258381:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.193@tcp1 of length 224 into portal 26 MB=0x5ffd4bf8c0d80 00000400:00000200:12.0:1616513082.258388:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.193@tcp1 of length 224/224 into md 0xf97c9b69 [1] + 0 00000400:00000200:12.0:1616513082.258393:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388110 00000400:00000200:12.0:1616513082.258395:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513082.258398:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513082.258403:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388110 00000800:00000200:12.0:1616513082.258406:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.258408:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.258409:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513082.258415:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513082.258417:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.258418:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.258427:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.193@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.258448:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.193@tcp1 of length 224 into portal 12 MB=0x5ffd4bf8c0e00 00000400:00000200:24.0:1616513082.258454:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.193@tcp1 of length 224/224 into md 0x865bbec1 [1] + 48944 00000400:00000200:24.0:1616513082.258459:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.258462:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.258469:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.258472:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.258476:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.258481:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664095329664 00010000:00000200:16.0:1616513082.258490:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688664095329664/t0(0) o400->6feac997-f07e-d862-d3a6-b3656039ee59@10.31.136.193@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.258503:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664095329664, offset 192 00000400:00000200:16.0:1616513082.258508:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.193@tcp1 00000400:00000200:16.0:1616513082.258515:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.193@tcp1 routed destination 00000400:00000200:16.0:1616513082.258528:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.193@tcp1(10.31.136.193@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.258534:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:0.0:1616513082.258538:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664095329792 00000800:00000200:16.0:1616513082.258540:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.258542:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.258544:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.258549:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513082.258551:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd706c00 x1688664095329792/t0(0) o400->6a3a919a-b226-1e1c-0331-0e290d8b0235@10.31.136.193@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513082.258555:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.258555:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513082.258565:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664095329792, offset 192 00000400:00000200:0.0:1616513082.258572:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.193@tcp1 00000400:00000200:0.0:1616513082.258579:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.193@tcp1 routed destination 00000400:00000200:0.0:1616513082.258595:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.193@tcp1(10.31.136.193@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.258601:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.258607:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513082.258609:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513082.258610:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.258612:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513082.258616:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513082.258617:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513082.258618:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513082.258621:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513082.258625:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.258630:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a438f8 00000800:00000200:14.0:1616513082.258633:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513082.258634:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513082.258636:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513082.258641:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.258644:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513082.258645:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513082.258651:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.258654:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:28.0:1616513082.258659:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.259959:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.260013:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.260023:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.260032:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.45@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.260042:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.45@tcp1 of length 224 into portal 26 MB=0x60199593ce640 00000400:00000200:2.0:1616513082.260048:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.45@tcp1 of length 224/224 into md 0xf97c9b89 [1] + 0 00000400:00000200:2.0:1616513082.260053:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cc1388550 00000400:00000200:2.0:1616513082.260055:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.260058:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.260062:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cc1388550 00000800:00000200:2.0:1616513082.260066:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.260069:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.260069:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.260073:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.260128:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690607999051328 00010000:00000200:16.0:1616513082.260139:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1690607999051328/t0(0) o400->fa56c3f5-cfdf-feb6-48ae-b2b79ca86228@10.31.140.45@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.260153:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690607999051328, offset 192 00000400:00000200:16.0:1616513082.260160:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.45@tcp1 00000400:00000200:16.0:1616513082.260167:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.45@tcp1 routed destination 00000400:00000200:16.0:1616513082.260182:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.45@tcp1(10.31.140.45@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.260189:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.260195:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.260199:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.260201:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.260206:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.260211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.260213:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513082.260261:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.260266:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.260269:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.260275:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.260280:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42550 00000800:00000200:14.0:1616513082.260283:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.260285:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.260553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.260608:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.260618:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.260627:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.45@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.260635:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.45@tcp1 of length 224 into portal 12 MB=0x60199593cec80 00000400:00000200:20.0:1616513082.260642:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.45@tcp1 of length 224/224 into md 0x865bbec1 [1] + 49168 00000400:00000200:20.0:1616513082.260647:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.260649:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513082.260656:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.260659:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.260661:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.260664:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.260724:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690607999052928 00010000:00000200:0.0:1616513082.260735:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08a880 x1690607999052928/t0(0) o400->b5b8f5ee-6aaf-8d2e-0dfa-358391dbf949@10.31.140.45@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.260748:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690607999052928, offset 192 00000400:00000200:0.0:1616513082.260754:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.45@tcp1 00000400:00000200:0.0:1616513082.260761:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.45@tcp1 routed destination 00000400:00000200:0.0:1616513082.260775:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.45@tcp1(10.31.140.45@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.260781:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.260786:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.260789:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.260791:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.260795:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.260800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.260854:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.260861:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.260863:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.260866:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.260870:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.260874:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:12.0:1616513082.260878:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.261296:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513082.261351:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.261360:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.261369:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.79@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.261379:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.79@tcp1 of length 224 into portal 26 MB=0x60195ebbee400 00000400:00000200:22.0:1616513082.261386:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.79@tcp1 of length 224/224 into md 0xf97c9b99 [1] + 0 00000400:00000200:22.0:1616513082.261390:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a2a8 00000400:00000200:22.0:1616513082.261393:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.261396:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513082.261400:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a2a8 00000800:00000200:22.0:1616513082.261405:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.261408:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.261410:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.261412:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.261417:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513082.261420:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.261424:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.79@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.261429:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.79@tcp1 of length 224 into portal 12 MB=0x60195ebbee480 00000400:00000200:22.0:1616513082.261450:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.79@tcp1 of length 224/224 into md 0xf964d519 [1] + 19040 00000400:00000200:22.0:1616513082.261454:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.261456:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513082.261462:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.261464:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.261466:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.261480:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690593277174912 00000100:00000200:16.0:1616513082.261484:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690593277174784 00010000:00000200:30.0:1616513082.261490:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec206c00 x1690593277174912/t0(0) o400->c1883c63-b21e-572d-08c7-0b00ac9fd483@10.31.140.79@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:16.0:1616513082.261495:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1690593277174784/t0(0) o400->317f5b1e-45b3-2070-d541-5f2132b6e2bf@10.31.140.79@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513082.261503:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690593277174912, offset 192 00000400:00000200:30.0:1616513082.261508:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.79@tcp1 00000100:00000200:16.0:1616513082.261510:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690593277174784, offset 192 00000400:00000200:30.0:1616513082.261515:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.79@tcp1 routed destination 00000400:00000200:16.0:1616513082.261516:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.79@tcp1 00000400:00000200:16.0:1616513082.261524:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.79@tcp1 routed destination 00000400:00000200:30.0:1616513082.261529:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.79@tcp1(10.31.140.79@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.261535:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513082.261539:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.79@tcp1(10.31.140.79@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.261541:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.261543:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513082.261545:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.261546:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.261549:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513082.261551:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.261554:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513082.261555:0:49688:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.261556:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.261561:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.261566:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.261567:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513082.261575:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.261579:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.261582:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513082.261584:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.261587:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.261592:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43dc0 00000800:00000200:20.0:1616513082.261595:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513082.261609:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.261616:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.261619:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513082.261620:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.261626:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.261630:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efd38 00000800:00000200:26.0:1616513082.261634:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.270062:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.270117:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.270126:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.270135:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.154@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.270144:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.154@tcp1 of length 224 into portal 26 MB=0x6018337271e80 00000400:00000200:28.0:1616513082.270151:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.154@tcp1 of length 224/224 into md 0xf97c9ba9 [1] + 0 00000400:00000200:28.0:1616513082.270156:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a220 00000400:00000200:28.0:1616513082.270159:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.270161:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.270166:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a220 00000800:00000200:28.0:1616513082.270169:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.270170:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.270172:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.270174:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.270229:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512937918080 00010000:00000200:16.0:1616513082.270240:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1690512937918080/t0(0) o400->e93eb688-a2d3-e9de-7bc9-6467a927bb1a@10.31.131.154@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.270255:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512937918080, offset 192 00000800:00000200:11.2:1616513082.270259:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:16.0:1616513082.270261:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.154@tcp1 00000400:00000200:16.0:1616513082.270268:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.154@tcp1 routed destination 00000400:00000200:16.0:1616513082.270290:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.154@tcp1(10.31.131.154@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.270296:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.270302:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.270305:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.270307:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.270312:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513082.270318:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.270320:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513082.270324:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.270334:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.270344:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.154@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.270353:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.154@tcp1 of length 224 into portal 12 MB=0x6018337272240 00000400:00000200:24.0:1616513082.270359:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.154@tcp1 of length 224/224 into md 0x865bbec1 [1] + 49392 00000400:00000200:24.0:1616513082.270364:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.270367:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.270374:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513082.270376:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513082.270377:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.270379:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.270382:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513082.270382:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.270385:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.270392:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.270398:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42cc0 00000800:00000200:22.0:1616513082.270401:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.270404:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513082.270446:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512937919040 00010000:00000200:20.0:1616513082.270459:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd701f80 x1690512937919040/t0(0) o400->0a9bb8e1-770f-a887-85c1-34bb0b1d4082@10.31.131.154@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513082.270474:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512937919040, offset 192 00000400:00000200:20.0:1616513082.270480:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.154@tcp1 00000400:00000200:20.0:1616513082.270488:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.154@tcp1 routed destination 00000400:00000200:20.0:1616513082.270502:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.154@tcp1(10.31.131.154@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513082.270509:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.270515:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.270518:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.270520:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.270524:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.270529:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.270540:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.270545:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.270548:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.270550:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.270554:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.270558:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a220 00000800:00000200:12.0:1616513082.270562:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.272528:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.272580:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.272588:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.272596:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.114@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.272605:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.114@tcp1 of length 224 into portal 26 MB=0x600074f1561c0 00000400:00000200:24.0:1616513082.272611:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.114@tcp1 of length 224/224 into md 0xf97c9bc9 [1] + 0 00000400:00000200:24.0:1616513082.272616:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b320 00000400:00000200:24.0:1616513082.272618:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.272620:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.272625:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b320 00000800:00000200:24.0:1616513082.272628:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.272631:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.272631:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.272634:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.272698:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688881251836352 00010000:00000200:16.0:1616513082.272719:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688881251836352/t0(0) o400->ca828ac3-1620-98c3-f7e8-b72331c7be00@10.31.136.114@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.272732:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688881251836352, offset 192 00000400:00000200:16.0:1616513082.272738:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.114@tcp1 00000400:00000200:16.0:1616513082.272746:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.114@tcp1 routed destination 00000400:00000200:16.0:1616513082.272760:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.114@tcp1(10.31.136.114@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513082.272762:0:49689:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.272766:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.272771:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.272774:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.272776:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513082.272780:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:16.0:1616513082.272786:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513082.272817:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.272826:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.272835:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.114@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.272844:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.114@tcp1 of length 224 into portal 12 MB=0x600074f156600 00000400:00000200:2.0:1616513082.272851:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.114@tcp1 of length 224/224 into md 0xf964d519 [1] + 19264 00000400:00000200:2.0:1616513082.272856:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.272859:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513082.272866:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513082.272869:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513082.272870:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513082.272877:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513082.272878:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513082.272880:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:30.0:1616513082.272881:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688881251837440 00000400:00000200:26.0:1616513082.272881:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:30.0:1616513082.272888:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd218480 x1688881251837440/t0(0) o400->c01da456-8532-2347-ea60-f311a517ef2c@10.31.136.114@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:26.0:1616513082.272888:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.272893:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42550 00000800:00000200:26.0:1616513082.272897:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.272900:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688881251837440, offset 192 00000400:00000200:30.0:1616513082.272905:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.114@tcp1 00000400:00000200:30.0:1616513082.272911:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.114@tcp1 routed destination 00000400:00000200:30.0:1616513082.272924:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.114@tcp1(10.31.136.114@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.272929:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.272934:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.272936:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513082.272938:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513082.272941:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.272947:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.272959:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.272965:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.272968:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.272974:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.272978:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee088 00000800:00000200:28.0:1616513082.272982:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.272984:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.279597:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.279651:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.279660:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.279670:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.232@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.279679:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.232@tcp1 of length 224 into portal 12 MB=0x5ffed35fe8dc0 00000400:00000200:20.0:1616513082.279684:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.232@tcp1 of length 224/224 into md 0x865bbec1 [1] + 49616 00000400:00000200:20.0:1616513082.279689:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.279692:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513082.279698:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513082.279699:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.279701:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.279704:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.279707:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513082.279765:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.279775:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:28.0:1616513082.279778:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688769161760192 00000400:00000200:14.0:1616513082.279784:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.232@tcp1 : PUT - for me 00010000:00000200:28.0:1616513082.279789:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08ba80 x1688769161760192/t0(0) o400->9be46bd7-0b85-dd7e-6dac-bfc13ecbcb4d@10.31.141.232@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513082.279793:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.232@tcp1 of length 224 into portal 26 MB=0x5ffed35fe8b40 00000400:00000200:14.0:1616513082.279799:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.232@tcp1 of length 224/224 into md 0xf97c9be9 [1] + 0 00000400:00000200:14.0:1616513082.279804:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b650 00000400:00000200:14.0:1616513082.279807:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.279810:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:28.0:1616513082.279813:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688769161760192, offset 192 00000400:00000200:14.0:1616513082.279814:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b650 00000800:00000200:14.0:1616513082.279817:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513082.279818:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.232@tcp1 00000800:00000200:2.0:1616513082.279818:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.279820:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.279822:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513082.279825:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.232@tcp1 routed destination 00000400:00000200:28.0:1616513082.279839:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.232@tcp1(10.31.141.232@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513082.279845:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513082.279850:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513082.279853:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513082.279855:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513082.279859:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.279864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.279880:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688769161759552 00010000:00000200:16.0:1616513082.279890:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688769161759552/t0(0) o400->6c30e291-8951-85e0-6374-dc266910cdd1@10.31.141.232@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.279903:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688769161759552, offset 192 00000400:00000200:16.0:1616513082.279909:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.232@tcp1 00000800:00000200:26.0:1616513082.279916:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513082.279916:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.232@tcp1 routed destination 00000800:00000200:26.0:1616513082.279923:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.279926:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513082.279930:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.232@tcp1(10.31.141.232@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513082.279933:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.279936:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc000 00000800:00000200:16.0:1616513082.279936:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513082.279940:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513082.279941:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513082.279943:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513082.279944:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.279946:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.279950:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.279955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.279956:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513082.279965:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.279969:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.279971:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.279976:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.279980:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a434b8 00000800:00000200:28.0:1616513082.279983:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.279986:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.290787:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.290843:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.290852:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.290860:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.16@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.290868:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.16@tcp1 of length 224 into portal 26 MB=0x5ffd3f373f4c0 00000400:00000200:24.0:1616513082.290874:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.16@tcp1 of length 224/224 into md 0xf97c9c09 [1] + 0 00000400:00000200:24.0:1616513082.290879:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a440 00000400:00000200:24.0:1616513082.290881:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.290884:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.290888:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a440 00000800:00000200:24.0:1616513082.290891:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.290895:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.290896:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.290899:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.290950:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660671198400 00010000:00000200:16.0:1616513082.290960:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688660671198400/t0(0) o400->b0ffc0f3-ce91-25f7-8a1e-17c1c88ab9e4@10.31.129.16@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.290973:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660671198400, offset 192 00000800:00000200:7.2:1616513082.290973:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513082.290982:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.16@tcp1 00000400:00000200:16.0:1616513082.290988:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.16@tcp1 routed destination 00000400:00000200:16.0:1616513082.291002:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.16@tcp1(10.31.129.16@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.291009:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.291014:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.291017:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.291019:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513082.291023:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:16.0:1616513082.291031:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513082.291031:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.291041:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.291050:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.16@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.291058:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.16@tcp1 of length 224 into portal 12 MB=0x5ffd3f373f940 00000400:00000200:2.0:1616513082.291064:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.16@tcp1 of length 224/224 into md 0xf964d519 [1] + 19488 00000400:00000200:2.0:1616513082.291068:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.291071:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513082.291078:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513082.291081:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513082.291082:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.291085:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.291088:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.291090:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.291095:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:30.0:1616513082.291098:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660671199552 00000400:00000200:2.0:1616513082.291099:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42330 00000800:00000200:2.0:1616513082.291102:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.291104:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:30.0:1616513082.291109:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d4800 x1688660671199552/t0(0) o400->f8da95d8-01e6-6d0b-cf5f-b4a23ad47b01@10.31.129.16@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.291123:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660671199552, offset 192 00000400:00000200:30.0:1616513082.291130:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.16@tcp1 00000400:00000200:30.0:1616513082.291136:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.16@tcp1 routed destination 00000400:00000200:30.0:1616513082.291151:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.16@tcp1(10.31.129.16@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.291156:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.291162:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.291164:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513082.291166:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513082.291170:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.291176:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.291187:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.291193:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.291196:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.291202:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.291206:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef8f8 00000800:00000200:28.0:1616513082.291210:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.291212:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.293211:0:49691:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513082.293266:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.293275:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.293295:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.125@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.293303:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.125@tcp1 of length 224 into portal 26 MB=0x60070f73051c0 00000400:00000200:22.0:1616513082.293309:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.125@tcp1 of length 224/224 into md 0xf97c9c29 [1] + 0 00000400:00000200:22.0:1616513082.293313:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79bf58 00000400:00000200:22.0:1616513082.293316:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.293318:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513082.293322:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bf58 00000800:00000200:22.0:1616513082.293325:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.293328:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.293329:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.293331:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.293338:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.293345:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.125@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.293350:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.125@tcp1 of length 224 into portal 12 MB=0x60070f7305440 00000400:00000200:22.0:1616513082.293354:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.125@tcp1 of length 224/224 into md 0xf964d519 [1] + 19712 00000400:00000200:22.0:1616513082.293358:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.293360:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513082.293365:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.293367:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.293368:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.293371:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.293375:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689335043740736 00010000:00000200:30.0:1616513082.293381:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec202400 x1689335043740736/t0(0) o400->b27d1e8c-2419-1a24-aab2-daa89c22c5c1@10.31.141.125@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513082.293385:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689335043740096 00000100:00000200:30.0:1616513082.293391:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689335043740736, offset 192 00010000:00000200:16.0:1616513082.293395:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1689335043740096/t0(0) o400->53013eb3-8de9-2420-4337-63e4353cca3b@10.31.141.125@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:30.0:1616513082.293396:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.125@tcp1 00000400:00000200:30.0:1616513082.293401:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.125@tcp1 routed destination 00000100:00000200:16.0:1616513082.293408:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689335043740096, offset 192 00000400:00000200:30.0:1616513082.293413:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.125@tcp1(10.31.141.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513082.293414:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.125@tcp1 00000800:00000200:30.0:1616513082.293419:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513082.293421:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.125@tcp1 routed destination 00000800:00000200:30.0:1616513082.293423:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.293425:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513082.293427:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513082.293430:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.293435:0:49691:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513082.293448:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.125@tcp1(10.31.141.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.293454:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.293460:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.293462:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.293464:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.293468:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.293474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.293475:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513082.293488:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.293497:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.293499:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513082.293501:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.293506:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.293510:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee440 00000800:00000200:26.0:1616513082.293514:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513082.293531:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.293537:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.293540:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.293547:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.293552:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42198 00000800:00000200:20.0:1616513082.293556:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.293559:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.311026:0:49693:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513082.311082:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.311092:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.311101:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.164@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.311110:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.164@tcp1 of length 224 into portal 26 MB=0x5ffd80c693580 00000400:00000200:22.0:1616513082.311116:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.164@tcp1 of length 224/224 into md 0xf97c9c49 [1] + 0 00000400:00000200:22.0:1616513082.311121:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79ac38 00000400:00000200:22.0:1616513082.311124:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.311127:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513082.311131:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79ac38 00000800:00000200:22.0:1616513082.311134:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.311136:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.311137:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.311140:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.311140:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513082.311199:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678269793664 00000800:00000200:28.0:1616513082.311204:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:16.0:1616513082.311210:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688678269793664/t0(0) o400->9408bae1-5081-a02c-13b7-d2c612994fb3@10.31.129.164@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513082.311213:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.311222:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.164@tcp1 : PUT - for me 00000100:00000200:16.0:1616513082.311225:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688678269793664, offset 192 00000400:00000200:28.0:1616513082.311231:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.164@tcp1 of length 224 into portal 12 MB=0x5ffd80c6938c0 00000400:00000200:16.0:1616513082.311231:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.164@tcp1 00000400:00000200:28.0:1616513082.311238:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.164@tcp1 of length 224/224 into md 0x865bbec1 [1] + 49840 00000400:00000200:16.0:1616513082.311238:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.164@tcp1 routed destination 00000400:00000200:28.0:1616513082.311243:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.311245:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513082.311264:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.164@tcp1(10.31.129.164@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513082.311265:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.311268:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513082.311269:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.311274:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513082.311274:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.311280:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.311282:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.311284:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513082.311286:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678269794496 00000800:00000200:16.0:1616513082.311289:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.311293:0:49693:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.311295:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:0.0:1616513082.311298:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed9b00 x1688678269794496/t0(0) o400->2448609f-30dd-d997-1639-a309ae52eda8@10.31.129.164@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.311312:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688678269794496, offset 192 00000400:00000200:0.0:1616513082.311319:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.164@tcp1 00000400:00000200:0.0:1616513082.311326:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.164@tcp1 routed destination 00000400:00000200:0.0:1616513082.311340:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.164@tcp1(10.31.129.164@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.311346:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513082.311349:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513082.311351:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.311354:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513082.311356:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513082.311356:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513082.311358:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513082.311360:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513082.311364:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513082.311365:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513082.311369:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a08 00000800:00000200:14.0:1616513082.311375:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.311377:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513082.311426:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.311445:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.311448:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.311454:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.311458:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04cc0 00000800:00000200:20.0:1616513082.311462:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.311465:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.317219:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513082.317275:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.317285:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513082.317294:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.180@tcp1 : PUT - for me 00000400:00000200:12.0:1616513082.317304:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.180@tcp1 of length 224 into portal 26 MB=0x60182e60de800 00000400:00000200:12.0:1616513082.317311:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.180@tcp1 of length 224/224 into md 0xf97c9c69 [1] + 0 00000400:00000200:12.0:1616513082.317315:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79abb0 00000400:00000200:12.0:1616513082.317318:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513082.317321:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513082.317325:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79abb0 00000800:00000200:12.0:1616513082.317328:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.317330:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513082.317331:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513082.317334:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.317386:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511577311232 00010000:00000200:16.0:1616513082.317397:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1690511577311232/t0(0) o400->cd73bc9c-86cd-495a-b2bd-e5ea6daf6b20@10.31.136.180@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.317410:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511577311232, offset 192 00000400:00000200:16.0:1616513082.317416:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.180@tcp1 00000400:00000200:16.0:1616513082.317423:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.180@tcp1 routed destination 00000400:00000200:16.0:1616513082.317447:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.180@tcp1(10.31.136.180@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.317453:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.317459:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.317462:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.317464:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.317468:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513082.317475:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.317476:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.317532:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.317539:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.317542:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.317548:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.317554:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43cb0 00000800:00000200:2.0:1616513082.317557:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.317560:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.317620:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.317674:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.317683:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.317692:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.180@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.317702:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.180@tcp1 of length 224 into portal 12 MB=0x60182e60de940 00000400:00000200:14.0:1616513082.317708:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.180@tcp1 of length 224/224 into md 0xf964d519 [1] + 19936 00000400:00000200:14.0:1616513082.317713:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.317716:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513082.317723:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.317726:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.317728:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.317730:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.317740:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511577311552 00010000:00000200:30.0:1616513082.317749:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba3600 x1690511577311552/t0(0) o400->51cc4fad-bdee-7b38-50c3-2e5f1319b04c@10.31.136.180@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.317761:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511577311552, offset 192 00000400:00000200:30.0:1616513082.317767:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.180@tcp1 00000400:00000200:30.0:1616513082.317773:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.180@tcp1 routed destination 00000400:00000200:30.0:1616513082.317786:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.180@tcp1(10.31.136.180@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.317792:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.317797:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.317800:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513082.317802:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513082.317806:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.317812:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.317873:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.317880:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.317882:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.317889:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.317893:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee000 00000800:00000200:20.0:1616513082.317897:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.317900:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.318623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.318677:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.318686:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.318695:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.46@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.318704:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.46@tcp1 of length 224 into portal 26 MB=0x601b073d37940 00000400:00000200:28.0:1616513082.318711:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.46@tcp1 of length 224/224 into md 0xf97c9c79 [1] + 0 00000400:00000200:28.0:1616513082.318716:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b188 00000400:00000200:28.0:1616513082.318719:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.318721:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.318726:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b188 00000800:00000200:28.0:1616513082.318729:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.318730:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.318732:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.318734:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.318801:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690707229374784 00010000:00000200:16.0:1616513082.318811:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1690707229374784/t0(0) o400->75e6e6fa-927b-0aa3-bc85-ccdcc130a9c1@10.31.132.46@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.318824:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690707229374784, offset 192 00000400:00000200:16.0:1616513082.318830:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.46@tcp1 00000400:00000200:16.0:1616513082.318837:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.46@tcp1 routed destination 00000400:00000200:16.0:1616513082.318851:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.46@tcp1(10.31.132.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.318857:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.318863:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.318865:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.318869:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.318874:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.318879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.318880:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513082.318944:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.318951:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.318953:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.318960:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.318966:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43320 00000800:00000200:2.0:1616513082.318969:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.318972:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.319033:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.319101:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.319110:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.319118:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.46@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.319127:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.46@tcp1 of length 224 into portal 12 MB=0x601b073d37a40 00000400:00000200:20.0:1616513082.319135:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.46@tcp1 of length 224/224 into md 0x865bbec1 [1] + 50064 00000400:00000200:20.0:1616513082.319140:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.319143:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513082.319150:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.319153:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.319155:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.319159:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.319171:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690707229375040 00010000:00000200:0.0:1616513082.319184:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b088d80 x1690707229375040/t0(0) o400->a99f7384-bccc-3e3d-084d-e3c0283939e2@10.31.132.46@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.319198:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690707229375040, offset 192 00000400:00000200:0.0:1616513082.319204:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.46@tcp1 00000400:00000200:0.0:1616513082.319212:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.46@tcp1 routed destination 00000400:00000200:0.0:1616513082.319227:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.46@tcp1(10.31.132.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.319233:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.319239:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.319241:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.319244:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.319248:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.319266:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.319276:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.319281:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.319283:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.319285:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.319290:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.319294:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04908 00000800:00000200:12.0:1616513082.319298:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.324975:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513082.324994:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513082.325030:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.325039:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.325048:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.118@tcp1 : PUT - for me 00000800:00000200:14.0:1616513082.325051:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513082.325057:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.118@tcp1 of length 224 into portal 26 MB=0x5fffbd71b6a00 00000800:00000200:14.0:1616513082.325060:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513082.325064:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.118@tcp1 of length 224/224 into md 0xf97c9c99 [1] + 0 00000400:00000200:24.0:1616513082.325069:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a000 00000400:00000200:14.0:1616513082.325069:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.118@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.325072:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.325074:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.325078:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a000 00000400:00000200:14.0:1616513082.325079:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.118@tcp1 of length 224 into portal 12 MB=0x5fffbd71b6ac0 00000800:00000200:24.0:1616513082.325081:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.325083:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.325084:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513082.325085:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.118@tcp1 of length 224/224 into md 0xf964d519 [1] + 20160 00000800:00000200:24.0:1616513082.325087:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513082.325090:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.325093:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513082.325100:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.325103:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.325104:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.325107:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513082.325112:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688831994325696 00010000:00000200:30.0:1616513082.325120:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba6780 x1688831994325696/t0(0) o400->df848ded-055b-17fa-5876-0ed3452c27e3@10.31.138.118@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.325132:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688831994325696, offset 192 00000400:00000200:30.0:1616513082.325137:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.118@tcp1 00000400:00000200:30.0:1616513082.325143:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.118@tcp1 routed destination 00000100:00000200:16.0:1616513082.325144:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688831994325504 00010000:00000200:16.0:1616513082.325155:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688831994325504/t0(0) o400->c8903ec0-c4db-41a7-5404-60c4b44aee98@10.31.138.118@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:30.0:1616513082.325156:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.118@tcp1(10.31.138.118@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.325162:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513082.325167:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513082.325169:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513082.325169:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688831994325504, offset 192 00000800:00000200:30.0:1616513082.325171:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513082.325175:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513082.325175:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.118@tcp1 00000800:00000200:7.2:1616513082.325179:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513082.325182:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.118@tcp1 routed destination 00000400:00000200:16.0:1616513082.325198:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.118@tcp1(10.31.138.118@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.325204:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.325210:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.325212:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.325215:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.325219:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.325224:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.325226:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513082.325232:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.325239:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.325241:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.325250:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.325254:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef430 00000800:00000200:2.0:1616513082.325258:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.325261:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513082.325288:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.325295:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.325297:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.325304:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.325311:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43540 00000800:00000200:28.0:1616513082.325315:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.325318:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.334534:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.334588:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.334597:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.334605:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.100@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.334614:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.100@tcp1 of length 224 into portal 26 MB=0x6054c85aeee80 00000400:00000200:20.0:1616513082.334620:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.100@tcp1 of length 224/224 into md 0xf97c9cb9 [1] + 0 00000400:00000200:20.0:1616513082.334625:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79bcb0 00000400:00000200:20.0:1616513082.334627:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.334630:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.334634:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bcb0 00000800:00000200:20.0:1616513082.334637:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.334639:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.334641:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.334643:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.334654:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.334662:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.100@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.334671:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.100@tcp1 of length 224 into portal 12 MB=0x6054c85aef040 00000400:00000200:24.0:1616513082.334677:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.100@tcp1 of length 224/224 into md 0x865bbec1 [1] + 50288 00000400:00000200:24.0:1616513082.334682:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.334685:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.334691:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.334695:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.334696:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.334696:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694676078751360 00000800:00000200:24.0:1616513082.334699:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:16.0:1616513082.334706:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1694676078751360/t0(0) o400->499833ad-76f9-6019-247f-232756273e3d@10.31.141.100@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.334720:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694676078751360, offset 192 00000400:00000200:16.0:1616513082.334725:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.100@tcp1 00000400:00000200:16.0:1616513082.334732:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.100@tcp1 routed destination 00000400:00000200:16.0:1616513082.334746:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.100@tcp1(10.31.141.100@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.334752:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.334758:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.334761:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.334763:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513082.334763:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694676078751808 00000800:00000200:16.0:1616513082.334767:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513082.334773:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.334773:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513082.334774:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd701b00 x1694676078751808/t0(0) o400->3d3602b2-786c-05b6-e760-332083eb6193@10.31.141.100@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.334788:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694676078751808, offset 192 00000400:00000200:0.0:1616513082.334794:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.100@tcp1 00000400:00000200:0.0:1616513082.334801:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.100@tcp1 routed destination 00000400:00000200:0.0:1616513082.334815:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.100@tcp1(10.31.141.100@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.334821:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513082.334826:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513082.334826:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.334830:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513082.334832:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513082.334832:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513082.334834:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513082.334836:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513082.334840:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513082.334841:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513082.334846:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42e58 00000800:00000200:14.0:1616513082.334849:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.334852:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513082.334897:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.334904:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.334906:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513082.334908:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.334912:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.334916:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04440 00000800:00000200:12.0:1616513082.334920:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.349091:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.349145:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.349155:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.349164:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.10@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.349173:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.10@tcp1 of length 224 into portal 12 MB=0x6018d11b6a480 00000400:00000200:28.0:1616513082.349179:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.10@tcp1 of length 224/224 into md 0x865bbec1 [1] + 50512 00000400:00000200:28.0:1616513082.349185:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.349187:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.349194:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.349197:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.349200:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.349204:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.349216:0:49697:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513082.349219:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690555259462784 00010000:00000200:0.0:1616513082.349231:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedf980 x1690555259462784/t0(0) o400->376cf4db-6147-544c-8aa0-9ac11693d896@10.31.143.10@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.349245:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690555259462784, offset 192 00000400:00000200:0.0:1616513082.349252:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.10@tcp1 00000400:00000200:0.0:1616513082.349259:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.10@tcp1 routed destination 00000400:00000200:0.0:1616513082.349274:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.10@tcp1(10.31.143.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.349281:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513082.349284:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.349287:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.349290:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513082.349292:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513082.349294:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.349297:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513082.349304:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.18@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.349314:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.18@tcp1 of length 224 into portal 26 MB=0x5fff2a8273100 00000400:00000200:2.0:1616513082.349320:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.18@tcp1 of length 224/224 into md 0xf97c9cc9 [1] + 0 00000400:00000200:2.0:1616513082.349325:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79af68 00000400:00000200:2.0:1616513082.349328:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.349330:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.349335:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79af68 00000800:00000200:2.0:1616513082.349338:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513082.349340:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513082.349340:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513082.349347:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:11.2:1616513082.349349:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513082.349349:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513082.349350:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513082.349353:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513082.349355:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.349359:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.10@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.349361:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.349364:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:2.0:1616513082.349366:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.349368:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.10@tcp1 of length 224 into portal 26 MB=0x6018d11b6a2c0 00000800:00000200:2.0:1616513082.349369:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.349375:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.10@tcp1 of length 224/224 into md 0xf97c9cf9 [1] + 0 00000400:00000200:22.0:1616513082.349380:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b210 00000400:00000200:22.0:1616513082.349382:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.349385:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513082.349390:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b210 00000800:00000200:22.0:1616513082.349393:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.349398:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.349400:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513082.349403:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688792551862528 00000100:00000200:16.0:1616513082.349408:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690555259462336 00010000:00000200:4.0:1616513082.349415:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688792551862528/t0(0) o400->c34728a9-0dc1-e2d1-1151-05d1fb2814ca@10.31.136.18@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513082.349429:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688792551862528, offset 192 00000800:00000200:20.0:1616513082.349430:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:16.0:1616513082.349431:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1690555259462336/t0(0) o400->7bd062a1-4964-e1c1-76bb-67d0e33c8f52@10.31.143.10@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:4.0:1616513082.349446:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.18@tcp1 00000800:00000200:20.0:1616513082.349450:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513082.349456:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.18@tcp1 routed destination 00000100:00000200:16.0:1616513082.349458:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690555259462336, offset 192 00000400:00000200:16.0:1616513082.349464:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.10@tcp1 00000400:00000200:4.0:1616513082.349471:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.18@tcp1(10.31.136.18@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.349477:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.349480:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.18@tcp1 : PUT - for me 00000400:00000200:16.0:1616513082.349480:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.10@tcp1 routed destination 00000800:00000200:12.0:1616513082.349482:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513082.349487:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513082.349490:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.18@tcp1 of length 224 into portal 12 MB=0x5fff2a8273200 00000800:00000200:4.0:1616513082.349490:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513082.349492:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513082.349494:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.10@tcp1(10.31.143.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513082.349496:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.18@tcp1 of length 224/224 into md 0x865bbec1 [1] + 50736 00000800:00000200:4.0:1616513082.349497:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.349501:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513082.349501:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513082.349502:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:20.0:1616513082.349504:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513082.349506:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:4.0:1616513082.349506:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513082.349509:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513082.349510:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513082.349511:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513082.349513:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:20.0:1616513082.349513:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513082.349515:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513082.349517:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513082.349518:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513082.349520:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513082.349520:0:49697:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513082.349520:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688792551862784 00000400:00000200:24.0:1616513082.349521:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:16.0:1616513082.349522:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:0.0:1616513082.349526:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b089f80 x1688792551862784/t0(0) o400->d186a81e-80c4-d301-ff58-6c5b8430cc17@10.31.136.18@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513082.349527:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513082.349530:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513082.349533:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbdc0 00000800:00000200:2.0:1616513082.349534:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513082.349536:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513082.349536:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1616513082.349537:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688792551862784, offset 192 00000800:00000200:22.0:1616513082.349539:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513082.349541:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:0.0:1616513082.349541:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.18@tcp1 00000400:00000200:2.0:1616513082.349545:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43430 00000800:00000200:2.0:1616513082.349547:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513082.349547:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.18@tcp1 routed destination 00000400:00000200:0.0:1616513082.349559:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.18@tcp1(10.31.136.18@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.349564:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.349569:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.349571:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.349573:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.349576:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.349580:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.349590:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.349595:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.349597:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513082.349599:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.349603:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.349607:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05d38 00000800:00000200:12.0:1616513082.349611:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.350387:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.350396:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.350401:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513082.350404:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.350410:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.350412:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.350418:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.176@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.350418:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.176@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.350427:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.176@tcp1 of length 224 into portal 26 MB=0x5ffdbfcfa1340 00000400:00000200:24.0:1616513082.350428:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.176@tcp1 of length 224 into portal 12 MB=0x5ffdbfcfa13c0 00000400:00000200:28.0:1616513082.350442:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.176@tcp1 of length 224/224 into md 0xf97c9d09 [1] + 0 00000400:00000200:24.0:1616513082.350443:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.176@tcp1 of length 224/224 into md 0x865bbec1 [1] + 50960 00000400:00000200:28.0:1616513082.350447:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79acc0 00000400:00000200:28.0:1616513082.350449:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513082.350449:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.350451:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:28.0:1616513082.350452:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.350456:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79acc0 00000800:00000200:24.0:1616513082.350458:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.350459:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.350461:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513082.350461:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513082.350463:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.350465:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.350467:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695190721472 00000100:00000200:16.0:1616513082.350471:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695190721344 00010000:00000200:0.0:1616513082.350472:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd703180 x1688695190721472/t0(0) o400->0a6c7b2b-613d-9870-1ab6-f7b3d5e6daa7@10.31.140.176@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:16.0:1616513082.350477:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688695190721344/t0(0) o400->5fe0bbe1-f6cb-b123-8d3b-f463a07aab7e@10.31.140.176@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.350488:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695190721344, offset 192 00000100:00000200:0.0:1616513082.350491:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695190721472, offset 192 00000400:00000200:16.0:1616513082.350494:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.176@tcp1 00000400:00000200:0.0:1616513082.350498:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.176@tcp1 00000400:00000200:16.0:1616513082.350500:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.176@tcp1 routed destination 00000400:00000200:0.0:1616513082.350508:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.176@tcp1 routed destination 00000400:00000200:16.0:1616513082.350514:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.176@tcp1(10.31.140.176@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.350520:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513082.350520:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.176@tcp1(10.31.140.176@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.350525:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.350526:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.350529:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513082.350529:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.350530:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.350532:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.350533:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513082.350533:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.350536:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.350537:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.350540:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513082.350542:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.350551:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513082.350553:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513082.350554:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.350556:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513082.350559:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513082.350560:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.350562:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513082.350563:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.350566:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb046e8 00000800:00000200:12.0:1616513082.350568:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513082.350569:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.350576:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42c38 00000800:00000200:14.0:1616513082.350580:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.350583:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.350626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.350635:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.350640:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513082.350643:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.350648:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.35@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.350656:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.35@tcp1 of length 224 into portal 26 MB=0x60184dd4fc700 00000400:00000200:24.0:1616513082.350661:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.35@tcp1 of length 224/224 into md 0xf97c9d19 [1] + 0 00000400:00000200:24.0:1616513082.350666:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b4b8 00000400:00000200:24.0:1616513082.350667:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.350669:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.350674:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b4b8 00000800:00000200:24.0:1616513082.350676:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.350677:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.350679:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.350684:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690520020567808 00010000:00000200:16.0:1616513082.350689:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1690520020567808/t0(0) o400->b862fc18-326a-42fb-d10b-a2b628aedb3b@10.31.140.35@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.350699:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690520020567808, offset 192 00000400:00000200:16.0:1616513082.350704:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.35@tcp1 00000400:00000200:16.0:1616513082.350710:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.35@tcp1 routed destination 00000400:00000200:16.0:1616513082.350724:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.35@tcp1(10.31.140.35@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.350729:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.350734:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.350736:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.350738:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.350740:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513082.350744:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.350746:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513082.350802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513082.350803:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.350809:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513082.350811:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513082.350812:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513082.350817:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513082.350819:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513082.350819:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.350824:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.35@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.350825:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43d38 00000800:00000200:22.0:1616513082.350828:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.350831:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513082.350831:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.35@tcp1 of length 224 into portal 12 MB=0x60184dd4fc800 00000400:00000200:20.0:1616513082.350836:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.35@tcp1 of length 224/224 into md 0x865bbec1 [1] + 51184 00000800:00000200:22.0:1616513082.350837:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513082.350839:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.350840:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513082.350840:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513082.350842:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513082.350842:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513082.350843:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513082.350847:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.350849:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.350851:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.350857:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690520020568064 00010000:00000200:0.0:1616513082.350862:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b088480 x1690520020568064/t0(0) o400->e15ba6f1-d1f4-e2c2-3e09-9f8dbfc68299@10.31.140.35@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.350871:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690520020568064, offset 192 00000400:00000200:0.0:1616513082.350875:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.35@tcp1 00000400:00000200:0.0:1616513082.350880:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.35@tcp1 routed destination 00000400:00000200:0.0:1616513082.350892:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.35@tcp1(10.31.140.35@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.350897:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.350901:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.350903:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.350905:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.350907:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.350912:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.350921:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.350925:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.350928:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513082.350929:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513082.350933:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.350936:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:28.0:1616513082.350939:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.357374:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513082.357429:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513082.357451:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513082.357460:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.131@tcp1 : PUT - for me 00000400:00000200:12.0:1616513082.357469:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.131@tcp1 of length 224 into portal 26 MB=0x604b4267024c0 00000400:00000200:12.0:1616513082.357475:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.131@tcp1 of length 224/224 into md 0xf97c9d29 [1] + 0 00000400:00000200:12.0:1616513082.357480:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79be48 00000400:00000200:12.0:1616513082.357483:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513082.357484:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:12.0:1616513082.357485:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513082.357490:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79be48 00000800:00000200:12.0:1616513082.357493:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513082.357495:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513082.357496:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513082.357498:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513082.357501:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.357503:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.131@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.357512:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.131@tcp1 of length 224 into portal 12 MB=0x604b426702580 00000400:00000200:20.0:1616513082.357518:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.131@tcp1 of length 224/224 into md 0x865bbec1 [1] + 51408 00000400:00000200:20.0:1616513082.357523:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.357526:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513082.357533:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.357536:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.357538:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.357554:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694021645771968 00010000:00000200:16.0:1616513082.357565:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1694021645771968/t0(0) o400->469d0d2f-a0d7-5abc-3ab6-c7312cd07c4f@10.31.143.131@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.357579:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694021645771968, offset 192 00000400:00000200:16.0:1616513082.357586:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.131@tcp1 00000400:00000200:16.0:1616513082.357593:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.131@tcp1 routed destination 00000100:00000200:0.0:1616513082.357599:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694021645772160 00000400:00000200:16.0:1616513082.357608:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.131@tcp1(10.31.143.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513082.357612:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08d100 x1694021645772160/t0(0) o400->0de0fc37-c369-b45b-b5a8-754f622f2408@10.31.143.131@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.357615:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.357620:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.357623:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.357625:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513082.357626:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694021645772160, offset 192 00000800:00000200:16.0:1616513082.357630:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513082.357633:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.131@tcp1 00000800:00000200:7.2:1616513082.357636:0:49698:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.357637:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513082.357640:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.131@tcp1 routed destination 00000400:00000200:0.0:1616513082.357655:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.131@tcp1(10.31.143.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.357664:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.357670:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.357673:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.357675:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.357679:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.357687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513082.357691:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.357698:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.357701:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513082.357706:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.357707:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.357713:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43100 00000800:00000200:26.0:1616513082.357717:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513082.357741:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.357748:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.357751:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513082.357752:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513082.357758:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.357762:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb055c8 00000800:00000200:28.0:1616513082.357766:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.370087:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513082.370145:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.370154:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.370163:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.49@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.370173:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.49@tcp1 of length 224 into portal 26 MB=0x5ffdd1f8c9d40 00000400:00000200:22.0:1616513082.370179:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.49@tcp1 of length 224/224 into md 0xf97c9d49 [1] + 0 00000400:00000200:22.0:1616513082.370184:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b3a8 00000400:00000200:22.0:1616513082.370187:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.370190:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513082.370194:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b3a8 00000800:00000200:22.0:1616513082.370197:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.370199:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.370199:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.370203:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.370277:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700065717568 00010000:00000200:16.0:1616513082.370288:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688700065717568/t0(0) o400->bdc828ba-2da1-ae01-23c4-a016b457e290@10.31.129.49@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.370302:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700065717568, offset 192 00000400:00000200:16.0:1616513082.370309:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.49@tcp1 00000400:00000200:16.0:1616513082.370317:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.49@tcp1 routed destination 00000400:00000200:16.0:1616513082.370332:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.49@tcp1(10.31.129.49@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.370339:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.370345:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.370349:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.370351:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.370356:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.370361:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.370363:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513082.370414:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.370421:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.370424:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.370430:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.370447:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a436d8 00000800:00000200:14.0:1616513082.370451:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.370453:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.370844:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.370855:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.370863:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513082.370865:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513082.370872:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.49@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.370881:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.49@tcp1 of length 224 into portal 12 MB=0x5ffdd1f8c9f00 00000400:00000200:2.0:1616513082.370889:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.49@tcp1 of length 224/224 into md 0xf964d519 [1] + 20384 00000400:00000200:2.0:1616513082.370894:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.370897:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513082.370904:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.370907:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.370909:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.370919:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:30.0:1616513082.370923:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700065718016 00010000:00000200:30.0:1616513082.370934:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d2d00 x1688700065718016/t0(0) o400->27c699f9-a200-6322-8707-5e2234f3e6a9@10.31.129.49@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513082.370949:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700065718016, offset 192 00000400:00000200:30.0:1616513082.370955:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.49@tcp1 00000400:00000200:30.0:1616513082.370962:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.49@tcp1 routed destination 00000400:00000200:30.0:1616513082.370977:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.49@tcp1(10.31.129.49@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513082.370983:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513082.370989:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.370989:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513082.370992:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513082.370994:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:30.0:1616513082.370998:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513082.370998:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.371007:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.69@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.371016:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.69@tcp1 of length 224 into portal 26 MB=0x5ffd465faabc0 00000400:00000200:20.0:1616513082.371023:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.69@tcp1 of length 224/224 into md 0xf97c9d59 [1] + 0 00000400:00000200:20.0:1616513082.371028:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a110 00000400:00000200:20.0:1616513082.371030:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.371033:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.371038:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a110 00000800:00000200:20.0:1616513082.371041:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513082.371043:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513082.371045:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513082.371050:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513082.371051:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:16.0:1616513082.371051:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662592629696 00000800:00000200:20.0:1616513082.371053:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.371054:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:16.0:1616513082.371057:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688662592629696/t0(0) o400->4d4de097-6447-cc43-836e-89ab8c1c0447@10.31.138.69@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:24.0:1616513082.371060:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.371064:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeff0 00000800:00000200:24.0:1616513082.371068:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.371068:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662592629696, offset 192 00000400:00000200:16.0:1616513082.371073:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.69@tcp1 00000400:00000200:16.0:1616513082.371079:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.69@tcp1 routed destination 00000400:00000200:16.0:1616513082.371092:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.69@tcp1(10.31.138.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.371098:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.371103:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.371105:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.371107:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.371110:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513082.371114:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.371114:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.371123:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.371127:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.371129:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513082.371133:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513082.371134:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.371138:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ba0 00000800:00000200:14.0:1616513082.371140:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.373763:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.373832:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.373841:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.373850:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.69@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.373859:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.69@tcp1 of length 224 into portal 12 MB=0x5ffd465faacc0 00000400:00000200:28.0:1616513082.373865:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.69@tcp1 of length 224/224 into md 0x865bbec1 [1] + 51632 00000400:00000200:28.0:1616513082.373870:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.373873:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.373881:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.373884:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.373885:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.373889:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.373943:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662592629952 00010000:00000200:0.0:1616513082.373954:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeeda400 x1688662592629952/t0(0) o400->a04f9276-79c5-b388-3ec7-d9585286e113@10.31.138.69@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.373968:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662592629952, offset 192 00000400:00000200:0.0:1616513082.373973:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.69@tcp1 00000400:00000200:0.0:1616513082.373980:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.69@tcp1 routed destination 00000400:00000200:0.0:1616513082.373994:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.69@tcp1(10.31.138.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.374000:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.374005:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.374008:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.374010:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.374014:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.374019:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.374072:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.374078:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.374080:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513082.374087:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.374092:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04dd0 00000800:00000200:24.0:1616513082.374095:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.374098:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.380612:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.380668:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.380677:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.380685:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.242@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.380695:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.242@tcp1 of length 224 into portal 26 MB=0x603e0ef704540 00000400:00000200:20.0:1616513082.380701:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.242@tcp1 of length 224/224 into md 0xf97c9d69 [1] + 0 00000400:00000200:20.0:1616513082.380706:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a330 00000400:00000200:20.0:1616513082.380709:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.380711:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.380716:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a330 00000800:00000200:20.0:1616513082.380719:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.380721:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.380722:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.380728:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.380731:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.380731:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513082.380740:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.242@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.380749:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.242@tcp1 of length 224 into portal 12 MB=0x603e0ef704600 00000400:00000200:28.0:1616513082.380755:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.242@tcp1 of length 224/224 into md 0x865bbec1 [1] + 51856 00000400:00000200:28.0:1616513082.380761:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.380763:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.380769:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.380773:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.380774:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.380779:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114484933952 00010000:00000200:16.0:1616513082.380790:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1693114484933952/t0(0) o400->ce2d6fb5-3e11-4530-41b3-9e14d56f6ce1@10.31.136.242@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.380804:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114484933952, offset 192 00000400:00000200:16.0:1616513082.380810:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.242@tcp1 00000400:00000200:16.0:1616513082.380818:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.242@tcp1 routed destination 00000400:00000200:16.0:1616513082.380833:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.242@tcp1(10.31.136.242@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513082.380833:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114484934144 00000800:00000200:16.0:1616513082.380840:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.380845:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513082.380845:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed9680 x1693114484934144/t0(0) o400->85428b99-f6aa-2b6d-1ccc-ecb4af9f8dfd@10.31.136.242@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.380848:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.380850:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.380855:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513082.380859:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114484934144, offset 192 00000100:00000200:16.0:1616513082.380862:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.380862:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513082.380866:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.242@tcp1 00000400:00000200:0.0:1616513082.380873:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.242@tcp1 routed destination 00000400:00000200:0.0:1616513082.380888:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.242@tcp1(10.31.136.242@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.380895:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.380901:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.380904:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.380906:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.380911:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.380915:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513082.380918:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.380925:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.380927:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.380934:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.380940:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43e48 00000800:00000200:22.0:1616513082.380943:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.380946:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513082.380968:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.380975:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.380990:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513082.380996:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.380997:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.381001:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:12.0:1616513082.381004:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.396496:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513082.396550:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513082.396551:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.396560:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.396570:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.191@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.396579:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.191@tcp1 of length 224 into portal 26 MB=0x5fa05114e5e00 00000400:00000200:28.0:1616513082.396585:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.191@tcp1 of length 224/224 into md 0xf97c9d79 [1] + 0 00000400:00000200:28.0:1616513082.396590:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b6d8 00000400:00000200:28.0:1616513082.396593:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.396595:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.396600:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b6d8 00000800:00000200:28.0:1616513082.396603:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.396606:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513082.396606:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513082.396608:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.396611:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513082.396616:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.396625:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.191@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.396634:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.191@tcp1 of length 224 into portal 12 MB=0x5fa05114e5e80 00000400:00000200:26.0:1616513082.396640:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.191@tcp1 of length 224/224 into md 0xf964d519 [1] + 20608 00000400:00000200:26.0:1616513082.396645:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.396648:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.396657:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.396660:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.396661:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.396664:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.396686:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682274555682304 00010000:00000200:16.0:1616513082.396697:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1682274555682304/t0(0) o400->6d485cb9-2b21-014c-ca42-7beb5b344a2e@10.31.131.191@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.396712:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1682274555682304, offset 192 00000400:00000200:16.0:1616513082.396718:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.191@tcp1 00000100:00000200:2.0:1616513082.396722:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682274555682432 00000400:00000200:16.0:1616513082.396725:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.191@tcp1 routed destination 00010000:00000200:2.0:1616513082.396734:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57bf00 x1682274555682432/t0(0) o400->5873c1f7-9582-d047-30d0-52e245cfa9cc@10.31.131.191@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513082.396741:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.191@tcp1(10.31.131.191@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.396747:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:2.0:1616513082.396748:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1682274555682432, offset 192 00000800:00000200:16.0:1616513082.396753:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513082.396755:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.191@tcp1 00000800:00000200:16.0:1616513082.396756:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.396758:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:2.0:1616513082.396762:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.191@tcp1 routed destination 00000800:00000200:16.0:1616513082.396763:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.396768:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.396770:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513082.396777:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.191@tcp1(10.31.131.191@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513082.396783:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513082.396789:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513082.396792:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513082.396794:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513082.396799:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.396804:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513082.396823:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.396838:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.396841:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513082.396843:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.396848:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.396853:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42880 00000800:00000200:22.0:1616513082.396857:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513082.396876:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.396882:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.396885:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.396891:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.396895:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eeded0 00000800:00000200:20.0:1616513082.396899:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.396901:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.398351:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513082.398404:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.398413:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.398422:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.166@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.398432:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.166@tcp1 of length 224 into portal 26 MB=0x6011d8d313740 00000400:00000200:26.0:1616513082.398457:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.166@tcp1 of length 224/224 into md 0xf97c9d99 [1] + 0 00000800:00000200:14.0:1616513082.398457:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513082.398462:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a198 00000400:00000200:26.0:1616513082.398465:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.398468:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513082.398468:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.398473:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a198 00000400:00000200:14.0:1616513082.398476:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.166@tcp1 : PUT - for me 00000800:00000200:26.0:1616513082.398477:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513082.398479:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513082.398481:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.398484:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513082.398485:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.166@tcp1 of length 224 into portal 12 MB=0x6011d8d313800 00000400:00000200:14.0:1616513082.398491:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.166@tcp1 of length 224/224 into md 0xf964d519 [1] + 20832 00000400:00000200:14.0:1616513082.398496:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.398499:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513082.398507:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.398510:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.398512:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513082.398527:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690076294756352 00010000:00000200:6.0:1616513082.398540:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba1680 x1690076294756352/t0(0) o400->105357ae-d873-cad5-5fac-ad18fe5c9ad6@10.31.138.166@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513082.398546:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690076294756160 00000100:00000200:6.0:1616513082.398554:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690076294756352, offset 192 00010000:00000200:16.0:1616513082.398558:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1690076294756160/t0(0) o400->7257e542-52de-bff9-1bf6-ef50501a618e@10.31.138.166@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:6.0:1616513082.398560:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.166@tcp1 00000400:00000200:6.0:1616513082.398568:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.166@tcp1 routed destination 00000100:00000200:16.0:1616513082.398572:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690076294756160, offset 192 00000400:00000200:16.0:1616513082.398578:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.166@tcp1 00000400:00000200:6.0:1616513082.398583:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.166@tcp1(10.31.138.166@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513082.398586:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.166@tcp1 routed destination 00000800:00000200:6.0:1616513082.398590:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:6.0:1616513082.398596:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513082.398598:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513082.398600:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513082.398601:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.166@tcp1(10.31.138.166@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513082.398605:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513082.398607:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513082.398610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.398628:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.398631:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.398633:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513082.398637:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513082.398637:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.398643:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513082.398644:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:16.0:1616513082.398644:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513082.398647:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513082.398648:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.398654:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.398658:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01f58 00000800:00000200:22.0:1616513082.398662:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513082.398667:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.398674:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.398677:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.398683:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513082.398684:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513082.398689:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ff0 00000800:00000200:28.0:1616513082.398692:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.402374:0:49703:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.402429:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.402449:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.402459:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.402468:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d0322b1500 00000400:00000200:14.0:1616513082.402474:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0xf964d519 [1] + 21056 00000400:00000200:14.0:1616513082.402482:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513082.402482:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513082.402484:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513082.402491:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.402494:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.402496:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513082.402505:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042590029056 00010000:00000200:6.0:1616513082.402581:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba1f80 x1693042590029056/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:708/0 lens 440/368 e 0 to 0 dl 1616513088 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:6.0:1616513082.402592:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042590029056, offset 192 00000400:00000200:6.0:1616513082.402597:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:6.0:1616513082.402602:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:6.0:1616513082.402613:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513082.402619:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:6.0:1616513082.402623:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513082.402625:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513082.402626:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513082.402630:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.402635:0:49703:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.402690:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.402697:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.402700:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.402706:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.402710:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01980 00000800:00000200:26.0:1616513082.402714:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.402717:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.421219:0:49705:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.421274:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.421284:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.421293:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.103@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.421302:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.103@tcp1 of length 224 into portal 26 MB=0x5ffd5c47cab80 00000400:00000200:2.0:1616513082.421309:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.103@tcp1 of length 224/224 into md 0xf97c9da9 [1] + 0 00000400:00000200:2.0:1616513082.421314:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a550 00000400:00000200:2.0:1616513082.421317:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.421319:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.421324:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a550 00000800:00000200:2.0:1616513082.421327:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.421328:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.421329:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.421333:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.421390:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668473174912 00010000:00000200:16.0:1616513082.421401:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688668473174912/t0(0) o400->36ab7259-165e-2db8-13a6-25e4787f583e@10.31.131.103@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.421415:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668473174912, offset 192 00000400:00000200:16.0:1616513082.421423:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.103@tcp1 00000400:00000200:16.0:1616513082.421431:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.103@tcp1 routed destination 00000400:00000200:16.0:1616513082.421462:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.103@tcp1(10.31.131.103@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513082.421468:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.421469:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.421475:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.421478:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.421480:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.421485:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:16.0:1616513082.421493:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513082.421524:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.421533:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.421542:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.103@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.421551:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.103@tcp1 of length 224 into portal 12 MB=0x5ffd5c47cae80 00000400:00000200:20.0:1616513082.421557:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.103@tcp1 of length 224/224 into md 0x865bbec1 [1] + 52080 00000400:00000200:20.0:1616513082.421562:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.421565:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513082.421572:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513082.421577:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513082.421577:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:20.0:1616513082.421584:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513082.421585:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513082.421587:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513082.421588:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513082.421594:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513082.421600:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42bb0 00000800:00000200:12.0:1616513082.421603:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.421636:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668473175680 00010000:00000200:0.0:1616513082.421648:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08e780 x1688668473175680/t0(0) o400->b66be606-6f16-2d54-9cad-640d6becb5c8@10.31.131.103@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.421662:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668473175680, offset 192 00000400:00000200:0.0:1616513082.421667:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.103@tcp1 00000400:00000200:0.0:1616513082.421674:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.103@tcp1 routed destination 00000400:00000200:0.0:1616513082.421688:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.103@tcp1(10.31.131.103@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.421694:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.421699:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.421702:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513082.421704:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.421708:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.421713:0:49705:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.421775:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.421782:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.421785:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.421792:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.421796:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05210 00000800:00000200:26.0:1616513082.421800:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.421803:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.427218:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.427273:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.427283:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.427292:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.216@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.427301:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.216@tcp1 of length 224 into portal 26 MB=0x6018631309940 00000400:00000200:28.0:1616513082.427308:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.216@tcp1 of length 224/224 into md 0xf97c9dc9 [1] + 0 00000400:00000200:28.0:1616513082.427312:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79bb18 00000400:00000200:28.0:1616513082.427315:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.427318:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.427322:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bb18 00000800:00000200:28.0:1616513082.427325:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.427328:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.427328:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.427332:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.427399:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513082.427402:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690525722777920 00010000:00000200:16.0:1616513082.427412:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1690525722777920/t0(0) o400->6ebdf3d3-99f8-2a31-a08c-588738a43a5e@10.31.141.216@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.427425:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690525722777920, offset 192 00000400:00000200:16.0:1616513082.427431:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.216@tcp1 00000400:00000200:16.0:1616513082.427449:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.216@tcp1 routed destination 00000800:00000200:14.0:1616513082.427454:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513082.427463:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.216@tcp1(10.31.141.216@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513082.427463:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.427469:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513082.427472:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.216@tcp1 : PUT - for me 00000800:00000200:16.0:1616513082.427475:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.427477:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.427479:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513082.427481:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.216@tcp1 of length 224 into portal 12 MB=0x6018631309b80 00000800:00000200:16.0:1616513082.427484:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513082.427487:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.216@tcp1 of length 224/224 into md 0xf964d519 [1] + 21496 00000800:00000200:11.2:1616513082.427488:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.427490:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513082.427492:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.427495:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513082.427502:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.427505:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.427507:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.427511:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513082.427517:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690525722778496 00010000:00000200:6.0:1616513082.427526:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba5100 x1690525722778496/t0(0) o400->c7ac1e29-9cc3-56dd-0987-38e8c5549221@10.31.141.216@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513082.427538:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690525722778496, offset 192 00000800:00000200:12.0:1616513082.427540:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:6.0:1616513082.427543:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.216@tcp1 00000800:00000200:12.0:1616513082.427547:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:6.0:1616513082.427548:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.216@tcp1 routed destination 00000400:00000200:12.0:1616513082.427550:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513082.427556:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:6.0:1616513082.427560:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.216@tcp1(10.31.141.216@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513082.427563:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:6.0:1616513082.427566:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513082.427567:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513082.427570:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:6.0:1616513082.427570:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513082.427572:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513082.427574:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513082.427578:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.427582:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.427635:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.427641:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.427644:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.427650:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.427653:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01dc0 00000800:00000200:26.0:1616513082.427657:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.427660:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.429964:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.430017:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.430026:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.430035:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.248@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.430044:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.248@tcp1 of length 224 into portal 26 MB=0x604192e09ba80 00000400:00000200:2.0:1616513082.430051:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.248@tcp1 of length 224/224 into md 0xf97c9dd9 [1] + 0 00000400:00000200:2.0:1616513082.430056:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b430 00000400:00000200:2.0:1616513082.430059:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.430061:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.430065:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b430 00000800:00000200:2.0:1616513082.430068:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.430070:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.430071:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.430077:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.430080:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.430081:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.430090:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.248@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.430098:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.248@tcp1 of length 224 into portal 12 MB=0x604192e09bb00 00000400:00000200:14.0:1616513082.430105:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.248@tcp1 of length 224/224 into md 0xf964d519 [1] + 21720 00000400:00000200:14.0:1616513082.430110:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.430112:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513082.430121:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.430124:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.430126:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.430128:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356053346944 00010000:00000200:16.0:1616513082.430138:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1693356053346944/t0(0) o400->46b5e9a3-ba64-0f59-6349-179af3f120cb@10.31.128.248@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.430152:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693356053346944, offset 192 00000400:00000200:16.0:1616513082.430157:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.248@tcp1 00000400:00000200:16.0:1616513082.430164:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.248@tcp1 routed destination 00000400:00000200:16.0:1616513082.430178:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.248@tcp1(10.31.128.248@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513082.430181:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356053347072 00000800:00000200:16.0:1616513082.430188:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:10.0:1616513082.430192:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba7980 x1693356053347072/t0(0) o400->e2475a08-3ba4-5c0a-e6b2-c333fceae52f@10.31.128.248@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.430193:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.430196:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.430197:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.430202:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:10.0:1616513082.430205:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693356053347072, offset 192 00000100:00000200:16.0:1616513082.430208:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.430208:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513082.430211:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.248@tcp1 00000400:00000200:10.0:1616513082.430218:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.248@tcp1 routed destination 00000400:00000200:10.0:1616513082.430232:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.248@tcp1(10.31.128.248@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513082.430238:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513082.430243:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513082.430246:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513082.430248:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513082.430252:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.430256:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513082.430261:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.430268:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.430271:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.430277:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.430283:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42110 00000800:00000200:20.0:1616513082.430286:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.430289:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513082.430310:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.430317:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.430320:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513082.430321:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.430327:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.430331:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83078 00000800:00000200:22.0:1616513082.430335:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.437956:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.438011:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.438020:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.438029:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.77@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.438039:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.77@tcp1 of length 224 into portal 26 MB=0x5ffd4040af980 00000400:00000200:14.0:1616513082.438046:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.77@tcp1 of length 224/224 into md 0xf97c9df9 [1] + 0 00000400:00000200:14.0:1616513082.438055:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79aa18 00000400:00000200:14.0:1616513082.438058:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.438061:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513082.438062:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513082.438065:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aa18 00000800:00000200:14.0:1616513082.438068:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.438070:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.438072:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.438110:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513082.438126:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660949531008 00010000:00000200:16.0:1616513082.438136:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688660949531008/t0(0) o400->8468f85e-6ddd-ec04-9f25-b8be623b3b8d@10.31.129.77@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.438150:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660949531008, offset 192 00000400:00000200:16.0:1616513082.438155:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.77@tcp1 00000400:00000200:16.0:1616513082.438163:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.77@tcp1 routed destination 00000800:00000200:2.0:1616513082.438163:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.438173:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513082.438177:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.77@tcp1(10.31.129.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513082.438181:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.77@tcp1 : PUT - for me 00000800:00000200:16.0:1616513082.438183:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.438188:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513082.438190:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.77@tcp1 of length 224 into portal 12 MB=0x5ffd4040afc00 00000800:00000200:16.0:1616513082.438191:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.438193:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.438197:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513082.438197:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.77@tcp1 of length 224/224 into md 0xf964d519 [1] + 21944 00000400:00000200:2.0:1616513082.438202:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513082.438203:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:2.0:1616513082.438204:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:11.2:1616513082.438205:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513082.438211:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.438214:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.438216:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.438219:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513082.438259:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513082.438266:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513082.438269:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513082.438275:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:10.0:1616513082.438275:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660949531648 00000400:00000200:12.0:1616513082.438281:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43210 00000800:00000200:12.0:1616513082.438284:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513082.438287:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:10.0:1616513082.438287:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d3600 x1688660949531648/t0(0) o400->0b565a62-6978-aa75-c298-e30e628b007e@10.31.129.77@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513082.438302:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660949531648, offset 192 00000400:00000200:10.0:1616513082.438310:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.77@tcp1 00000400:00000200:10.0:1616513082.438317:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.77@tcp1 routed destination 00000400:00000200:10.0:1616513082.438332:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.77@tcp1(10.31.129.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513082.438338:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513082.438344:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513082.438346:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513082.438348:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513082.438353:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.438358:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.438369:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.438374:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.438376:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513082.438378:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.438382:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.438386:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83ba0 00000800:00000200:26.0:1616513082.438390:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.443635:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.443692:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.443700:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.443709:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.42@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.443717:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.42@tcp1 of length 224 into portal 26 MB=0x5ffd4f1cd31c0 00000400:00000200:2.0:1616513082.443723:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.42@tcp1 of length 224/224 into md 0xf97c9e09 [1] + 0 00000400:00000200:2.0:1616513082.443728:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79ba90 00000400:00000200:2.0:1616513082.443732:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.443734:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.443738:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79ba90 00000800:00000200:2.0:1616513082.443741:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.443744:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.443745:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513082.443752:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.443755:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.443756:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.443765:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.42@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.443774:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.42@tcp1 of length 224 into portal 12 MB=0x5ffd4f1cd3300 00000400:00000200:22.0:1616513082.443780:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.42@tcp1 of length 224/224 into md 0xf964d519 [1] + 22168 00000400:00000200:22.0:1616513082.443786:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.443788:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513082.443795:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.443798:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.443800:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.443802:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664938459584 00010000:00000200:16.0:1616513082.443812:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688664938459584/t0(0) o400->dbe1fef1-1920-e2be-2a22-048d20ddd4a4@10.31.136.42@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.443825:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664938459584, offset 192 00000400:00000200:16.0:1616513082.443831:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.42@tcp1 00000400:00000200:16.0:1616513082.443838:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.42@tcp1 routed destination 00000400:00000200:16.0:1616513082.443852:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.42@tcp1(10.31.136.42@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513082.443857:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664938459904 00000800:00000200:16.0:1616513082.443858:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.443863:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.443866:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.443869:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:10.0:1616513082.443869:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec203a80 x1688664938459904/t0(0) o400->4f24cbef-4441-a073-6be7-101bf1a1c62b@10.31.136.42@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.443874:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.443880:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.443880:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:10.0:1616513082.443883:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664938459904, offset 192 00000400:00000200:10.0:1616513082.443889:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.42@tcp1 00000400:00000200:10.0:1616513082.443897:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.42@tcp1 routed destination 00000400:00000200:10.0:1616513082.443911:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.42@tcp1(10.31.136.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513082.443918:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513082.443923:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513082.443926:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513082.443928:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513082.443932:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513082.443937:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513082.443938:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513082.443944:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.443947:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.443954:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513082.443954:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.443959:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a423b8 00000800:00000200:20.0:1616513082.443962:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.443964:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.443966:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.443967:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.443970:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.443971:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513082.443991:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.443998:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.444002:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.444003:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.444009:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.444013:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83d38 00000800:00000200:26.0:1616513082.444017:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.454920:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.454975:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.454985:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.454994:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.198@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.455002:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.198@tcp1 of length 224 into portal 26 MB=0x5ffdcaf74e880 00000400:00000200:14.0:1616513082.455009:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.198@tcp1 of length 224/224 into md 0xf97c9e29 [1] + 0 00000400:00000200:14.0:1616513082.455014:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b298 00000400:00000200:14.0:1616513082.455016:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.455019:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513082.455023:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b298 00000800:00000200:14.0:1616513082.455026:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.455029:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.455030:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.455033:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.455074:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513082.455085:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698185115776 00010000:00000200:16.0:1616513082.455095:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688698185115776/t0(0) o400->12f6f6a0-8078-7688-80b6-2ebea7450d58@10.31.128.198@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.455108:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698185115776, offset 192 00000400:00000200:16.0:1616513082.455114:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.198@tcp1 00000400:00000200:16.0:1616513082.455121:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.198@tcp1 routed destination 00000800:00000200:24.0:1616513082.455130:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513082.455135:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.198@tcp1(10.31.128.198@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513082.455141:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.455141:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.455146:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.455149:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513082.455150:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.198@tcp1 : PUT - for me 00000800:00000200:16.0:1616513082.455151:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513082.455156:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:24.0:1616513082.455158:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.198@tcp1 of length 224 into portal 12 MB=0x5ffdcaf74ebc0 00000100:00000200:16.0:1616513082.455162:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513082.455164:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.198@tcp1 of length 224/224 into md 0x865bbec1 [1] + 52304 00000400:00000200:24.0:1616513082.455170:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.455172:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.455180:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513082.455183:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513082.455184:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:24.0:1616513082.455192:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513082.455193:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513082.455195:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.455196:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.455202:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.455208:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43078 00000800:00000200:20.0:1616513082.455211:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.455246:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698185116608 00010000:00000200:0.0:1616513082.455258:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd704380 x1688698185116608/t0(0) o400->2d7e9606-1514-a190-d57c-5dbafef72a17@10.31.128.198@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.455273:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698185116608, offset 192 00000400:00000200:0.0:1616513082.455279:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.198@tcp1 00000400:00000200:0.0:1616513082.455286:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.198@tcp1 routed destination 00000400:00000200:0.0:1616513082.455302:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.198@tcp1(10.31.128.198@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.455308:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.455314:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.455317:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513082.455319:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.455324:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.455328:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.455381:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.455387:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.455390:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.455397:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.455401:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05870 00000800:00000200:2.0:1616513082.455405:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.455408:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.464816:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513082.464825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513082.464873:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.464882:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.464891:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.109.31@tcp1 : PUT - for me 00000800:00000200:22.0:1616513082.464894:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513082.464900:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.109.31@tcp1 of length 224 into portal 26 MB=0x5fa58fffb7200 00000800:00000200:22.0:1616513082.464903:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513082.464907:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.109.31@tcp1 of length 224/224 into md 0xf97c9e49 [1] + 0 00000400:00000200:28.0:1616513082.464911:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b980 00000400:00000200:22.0:1616513082.464912:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.109.31@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.464914:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.464917:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.464921:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b980 00000400:00000200:22.0:1616513082.464921:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.109.31@tcp1 of length 224 into portal 12 MB=0x5fa58fffb7280 00000800:00000200:28.0:1616513082.464924:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.464927:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513082.464927:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.109.31@tcp1 of length 224/224 into md 0xf964d519 [1] + 22392 00000800:00000200:28.0:1616513082.464929:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.464932:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513082.464934:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.464937:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513082.464944:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.464947:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.464949:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.464952:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.464988:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682635042288128 00010000:00000200:16.0:1616513082.464998:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1682635042288128/t0(0) o400->04e6b7cb-8bb2-c244-7911-bf763f8eb769@10.242.109.31@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.465010:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1682635042288128, offset 192 00000400:00000200:16.0:1616513082.465015:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.109.31@tcp1 00000100:00000200:10.0:1616513082.465018:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682635042288256 00000400:00000200:16.0:1616513082.465022:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.109.31@tcp1 routed destination 00010000:00000200:10.0:1616513082.465030:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec204800 x1682635042288256/t0(0) o400->230dbbd8-5517-59ae-8187-4a51d5f84a24@10.242.109.31@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513082.465035:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.109.31@tcp1(10.242.109.31@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.465041:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:10.0:1616513082.465043:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1682635042288256, offset 192 00000800:00000200:16.0:1616513082.465045:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.465048:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:10.0:1616513082.465049:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.109.31@tcp1 00000800:00000200:16.0:1616513082.465050:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.465054:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513082.465055:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.109.31@tcp1 routed destination 00000800:00000200:11.2:1616513082.465058:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.465060:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:10.0:1616513082.465069:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.109.31@tcp1(10.242.109.31@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513082.465075:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513082.465080:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513082.465084:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513082.465086:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513082.465091:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.465095:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513082.465112:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.465119:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.465121:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.465123:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.465128:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.465133:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43210 00000800:00000200:20.0:1616513082.465137:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513082.465151:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.465158:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.465161:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.465168:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.465172:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e834b8 00000800:00000200:2.0:1616513082.465176:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.465178:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.471644:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.471699:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.471708:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513082.471710:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513082.471717:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.49@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.471726:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.49@tcp1 of length 224 into portal 26 MB=0x5ffe0923a4d80 00000800:00000200:14.0:1616513082.471727:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513082.471732:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.49@tcp1 of length 224/224 into md 0xf97c9e69 [1] + 0 00000800:00000200:14.0:1616513082.471736:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513082.471737:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b540 00000400:00000200:24.0:1616513082.471740:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.471742:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513082.471745:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.49@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.471747:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b540 00000800:00000200:24.0:1616513082.471750:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.471753:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.471753:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513082.471754:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.49@tcp1 of length 224 into portal 12 MB=0x5ffe0923a5000 00000800:00000200:24.0:1616513082.471757:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513082.471760:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.49@tcp1 of length 224/224 into md 0xf964d519 [1] + 22616 00000400:00000200:14.0:1616513082.471765:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.471768:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513082.471774:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.471777:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.471779:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.471782:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.471814:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714874604928 00010000:00000200:16.0:1616513082.471825:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688714874604928/t0(0) o400->4ada40e9-5ba2-2c63-a848-ae4630f5f76f@10.31.142.49@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.471841:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688714874604928, offset 192 00000100:00000200:10.0:1616513082.471842:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714874605568 00000400:00000200:16.0:1616513082.471847:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.49@tcp1 00010000:00000200:10.0:1616513082.471854:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba5e80 x1688714874605568/t0(0) o400->4988a4a0-470b-a8f4-d8cd-9c82d267be1f@10.31.142.49@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513082.471855:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.49@tcp1 routed destination 00000100:00000200:10.0:1616513082.471869:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688714874605568, offset 192 00000400:00000200:16.0:1616513082.471870:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.49@tcp1(10.31.142.49@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513082.471875:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.49@tcp1 00000800:00000200:16.0:1616513082.471876:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.471882:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:10.0:1616513082.471883:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.49@tcp1 routed destination 00000800:00000200:16.0:1616513082.471884:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.471887:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.471891:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.471897:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513082.471897:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.49@tcp1(10.31.142.49@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513082.471898:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513082.471904:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513082.471907:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513082.471910:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513082.471912:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:10.0:1616513082.471913:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513082.471915:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513082.471916:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513082.471919:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513082.471923:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513082.471926:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513082.471928:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:28.0:1616513082.471932:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.471934:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513082.471982:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.471989:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.471992:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.471999:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.472003:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82ff0 00000800:00000200:2.0:1616513082.472007:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.472009:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.474384:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.474396:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.474407:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513082.474414:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513082.474416:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.112@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.474425:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.112@tcp1 of length 224 into portal 26 MB=0x604a2f797e600 00000400:00000200:14.0:1616513082.474432:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.112@tcp1 of length 224/224 into md 0xf97c9e79 [1] + 0 00000400:00000200:14.0:1616513082.474447:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79ad48 00000400:00000200:14.0:1616513082.474450:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.474453:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513082.474457:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79ad48 00000800:00000200:14.0:1616513082.474460:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.474463:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.474465:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.474467:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513082.474475:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.474484:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.474493:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.112@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.474502:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.112@tcp1 of length 224 into portal 12 MB=0x604a2f797e680 00000400:00000200:28.0:1616513082.474508:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.112@tcp1 of length 224/224 into md 0x865bbec1 [1] + 52528 00000400:00000200:28.0:1616513082.474514:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.474516:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:16.0:1616513082.474518:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947845404160 00000800:00000200:28.0:1616513082.474523:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.474526:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.474528:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:16.0:1616513082.474528:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1693947845404160/t0(0) o400->859c499e-a979-e9e0-f1ef-9287aa1b3c26@10.31.140.112@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513082.474531:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.474541:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693947845404160, offset 192 00000400:00000200:16.0:1616513082.474547:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.112@tcp1 00000400:00000200:16.0:1616513082.474554:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.112@tcp1 routed destination 00000400:00000200:16.0:1616513082.474568:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.112@tcp1(10.31.140.112@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.474574:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.474579:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.474581:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.474583:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.474588:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.474593:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.474594:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513082.474603:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513082.474603:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947845404288 00000800:00000200:28.0:1616513082.474606:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.474609:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.474615:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00010000:00000200:0.0:1616513082.474615:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed8000 x1693947845404288/t0(0) o400->2b72b894-c95d-49aa-0911-427f5a6cd7f6@10.31.140.112@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513082.474619:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ff0 00000800:00000200:28.0:1616513082.474622:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.474625:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.474630:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693947845404288, offset 192 00000400:00000200:0.0:1616513082.474636:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.112@tcp1 00000400:00000200:0.0:1616513082.474643:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.112@tcp1 routed destination 00000400:00000200:0.0:1616513082.474660:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.112@tcp1(10.31.140.112@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.474667:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.474672:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.474675:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513082.474677:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.474682:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.474687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.474739:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.474745:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.474747:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.474754:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.474757:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04660 00000800:00000200:2.0:1616513082.474761:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.474766:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.478692:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.478746:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.478755:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.478764:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.195@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.478774:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.195@tcp1 of length 224 into portal 26 MB=0x5ffe3b2e68100 00000400:00000200:28.0:1616513082.478781:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.195@tcp1 of length 224/224 into md 0xf97c9ea9 [1] + 0 00000400:00000200:28.0:1616513082.478785:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a5d8 00000400:00000200:28.0:1616513082.478788:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.478791:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.478795:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a5d8 00000800:00000200:28.0:1616513082.478798:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.478800:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.478801:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.478804:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.478859:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728307663104 00010000:00000200:16.0:1616513082.478869:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688728307663104/t0(0) o400->0acd30d4-7cc9-454c-8944-a6ab95a2746e@10.31.128.195@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.478882:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688728307663104, offset 192 00000400:00000200:16.0:1616513082.478888:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.195@tcp1 00000400:00000200:16.0:1616513082.478896:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.195@tcp1 routed destination 00000400:00000200:16.0:1616513082.478911:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.195@tcp1(10.31.128.195@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.478917:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.478922:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.478924:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.478926:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.478931:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.478936:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.478937:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513082.478946:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.478950:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.478952:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.478957:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.478962:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43e48 00000800:00000200:28.0:1616513082.478964:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.478967:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.479173:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.479184:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.479192:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513082.479193:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.479201:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.195@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.479210:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.195@tcp1 of length 224 into portal 12 MB=0x5ffe3b2e68540 00000400:00000200:24.0:1616513082.479218:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.195@tcp1 of length 224/224 into md 0x865bbec1 [1] + 52752 00000400:00000200:24.0:1616513082.479223:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.479226:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.479233:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.479236:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.479238:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.479296:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728307664192 00010000:00000200:0.0:1616513082.479307:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd702400 x1688728307664192/t0(0) o400->bc67ce47-316f-63ed-fb60-f198699d9b5e@10.31.128.195@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.479321:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688728307664192, offset 192 00000400:00000200:0.0:1616513082.479327:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.195@tcp1 00000400:00000200:0.0:1616513082.479334:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.195@tcp1 routed destination 00000400:00000200:0.0:1616513082.479348:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.195@tcp1(10.31.128.195@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.479354:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.479359:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.479362:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513082.479364:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.479368:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.479374:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.479424:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.479431:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.479445:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.479451:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.479455:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04660 00000800:00000200:14.0:1616513082.479459:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.479462:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.530002:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513082.530049:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513082.530061:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.530072:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.530081:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.46@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.530091:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.46@tcp1 of length 224 into portal 26 MB=0x5fff48df9af40 00000400:00000200:28.0:1616513082.530099:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.46@tcp1 of length 224/224 into md 0xf97c9eb9 [1] + 0 00000400:00000200:28.0:1616513082.530104:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a880 00000800:00000200:2.0:1616513082.530106:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513082.530107:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.530110:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.530116:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a880 00000800:00000200:2.0:1616513082.530116:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:28.0:1616513082.530119:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.530122:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.530122:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513082.530124:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.46@tcp1 : PUT - for me 00000800:00000200:28.0:1616513082.530126:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513082.530133:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.46@tcp1 of length 224 into portal 12 MB=0x5fff48df9b140 00000400:00000200:2.0:1616513082.530140:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.46@tcp1 of length 224/224 into md 0xf964d519 [1] + 22840 00000400:00000200:2.0:1616513082.530145:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.530147:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513082.530154:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.530157:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.530159:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.530161:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513082.530176:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688800702607680 00010000:00000200:10.0:1616513082.530188:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d7980 x1688800702607680/t0(0) o400->7e26dcb3-a8b3-c568-9f0c-4b10428e2fa2@10.31.142.46@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513082.530193:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688800702607168 00000100:00000200:10.0:1616513082.530202:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688800702607680, offset 192 00010000:00000200:16.0:1616513082.530204:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688800702607168/t0(0) o400->41ba2a93-57ad-bae7-e885-d78877be40c0@10.31.142.46@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:10.0:1616513082.530208:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.46@tcp1 00000400:00000200:10.0:1616513082.530215:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.46@tcp1 routed destination 00000100:00000200:16.0:1616513082.530217:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688800702607168, offset 192 00000400:00000200:16.0:1616513082.530223:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.46@tcp1 00000400:00000200:10.0:1616513082.530229:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.46@tcp1(10.31.142.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513082.530235:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513082.530237:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.46@tcp1 routed destination 00000800:00000200:10.0:1616513082.530241:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513082.530244:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513082.530246:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513082.530250:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513082.530251:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.46@tcp1(10.31.142.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513082.530256:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.530258:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.530263:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513082.530266:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.530266:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.530268:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513082.530270:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.530272:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513082.530273:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513082.530278:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:16.0:1616513082.530279:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513082.530281:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82660 00000800:00000200:28.0:1616513082.530284:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513082.530285:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513082.530287:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513082.530341:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.530348:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.530351:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.530358:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.530363:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a436d8 00000800:00000200:14.0:1616513082.530367:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.530370:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.531489:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.531543:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.531552:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.531561:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.208@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.531571:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.208@tcp1 of length 224 into portal 26 MB=0x5ffd5198e9d00 00000400:00000200:20.0:1616513082.531580:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.208@tcp1 of length 224/224 into md 0xf97c9ec9 [1] + 0 00000400:00000200:20.0:1616513082.531584:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a908 00000400:00000200:20.0:1616513082.531587:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.531590:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.531594:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a908 00000800:00000200:28.0:1616513082.531596:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.531597:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.531600:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.531602:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.531657:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665605446912 00010000:00000200:16.0:1616513082.531668:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688665605446912/t0(0) o400->99ed892a-7669-7476-c4e8-90035d6a4423@10.31.136.208@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.531682:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665605446912, offset 192 00000400:00000200:16.0:1616513082.531688:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.208@tcp1 00000400:00000200:16.0:1616513082.531696:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.208@tcp1 routed destination 00000400:00000200:16.0:1616513082.531711:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.208@tcp1(10.31.136.208@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.531718:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.531723:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.531728:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.531730:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.531735:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.531740:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.531741:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513082.531750:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.531755:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.531758:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.531764:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.531770:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43d38 00000800:00000200:28.0:1616513082.531775:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.531778:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.531885:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.531895:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.531901:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.531904:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513082.531909:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.208@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.531918:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.208@tcp1 of length 224 into portal 12 MB=0x5ffd5198e9ec0 00000400:00000200:28.0:1616513082.531924:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.208@tcp1 of length 224/224 into md 0x865bbec1 [1] + 52976 00000400:00000200:28.0:1616513082.531929:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.531932:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.531938:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.531941:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.531943:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.532002:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665605447360 00010000:00000200:0.0:1616513082.532014:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedf500 x1688665605447360/t0(0) o400->4060733b-fb97-c7fc-2570-78229f9dbe8e@10.31.136.208@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.532027:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665605447360, offset 192 00000400:00000200:0.0:1616513082.532033:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.208@tcp1 00000400:00000200:0.0:1616513082.532040:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.208@tcp1 routed destination 00000400:00000200:0.0:1616513082.532054:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.208@tcp1(10.31.136.208@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.532060:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.532065:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.532068:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513082.532071:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.532076:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.532081:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.532138:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.532145:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.532147:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.532154:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.532158:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05870 00000800:00000200:2.0:1616513082.532162:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.532165:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.544967:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.545023:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.545033:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.545042:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.59@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.545052:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.59@tcp1 of length 224 into portal 26 MB=0x5ffd405c20000 00000400:00000200:14.0:1616513082.545058:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.59@tcp1 of length 224/224 into md 0xf97c9ed9 [1] + 0 00000400:00000200:14.0:1616513082.545063:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79ae58 00000400:00000200:14.0:1616513082.545066:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.545071:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513082.545075:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79ae58 00000800:00000200:22.0:1616513082.545077:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.545078:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.545081:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.545082:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.545140:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660978302976 00010000:00000200:16.0:1616513082.545151:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688660978302976/t0(0) o400->f4e95c6e-e19f-98a0-715c-66ae5b257fe6@10.31.128.59@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.545165:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660978302976, offset 192 00000400:00000200:16.0:1616513082.545172:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.59@tcp1 00000400:00000200:16.0:1616513082.545180:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.59@tcp1 routed destination 00000400:00000200:16.0:1616513082.545195:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.59@tcp1(10.31.128.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.545201:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.545207:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.545210:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.545212:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.545217:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.545223:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.545263:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513082.545293:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513082.545320:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.545337:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.545340:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.545347:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.545352:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43430 00000800:00000200:28.0:1616513082.545355:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.545358:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513082.545359:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.545369:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.545377:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.59@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.545387:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.59@tcp1 of length 224 into portal 12 MB=0x5ffd405c20340 00000400:00000200:2.0:1616513082.545393:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.59@tcp1 of length 224/224 into md 0xf964d519 [1] + 23064 00000400:00000200:2.0:1616513082.545398:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.545401:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513082.545409:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.545412:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.545414:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.545417:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513082.545427:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660978303808 00010000:00000200:14.0:1616513082.545448:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d3f00 x1688660978303808/t0(0) o400->7fd8d61b-8b05-2ccf-939d-84f3546c0f4a@10.31.128.59@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513082.545460:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660978303808, offset 192 00000400:00000200:14.0:1616513082.545466:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.59@tcp1 00000400:00000200:14.0:1616513082.545472:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.59@tcp1 routed destination 00000400:00000200:14.0:1616513082.545485:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.59@tcp1(10.31.128.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513082.545491:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513082.545496:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513082.545498:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.545500:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.545503:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.545519:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.545530:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.545535:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.545538:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513082.545539:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.545544:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.545548:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757d38 00000800:00000200:22.0:1616513082.545552:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.548067:0:49718:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.548122:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.548131:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.548140:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.181@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.548149:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.181@tcp1 of length 224 into portal 12 MB=0x602a6ef7d8bc0 00000400:00000200:2.0:1616513082.548157:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.181@tcp1 of length 224/224 into md 0xf964d519 [1] + 23288 00000400:00000200:2.0:1616513082.548163:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.548165:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513082.548171:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.548174:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.548176:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.548180:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.548183:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:14.0:1616513082.548235:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691765866073024 00000800:00000200:20.0:1616513082.548242:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:14.0:1616513082.548247:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d0000 x1691765866073024/t0(0) o400->0f2a0b07-8180-08ce-cc55-e4a82c39dc2c@10.31.128.181@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513082.548251:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.548260:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.181@tcp1 : PUT - for me 00000100:00000200:14.0:1616513082.548261:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691765866073024, offset 192 00000400:00000200:14.0:1616513082.548267:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.181@tcp1 00000400:00000200:20.0:1616513082.548268:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.181@tcp1 of length 224 into portal 26 MB=0x602a6ef7d8b80 00000400:00000200:14.0:1616513082.548274:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.181@tcp1 routed destination 00000400:00000200:20.0:1616513082.548275:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.181@tcp1 of length 224/224 into md 0xf97c9ee9 [1] + 0 00000400:00000200:20.0:1616513082.548280:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79bba0 00000400:00000200:20.0:1616513082.548283:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.548286:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513082.548289:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.181@tcp1(10.31.128.181@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513082.548290:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bba0 00000800:00000200:20.0:1616513082.548294:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513082.548296:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513082.548297:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.548299:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513082.548301:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513082.548302:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513082.548304:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513082.548306:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513082.548311:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.548316:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.548328:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.548335:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.548337:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.548344:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.548348:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756198 00000800:00000200:28.0:1616513082.548352:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.548354:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691765866072960 00000800:00000200:28.0:1616513082.548355:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:16.0:1616513082.548365:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1691765866072960/t0(0) o400->8206db34-422d-3cfb-5eee-be1833a9516d@10.31.128.181@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.548379:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691765866072960, offset 192 00000400:00000200:16.0:1616513082.548386:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.181@tcp1 00000400:00000200:16.0:1616513082.548393:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.181@tcp1 routed destination 00000400:00000200:16.0:1616513082.548408:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.181@tcp1(10.31.128.181@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.548415:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.548421:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.548424:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.548426:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.548430:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.548440:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.548451:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:16.0:1616513082.548453:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513082.548456:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.548459:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.548464:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.548469:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43540 00000800:00000200:14.0:1616513082.548471:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.548474:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.566156:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513082.566200:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513082.566214:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.566224:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.566232:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.236@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.566242:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.236@tcp1 of length 224 into portal 26 MB=0x5ffd4be2a3e00 00000400:00000200:28.0:1616513082.566249:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.236@tcp1 of length 224/224 into md 0xf97c9ef9 [1] + 0 00000400:00000200:28.0:1616513082.566253:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b100 00000400:00000200:28.0:1616513082.566256:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.566264:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513082.566266:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513082.566269:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b100 00000800:00000200:28.0:1616513082.566272:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.566274:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513082.566276:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513082.566276:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.566280:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513082.566285:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.236@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.566293:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.236@tcp1 of length 224 into portal 12 MB=0x5ffd4be2a3f00 00000400:00000200:26.0:1616513082.566300:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.236@tcp1 of length 224/224 into md 0xf964d519 [1] + 23512 00000400:00000200:26.0:1616513082.566305:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.566308:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.566315:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.566317:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.566319:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.566322:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.566334:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664072142336 00010000:00000200:16.0:1616513082.566345:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688664072142336/t0(0) o400->2f329309-0366-6b68-8e15-8a1d459da28c@10.31.131.236@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.566359:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664072142336, offset 192 00000400:00000200:16.0:1616513082.566365:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.236@tcp1 00000400:00000200:16.0:1616513082.566373:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.236@tcp1 routed destination 00000100:00000200:14.0:1616513082.566383:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664072142592 00000400:00000200:16.0:1616513082.566388:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.236@tcp1(10.31.131.236@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.566395:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:14.0:1616513082.566395:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57e780 x1688664072142592/t0(0) o400->6fa742dc-f3b7-a6e1-8a1e-4b4b3ba6d479@10.31.131.236@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.566401:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.566403:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.566405:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:14.0:1616513082.566409:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664072142592, offset 192 00000800:00000200:16.0:1616513082.566410:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513082.566415:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.236@tcp1 00000800:00000200:11.2:1616513082.566415:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.566417:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513082.566423:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.236@tcp1 routed destination 00000800:00000200:28.0:1616513082.566426:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.566430:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.566432:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.566451:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.236@tcp1(10.31.131.236@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513082.566455:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513082.566458:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513082.566460:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43cb0 00000800:00000200:28.0:1616513082.566462:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513082.566464:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513082.566465:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513082.566466:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.566468:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.566473:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.566478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.566490:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.566494:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.566496:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.566501:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.566504:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756220 00000800:00000200:14.0:1616513082.566507:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.566509:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513082.586582:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926144, portal 4 00000100:00000200:10.0:1616513082.586588:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926144, offset 0 00000400:00000200:10.0:1616513082.586595:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:10.0:1616513082.586603:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000100:00000200:31.0:1616513082.586606:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926128, portal 4 00000100:00000200:31.0:1616513082.586612:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926128, offset 0 00000400:00000200:10.0:1616513082.586616:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000400:00000200:31.0:1616513082.586620:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:10.0:1616513082.586623:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:31.0:1616513082.586627:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:10.0:1616513082.586630:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513082.586633:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:10.0:1616513082.586635:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:31.0:1616513082.586641:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513082.586641:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:31.0:1616513082.586648:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:31.0:1616513082.586656:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513082.586658:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:31.0:1616513082.586661:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513082.586668:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513082.586717:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:1.2:1616513082.586717:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513082.586738:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:27.0:1616513082.586739:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513082.586744:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:1.0:1616513082.586744:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:27.0:1616513082.586747:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513082.586748:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513082.586748:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513082.586749:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513082.586755:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:1.0:1616513082.586755:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:27.0:1616513082.586761:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456eec00 x1693067725926128/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513089 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513082.586763:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dec201b00 x1693067725926144/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513089 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513082.586774:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c3c28 00000400:00000200:1.0:1616513082.586775:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83dc0 00000800:00000200:27.0:1616513082.586779:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:1.0:1616513082.586781:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:27.2:1616513082.586853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513082.586864:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513082.586871:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513082.586872:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:1.2:1616513082.586874:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:27.0:1616513082.586879:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:1.0:1616513082.586885:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:27.0:1616513082.586888:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626ef0 00000800:00000200:1.0:1616513082.586891:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513082.586892:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:27.0:1616513082.586894:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166c0d [1] + 192 00000400:00000200:27.0:1616513082.586900:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:1.0:1616513082.586900:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000100:00000200:27.0:1616513082.586904:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456eec00 x1693067725926128/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513089 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513082.586909:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626f00 00000400:00000200:1.0:1616513082.586917:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffc95 [1] + 192 00000800:00000200:27.0:1616513082.586918:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513082.586920:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513082.586922:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:1.0:1616513082.586923:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:31.0:1616513082.586924:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456eec00 x1693067725926128/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513089 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513082.586928:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dec201b00 x1693067725926144/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513089 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513082.586937:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456eec00 x1693067725926128/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513089 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513082.586942:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513082.586945:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:31.0:1616513082.586947:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000800:00000200:1.0:1616513082.586947:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:10.0:1616513082.586950:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dec201b00 x1693067725926144/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513089 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513082.586963:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dec201b00 x1693067725926144/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513089 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513082.586974:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83540 00000800:00000200:7.2:1616513082.589941:0:49722:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513082.589998:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.590007:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.590017:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.153@tcp1 : PUT - for me 00000800:00000200:11.2:1616513082.590025:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513082.590027:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.153@tcp1 of length 224 into portal 26 MB=0x5ffe985e52340 00000400:00000200:26.0:1616513082.590034:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.153@tcp1 of length 224/224 into md 0xf97c9f09 [1] + 0 00000400:00000200:26.0:1616513082.590038:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79add0 00000400:00000200:26.0:1616513082.590041:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.590044:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513082.590048:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79add0 00000800:00000200:26.0:1616513082.590052:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.590052:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.590054:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.590056:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513082.590084:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.590094:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.590103:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.153@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.590112:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.153@tcp1 of length 224 into portal 12 MB=0x5ffe985e526c0 00000400:00000200:20.0:1616513082.590119:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.153@tcp1 of length 224/224 into md 0x865bbec1 [1] + 53200 00000100:00000200:16.0:1616513082.590119:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753322402624 00000400:00000200:20.0:1616513082.590124:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.590127:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:16.0:1616513082.590130:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688753322402624/t0(0) o400->5a9ce89d-facb-a5fe-30cd-e04466f6b5cc@10.31.141.153@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513082.590134:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.590137:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.590139:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.590143:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.590145:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688753322402624, offset 192 00000400:00000200:16.0:1616513082.590151:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.153@tcp1 00000400:00000200:16.0:1616513082.590159:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.153@tcp1 routed destination 00000400:00000200:16.0:1616513082.590174:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.153@tcp1(10.31.141.153@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.590181:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.590187:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.590190:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.590192:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.590197:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.590202:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.590203:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513082.590206:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753322403520 00000800:00000200:28.0:1616513082.590213:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.590219:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:0.0:1616513082.590219:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce9200 x1688753322403520/t0(0) o400->5653a6cf-5467-11a2-fc14-811368834379@10.31.141.153@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513082.590221:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.590228:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.590234:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42198 00000800:00000200:28.0:1616513082.590237:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513082.590237:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688753322403520, offset 192 00000800:00000200:28.0:1616513082.590240:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513082.590243:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.153@tcp1 00000400:00000200:0.0:1616513082.590250:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.153@tcp1 routed destination 00000400:00000200:0.0:1616513082.590265:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.153@tcp1(10.31.141.153@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.590272:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.590278:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.590281:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513082.590283:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.590288:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.590293:0:49722:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.590348:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.590355:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.590358:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.590365:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.590369:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05210 00000800:00000200:14.0:1616513082.590373:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.590376:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.598759:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.598814:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.598823:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.598832:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.137@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.598841:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.137@tcp1 of length 224 into portal 26 MB=0x5ffd7b439e500 00000400:00000200:2.0:1616513082.598848:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.137@tcp1 of length 224/224 into md 0xf97c9f19 [1] + 0 00000400:00000200:2.0:1616513082.598852:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a990 00000400:00000200:2.0:1616513082.598855:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.598858:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.598862:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a990 00000800:00000200:2.0:1616513082.598865:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.598867:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.598868:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.598874:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.598877:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.598878:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.598886:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.137@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.598895:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.137@tcp1 of length 224 into portal 12 MB=0x5ffd7b439e580 00000400:00000200:26.0:1616513082.598903:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.137@tcp1 of length 224/224 into md 0xf964d519 [1] + 23736 00000400:00000200:26.0:1616513082.598908:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.598911:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.598918:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.598921:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.598923:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.598926:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676790297856 00010000:00000200:16.0:1616513082.598937:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688676790297856/t0(0) o400->b436fbb0-b4ab-4aba-918e-fb59106bd50f@10.31.143.137@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:14.0:1616513082.598940:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676790297984 00000100:00000200:16.0:1616513082.598951:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688676790297856, offset 192 00010000:00000200:14.0:1616513082.598953:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc578900 x1688676790297984/t0(0) o400->3fa2a977-98fc-ca9d-dd0e-371077c778c1@10.31.143.137@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513082.598959:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.137@tcp1 00000100:00000200:14.0:1616513082.598966:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688676790297984, offset 192 00000400:00000200:16.0:1616513082.598967:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.137@tcp1 routed destination 00000400:00000200:14.0:1616513082.598972:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.137@tcp1 00000400:00000200:14.0:1616513082.598979:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.137@tcp1 routed destination 00000400:00000200:16.0:1616513082.598982:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.137@tcp1(10.31.143.137@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.598989:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513082.598993:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.137@tcp1(10.31.143.137@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.598995:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.598997:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.598999:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513082.598999:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.599004:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513082.599004:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513082.599007:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.599009:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.599011:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513082.599013:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.599018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513082.599051:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513082.599073:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.599080:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.599083:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513082.599084:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.599090:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.599094:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756220 00000800:00000200:22.0:1616513082.599098:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513082.599108:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.599115:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.599117:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.599124:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.599129:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a434b8 00000800:00000200:28.0:1616513082.599133:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.599135:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.622587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.622644:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.622653:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.622662:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.209@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.622671:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.209@tcp1 of length 224 into portal 26 MB=0x60183bb517780 00000400:00000200:28.0:1616513082.622678:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.209@tcp1 of length 224/224 into md 0xf97c9f29 [1] + 0 00000400:00000200:28.0:1616513082.622683:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b7e8 00000400:00000200:28.0:1616513082.622685:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.622690:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.622695:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b7e8 00000800:00000200:24.0:1616513082.622697:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.622698:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.622701:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.622703:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.622758:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515155285888 00010000:00000200:16.0:1616513082.622769:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1690515155285888/t0(0) o400->25022573-a214-6272-cc8f-2b013fefb4cb@10.31.132.209@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.622783:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690515155285888, offset 192 00000400:00000200:16.0:1616513082.622790:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.209@tcp1 00000400:00000200:16.0:1616513082.622797:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.209@tcp1 routed destination 00000400:00000200:16.0:1616513082.622813:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.209@tcp1(10.31.132.209@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.622819:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.622825:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.622828:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.622830:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.622835:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.622841:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.622852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.622861:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.622865:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.622868:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.622873:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.622878:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42cc0 00000800:00000200:28.0:1616513082.622881:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.622884:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.623293:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.623304:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.623312:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513082.623313:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.623321:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.209@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.623329:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.209@tcp1 of length 224 into portal 12 MB=0x60183bb517940 00000400:00000200:24.0:1616513082.623336:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.209@tcp1 of length 224/224 into md 0x865bbec1 [1] + 53424 00000400:00000200:24.0:1616513082.623341:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.623344:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.623351:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.623354:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.623356:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.623417:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515155286336 00010000:00000200:0.0:1616513082.623430:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd703a80 x1690515155286336/t0(0) o400->98d5285c-fd2c-e60e-c71e-43a5be9f965e@10.31.132.209@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.623464:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690515155286336, offset 192 00000400:00000200:0.0:1616513082.623470:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.209@tcp1 00000400:00000200:0.0:1616513082.623478:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.209@tcp1 routed destination 00000400:00000200:0.0:1616513082.623493:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.209@tcp1(10.31.132.209@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.623500:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.623506:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.623509:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513082.623511:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.623515:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.623523:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.623536:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.623542:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.623545:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.623552:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.623555:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:14.0:1616513082.623559:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.623562:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.650978:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.651035:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.651044:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.651053:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.162@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.651063:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.162@tcp1 of length 224 into portal 26 MB=0x5ffd4dbf8ed80 00000400:00000200:28.0:1616513082.651069:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.162@tcp1 of length 224/224 into md 0xf97c9f49 [1] + 0 00000400:00000200:28.0:1616513082.651075:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79aaa0 00000400:00000200:28.0:1616513082.651077:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.651080:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.651085:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aaa0 00000800:00000200:28.0:1616513082.651088:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.651090:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.651091:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.651094:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.651149:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664572226944 00010000:00000200:16.0:1616513082.651160:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688664572226944/t0(0) o400->37ec9de5-a00c-cf4d-ef46-35a75151e500@10.31.132.162@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.651175:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664572226944, offset 192 00000400:00000200:16.0:1616513082.651181:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.162@tcp1 00000400:00000200:16.0:1616513082.651189:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.162@tcp1 routed destination 00000400:00000200:16.0:1616513082.651204:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.162@tcp1(10.31.132.162@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.651215:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.651221:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.651224:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.651226:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.651231:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.651236:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.651237:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513082.651246:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.651250:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.651252:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.651258:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.651263:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42198 00000800:00000200:28.0:1616513082.651265:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.651268:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.651298:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.651310:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.651318:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513082.651319:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.651327:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.162@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.651336:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.162@tcp1 of length 224 into portal 12 MB=0x5ffd4dbf8eec0 00000400:00000200:20.0:1616513082.651342:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.162@tcp1 of length 224/224 into md 0x865bbec1 [1] + 53648 00000400:00000200:20.0:1616513082.651347:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.651349:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513082.651356:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.651359:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.651361:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513082.651421:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664572227264 00010000:00000200:0.0:1616513082.651446:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cede80 x1688664572227264/t0(0) o400->00d43f34-943b-6d96-d90c-a86697569fa3@10.31.132.162@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513082.651460:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664572227264, offset 192 00000400:00000200:0.0:1616513082.651468:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.162@tcp1 00000400:00000200:0.0:1616513082.651476:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.162@tcp1 routed destination 00000400:00000200:0.0:1616513082.651492:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.162@tcp1(10.31.132.162@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.651498:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.651504:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513082.651507:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513082.651509:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513082.651514:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.651521:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.651577:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.651585:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.651588:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.651594:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.651598:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04dd0 00000800:00000200:2.0:1616513082.651602:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.651605:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.660975:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.661032:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.661043:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.661052:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.159@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.661061:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.159@tcp1 of length 224 into portal 26 MB=0x604b7f723cb40 00000400:00000200:28.0:1616513082.661068:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.159@tcp1 of length 224/224 into md 0xf97c9f59 [1] + 0 00000400:00000200:28.0:1616513082.661072:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79ba08 00000400:00000200:28.0:1616513082.661075:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.661078:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.661083:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79ba08 00000800:00000200:28.0:1616513082.661086:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.661087:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.661089:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.661095:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.661098:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.661099:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.661108:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.159@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.661116:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.159@tcp1 of length 224 into portal 12 MB=0x604b7f723cbc0 00000400:00000200:24.0:1616513082.661122:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.159@tcp1 of length 224/224 into md 0x865bbec1 [1] + 53872 00000400:00000200:24.0:1616513082.661128:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.661130:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.661137:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.661140:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.661142:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.661146:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694038032108352 00010000:00000200:16.0:1616513082.661157:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1694038032108352/t0(0) o400->7adf62df-e92e-cebe-bc03-b3a697db6ca7@10.31.140.159@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.661171:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694038032108352, offset 192 00000400:00000200:16.0:1616513082.661178:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.159@tcp1 00000400:00000200:16.0:1616513082.661185:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.159@tcp1 routed destination 00000400:00000200:16.0:1616513082.661200:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.159@tcp1(10.31.140.159@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513082.661201:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694038032108480 00000800:00000200:16.0:1616513082.661206:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.661212:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:4.0:1616513082.661213:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd702880 x1694038032108480/t0(0) o400->0c096a6e-1ecb-652a-3efd-e64ebb49cea0@10.31.140.159@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.661215:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.661217:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.661221:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.661228:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:4.0:1616513082.661228:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694038032108480, offset 192 00000400:00000200:4.0:1616513082.661234:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.159@tcp1 00000800:00000200:11.2:1616513082.661235:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:4.0:1616513082.661242:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.159@tcp1 routed destination 00000400:00000200:4.0:1616513082.661257:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.159@tcp1(10.31.140.159@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.661263:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513082.661269:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.661272:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.661274:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.661279:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.661285:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513082.661289:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.661296:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.661299:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.661300:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.661305:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.661311:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43540 00000800:00000200:20.0:1616513082.661314:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513082.661340:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.661347:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.661350:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.661356:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.661360:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bba90 00000800:00000200:14.0:1616513082.661364:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.661367:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.661373:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513082.661376:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.661378:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.661379:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.661381:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.665653:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513082.665659:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513082.665709:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.665718:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513082.665727:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513082.665727:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.96@tcp1 : PUT - for me 00000800:00000200:24.0:1616513082.665736:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513082.665736:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.96@tcp1 of length 224 into portal 26 MB=0x60183477e7380 00000400:00000200:22.0:1616513082.665742:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.96@tcp1 of length 224/224 into md 0xf97c9f79 [1] + 0 00000400:00000200:24.0:1616513082.665746:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.96@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.665747:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79bc28 00000400:00000200:22.0:1616513082.665750:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.665752:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.665754:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.96@tcp1 of length 224 into portal 12 MB=0x60183477e75c0 00000400:00000200:22.0:1616513082.665757:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bc28 00000800:00000200:22.0:1616513082.665760:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.665760:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513082.665761:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.96@tcp1 of length 224/224 into md 0x865bbec1 [1] + 54096 00000800:00000200:22.0:1616513082.665762:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.665764:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513082.665768:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.665770:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.665776:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.665780:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513082.665781:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.665785:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.665824:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513212076928 00010000:00000200:16.0:1616513082.665836:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1690513212076928/t0(0) o400->56455637-7664-eb05-435e-81aa5b4a38c7@10.31.141.96@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513082.665849:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513212077504 00000100:00000200:16.0:1616513082.665850:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513212076928, offset 192 00000400:00000200:16.0:1616513082.665856:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.96@tcp1 00010000:00000200:4.0:1616513082.665862:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd703600 x1690513212077504/t0(0) o400->f5c93678-4fda-194a-1510-7c5aa87a5fbc@10.31.141.96@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513082.665863:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.96@tcp1 routed destination 00000100:00000200:4.0:1616513082.665876:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513212077504, offset 192 00000400:00000200:16.0:1616513082.665878:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.96@tcp1(10.31.141.96@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:4.0:1616513082.665883:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.96@tcp1 00000800:00000200:16.0:1616513082.665885:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:4.0:1616513082.665890:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.96@tcp1 routed destination 00000800:00000200:16.0:1616513082.665897:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.665900:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:4.0:1616513082.665905:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.96@tcp1(10.31.141.96@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.665909:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513082.665912:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.665914:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513082.665918:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:11.2:1616513082.665919:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.665920:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513082.665923:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.665925:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.665931:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.665935:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513082.665974:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.665980:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.665982:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.665988:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.665993:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43d38 00000800:00000200:26.0:1616513082.665993:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513082.665996:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.665999:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513082.666000:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.666003:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.666009:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.666013:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bacc0 00000800:00000200:26.0:1616513082.666017:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.666020:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.678518:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.678575:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.678583:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513082.678592:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.69@tcp1 : PUT - for me 00000400:00000200:0.0:1616513082.678601:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.69@tcp1 of length 224 into portal 26 MB=0x5ffd422fca740 00000400:00000200:0.0:1616513082.678607:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.69@tcp1 of length 224/224 into md 0xf97c9f89 [1] + 0 00000400:00000200:0.0:1616513082.678611:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79bdc0 00000400:00000200:0.0:1616513082.678614:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513082.678617:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513082.678621:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bdc0 00000800:00000200:0.0:1616513082.678624:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.678626:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.678627:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513082.678631:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.678690:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661468686144 00010000:00000200:16.0:1616513082.678700:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688661468686144/t0(0) o400->94b7f1f6-65ef-b6f5-af44-6a1a85de943b@10.31.133.69@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.678713:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661468686144, offset 192 00000400:00000200:16.0:1616513082.678719:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.69@tcp1 00000400:00000200:16.0:1616513082.678726:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.69@tcp1 routed destination 00000400:00000200:16.0:1616513082.678740:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.69@tcp1(10.31.133.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.678746:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.678751:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.678754:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.678756:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.678760:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.678766:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.678766:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.678819:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.678825:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.678828:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.678834:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.678839:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43e48 00000800:00000200:28.0:1616513082.678842:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.678845:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.679140:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.679151:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.679158:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.679170:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.679177:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.69@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.679186:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.69@tcp1 of length 224 into portal 12 MB=0x5ffd422fca840 00000400:00000200:24.0:1616513082.679192:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.69@tcp1 of length 224/224 into md 0x865bbec1 [1] + 54320 00000400:00000200:24.0:1616513082.679197:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.679200:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.679206:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.679209:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.679211:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513082.679268:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661468686400 00010000:00000200:4.0:1616513082.679279:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd707980 x1688661468686400/t0(0) o400->fa1cbf35-dece-4bce-fd5c-6ce071616bb8@10.31.133.69@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.679293:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661468686400, offset 192 00000400:00000200:4.0:1616513082.679299:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.69@tcp1 00000400:00000200:4.0:1616513082.679306:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.69@tcp1 routed destination 00000400:00000200:4.0:1616513082.679320:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.69@tcp1(10.31.133.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.679326:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513082.679331:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.679333:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.679335:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.679340:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.679348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.679404:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.679411:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.679416:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.679422:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.679426:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb298 00000800:00000200:14.0:1616513082.679430:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.679433:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.681536:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513082.681590:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.681599:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.681608:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.118@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.681616:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.118@tcp1 of length 224 into portal 26 MB=0x60182f8b73fc0 00000400:00000200:26.0:1616513082.681623:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.118@tcp1 of length 224/224 into md 0xf97c9fa9 [1] + 0 00000400:00000200:26.0:1616513082.681628:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b5c8 00000400:00000200:26.0:1616513082.681631:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.681633:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513082.681637:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b5c8 00000800:00000200:26.0:1616513082.681640:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.681642:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.681643:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.681649:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.681652:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.681653:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.681661:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.118@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.681670:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.118@tcp1 of length 224 into portal 12 MB=0x60182f8b74040 00000400:00000200:2.0:1616513082.681677:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.118@tcp1 of length 224/224 into md 0xf964d519 [1] + 23960 00000400:00000200:2.0:1616513082.681684:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.681686:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513082.681695:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.681698:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.681700:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.681701:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511890399168 00010000:00000200:16.0:1616513082.681713:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1690511890399168/t0(0) o400->8bf8094b-abf2-bf18-5a0c-cd8ef37287eb@10.31.140.118@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.681727:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511890399168, offset 192 00000400:00000200:16.0:1616513082.681733:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.118@tcp1 00000400:00000200:16.0:1616513082.681740:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.118@tcp1 routed destination 00000400:00000200:16.0:1616513082.681755:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.118@tcp1(10.31.140.118@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513082.681761:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511890399296 00000800:00000200:16.0:1616513082.681762:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.681767:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.681770:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.681772:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:18.0:1616513082.681773:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d4c80 x1690511890399296/t0(0) o400->6e1088bb-aca5-5f03-c839-321efa46fa7c@10.31.140.118@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.681777:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.681784:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513082.681787:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511890399296, offset 192 00000400:00000200:18.0:1616513082.681793:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.118@tcp1 00000400:00000200:18.0:1616513082.681800:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.118@tcp1 routed destination 00000400:00000200:18.0:1616513082.681815:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.118@tcp1(10.31.140.118@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.681822:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513082.681825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513082.681827:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.681830:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513082.681834:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513082.681839:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.681844:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.681856:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.681863:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.681866:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513082.681867:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.681875:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.681879:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd7e8 00000800:00000200:28.0:1616513082.681881:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513082.681883:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513082.681888:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.681891:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.681897:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.681903:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:28.0:1616513082.681906:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.681909:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.687712:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.687767:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.687776:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.687785:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.73@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.687794:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.73@tcp1 of length 224 into portal 26 MB=0x6037bd645cac0 00000400:00000200:2.0:1616513082.687801:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.73@tcp1 of length 224/224 into md 0xf97c9fb9 [1] + 0 00000400:00000200:2.0:1616513082.687806:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79aee0 00000400:00000200:2.0:1616513082.687808:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.687811:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.687815:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aee0 00000800:00000200:2.0:1616513082.687818:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.687820:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.687821:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.687824:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.687878:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692680271022784 00010000:00000200:16.0:1616513082.687889:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1692680271022784/t0(0) o400->08aa445d-58b3-fcb6-8847-78f9e2f8ace6@10.31.136.73@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.687905:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692680271022784, offset 192 00000400:00000200:16.0:1616513082.687911:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.73@tcp1 00000400:00000200:16.0:1616513082.687919:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.73@tcp1 routed destination 00000400:00000200:16.0:1616513082.687934:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.73@tcp1(10.31.136.73@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.687941:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.687946:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.687949:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.687951:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.687956:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.687962:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.688003:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.688062:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.688068:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513082.688071:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513082.688077:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513082.688081:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43078 00000800:00000200:0.0:1616513082.688085:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513082.688087:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.688561:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513082.688588:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513082.688616:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.688626:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.688635:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.224@tcp1 : PUT - for me 00000800:00000200:26.0:1616513082.688643:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513082.688645:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.224@tcp1 of length 224 into portal 26 MB=0x5ffdd15f07600 00000400:00000200:28.0:1616513082.688652:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.224@tcp1 of length 224/224 into md 0xf97c9fd9 [1] + 0 00000800:00000200:26.0:1616513082.688652:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513082.688657:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79a6e8 00000400:00000200:28.0:1616513082.688659:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.688661:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.224@tcp1 : PUT - for me 00000100:00000200:28.0:1616513082.688662:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.688667:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a6e8 00000800:00000200:28.0:1616513082.688670:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513082.688670:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.224@tcp1 of length 224 into portal 12 MB=0x5ffdd15f07680 00000800:00000200:20.0:1616513082.688672:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.688673:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513082.688676:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.224@tcp1 of length 224/224 into md 0xf964d519 [1] + 24184 00000800:00000200:28.0:1616513082.688677:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.688679:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699904488960 00010000:00000200:16.0:1616513082.688685:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688699904488960/t0(0) o400->fb021f05-e26a-cd7c-cd6d-5607e916844c@10.31.141.224@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513082.688686:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.688689:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.688695:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:16.0:1616513082.688697:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699904488960, offset 192 00000800:00000200:26.0:1616513082.688699:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.688701:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513082.688702:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.224@tcp1 00000800:00000200:26.0:1616513082.688703:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513082.688708:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.224@tcp1 routed destination 00000400:00000200:16.0:1616513082.688721:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.224@tcp1(10.31.141.224@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.688727:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.688732:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.688734:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.688736:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.688740:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513082.688744:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.688745:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.688755:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.688760:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:18.0:1616513082.688762:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699904489088 00000400:00000200:14.0:1616513082.688763:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513082.688765:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513082.688769:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:18.0:1616513082.688774:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57b180 x1688699904489088/t0(0) o400->dd1a75be-b3ab-1277-8d38-fab1450b810f@10.31.141.224@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513082.688774:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43e48 00000800:00000200:14.0:1616513082.688778:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513082.688789:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699904489088, offset 192 00000400:00000200:18.0:1616513082.688795:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.224@tcp1 00000400:00000200:18.0:1616513082.688803:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.224@tcp1 routed destination 00000400:00000200:18.0:1616513082.688818:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.224@tcp1(10.31.141.224@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.688825:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513082.688831:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.688834:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513082.688836:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513082.688841:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.688846:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.688854:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.688858:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513082.688859:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513082.688864:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513082.688865:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513082.688868:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd870 00000800:00000200:0.0:1616513082.688870:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.690398:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.690454:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.690463:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.690472:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.73@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.690480:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.73@tcp1 of length 224 into portal 12 MB=0x6037bd645cc00 00000400:00000200:20.0:1616513082.690487:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.73@tcp1 of length 224/224 into md 0x865bbec1 [1] + 54544 00000400:00000200:20.0:1616513082.690492:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.690494:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513082.690501:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.690504:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.690506:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.690508:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513082.690563:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692680271023104 00010000:00000200:4.0:1616513082.690576:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cef500 x1692680271023104/t0(0) o400->f08ad355-f524-f229-fc0b-ed51ae1ad3d0@10.31.136.73@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.690590:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692680271023104, offset 192 00000400:00000200:4.0:1616513082.690597:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.73@tcp1 00000400:00000200:4.0:1616513082.690604:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.73@tcp1 routed destination 00000400:00000200:4.0:1616513082.690624:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.73@tcp1(10.31.136.73@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.690630:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513082.690636:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.690639:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.690641:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.690646:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.690650:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.690666:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.690672:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.690675:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.690682:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.690686:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbb18 00000800:00000200:26.0:1616513082.690689:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.690692:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.698543:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.698598:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.698607:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.698616:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.47@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.698625:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.47@tcp1 of length 224 into portal 26 MB=0x5ffdf9eb80600 00000400:00000200:2.0:1616513082.698631:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.47@tcp1 of length 224/224 into md 0xf97c9fe9 [1] + 0 00000400:00000200:2.0:1616513082.698636:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b760 00000400:00000200:2.0:1616513082.698639:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.698641:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.698645:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b760 00000800:00000200:2.0:1616513082.698648:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.698649:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.698651:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.698655:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.698658:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.698659:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.698667:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.47@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.698675:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.47@tcp1 of length 224 into portal 12 MB=0x5ffdf9eb80940 00000400:00000200:14.0:1616513082.698681:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.47@tcp1 of length 224/224 into md 0xf964d519 [1] + 24408 00000400:00000200:14.0:1616513082.698686:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.698688:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513082.698694:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.698697:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.698699:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.698708:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688710789203456 00010000:00000200:16.0:1616513082.698720:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688710789203456/t0(0) o400->47ad040f-aa40-2a8f-2c73-4ac06b6a70d3@10.31.142.47@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.698734:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688710789203456, offset 192 00000400:00000200:16.0:1616513082.698740:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.47@tcp1 00000400:00000200:16.0:1616513082.698747:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.47@tcp1 routed destination 00000100:00000200:18.0:1616513082.698757:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688710789204288 00000400:00000200:16.0:1616513082.698763:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.47@tcp1(10.31.142.47@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:18.0:1616513082.698768:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba3f00 x1688710789204288/t0(0) o400->7d242ec7-19e6-5e72-fa40-0ac6fd2508e9@10.31.142.47@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.698770:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.698776:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.698778:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.698780:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:18.0:1616513082.698782:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688710789204288, offset 192 00000800:00000200:16.0:1616513082.698785:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513082.698787:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.47@tcp1 00000100:00000200:16.0:1616513082.698792:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513082.698794:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.47@tcp1 routed destination 00000400:00000200:18.0:1616513082.698807:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.47@tcp1(10.31.142.47@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.698813:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513082.698819:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.698821:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513082.698823:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513082.698827:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513082.698830:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513082.698832:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.698843:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.698850:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.698852:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513082.698854:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.698859:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.698862:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd8f8 00000800:00000200:22.0:1616513082.698866:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513082.698884:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.698890:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.698893:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513082.698897:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.698899:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.698904:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43540 00000800:00000200:24.0:1616513082.698907:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.700219:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.700275:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.700284:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513082.700290:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513082.700292:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.158@tcp1 : PUT - for me 00000800:00000200:28.0:1616513082.700300:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.700300:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.158@tcp1 of length 224 into portal 26 MB=0x5ffdc1ad0b9c0 00000800:00000200:0.0:1616513082.700301:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.700308:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.158@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.700308:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.158@tcp1 of length 224/224 into md 0xf97ca009 [1] + 0 00000400:00000200:20.0:1616513082.700312:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841e79b8f8 00000400:00000200:20.0:1616513082.700315:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.700316:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.158@tcp1 of length 224 into portal 12 MB=0x5ffdc1ad0bbc0 00000100:00000200:20.0:1616513082.700317:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.700321:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b8f8 00000400:00000200:28.0:1616513082.700322:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.158@tcp1 of length 224/224 into md 0x865bbec1 [1] + 54768 00000800:00000200:20.0:1616513082.700324:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513082.700326:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513082.700327:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.700328:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513082.700329:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.700335:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.700338:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.700340:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.700383:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695691327936 00010000:00000200:16.0:1616513082.700393:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688695691327936/t0(0) o400->7d15bdde-286d-4f1e-69cb-2d446be3b759@10.31.141.158@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513082.700404:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695691328448 00000100:00000200:16.0:1616513082.700406:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695691327936, offset 192 00000400:00000200:16.0:1616513082.700412:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.158@tcp1 00010000:00000200:4.0:1616513082.700415:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedf080 x1688695691328448/t0(0) o400->708f550f-b28c-ee11-39d5-88cdea54f415@10.31.141.158@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513082.700419:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.158@tcp1 routed destination 00000400:00000200:16.0:1616513082.700432:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.158@tcp1(10.31.141.158@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.700450:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.700456:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:4.0:1616513082.700457:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695691328448, offset 192 00000800:00000200:16.0:1616513082.700458:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.700460:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:4.0:1616513082.700463:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.158@tcp1 00000800:00000200:16.0:1616513082.700466:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513082.700470:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.158@tcp1 routed destination 00000800:00000200:11.2:1616513082.700471:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.700473:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:4.0:1616513082.700484:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.158@tcp1(10.31.141.158@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.700490:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513082.700495:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.700498:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.700500:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.700504:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.700509:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513082.700536:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.700543:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.700546:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513082.700547:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.700552:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.700558:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42cc0 00000800:00000200:24.0:1616513082.700561:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513082.700561:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.700568:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.700570:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.700576:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.700580:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb7e8 00000800:00000200:14.0:1616513082.700584:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.700587:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.713004:0:49735:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513082.713019:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513082.713059:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.713083:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513082.713090:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513082.713091:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.207@tcp1 : PUT - for me 00000800:00000200:20.0:1616513082.713099:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513082.713100:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.207@tcp1 of length 224 into portal 12 MB=0x602be297c7340 00000400:00000200:2.0:1616513082.713106:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.207@tcp1 of length 224/224 into md 0xf964d519 [1] + 24632 00000400:00000200:20.0:1616513082.713108:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.207@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.713111:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.713114:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513082.713116:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.207@tcp1 of length 224 into portal 26 MB=0x602be297c72c0 00000800:00000200:2.0:1616513082.713120:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.713121:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513082.713123:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.207@tcp1 of length 224/224 into md 0xf97ca019 [1] + 0 00000800:00000200:2.0:1616513082.713123:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.713125:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513082.713127:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71100 00000400:00000200:20.0:1616513082.713130:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.713132:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.713136:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71100 00000800:00000200:20.0:1616513082.713139:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.713142:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.713143:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.713146:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513082.713184:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691865623327552 00010000:00000200:18.0:1616513082.713196:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d5e80 x1691865623327552/t0(0) o400->5c74afbf-7af5-2551-ffe8-22208a0975bf@10.31.132.207@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513082.713202:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691865623327424 00000100:00000200:18.0:1616513082.713209:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691865623327552, offset 192 00010000:00000200:16.0:1616513082.713212:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1691865623327424/t0(0) o400->83890a7c-622c-5f08-5770-27403d95c4bd@10.31.132.207@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513082.713219:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.207@tcp1 00000400:00000200:18.0:1616513082.713226:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.207@tcp1 routed destination 00000100:00000200:16.0:1616513082.713226:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691865623327424, offset 192 00000400:00000200:16.0:1616513082.713232:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.207@tcp1 00000400:00000200:18.0:1616513082.713240:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.207@tcp1(10.31.132.207@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.713246:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513082.713248:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.207@tcp1 routed destination 00000800:00000200:18.0:1616513082.713251:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.713254:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513082.713256:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513082.713260:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513082.713262:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.207@tcp1(10.31.132.207@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513082.713265:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.713268:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.713274:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.713278:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.713280:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.713284:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.713289:0:49735:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.713290:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513082.713318:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.713325:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513082.713328:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513082.713334:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513082.713338:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cda08 00000800:00000200:0.0:1616513082.713342:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513082.713344:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513082.713345:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513082.713350:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.713353:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.713359:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.713364:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43e48 00000800:00000200:14.0:1616513082.713369:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.713372:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.719926:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.719983:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.719992:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.720001:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.79@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.720009:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.79@tcp1 of length 224 into portal 26 MB=0x5ffd462cbd900 00000400:00000200:28.0:1616513082.720016:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.79@tcp1 of length 224/224 into md 0xf97ca039 [1] + 0 00000400:00000200:28.0:1616513082.720021:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f706e8 00000400:00000200:28.0:1616513082.720023:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.720026:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.720030:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f706e8 00000800:00000200:28.0:1616513082.720035:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.720037:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.720038:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.720043:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.720046:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.720047:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.720056:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.79@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.720065:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.79@tcp1 of length 224 into portal 12 MB=0x5ffd462cbdb40 00000400:00000200:20.0:1616513082.720071:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.79@tcp1 of length 224/224 into md 0x865bbec1 [1] + 54992 00000400:00000200:20.0:1616513082.720076:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.720078:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513082.720085:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.720088:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.720090:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.720093:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662539229440 00010000:00000200:16.0:1616513082.720104:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688662539229440/t0(0) o400->e036a5af-8a48-d782-784c-9f904572fa07@10.31.138.79@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.720119:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662539229440, offset 192 00000400:00000200:16.0:1616513082.720125:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.79@tcp1 00000400:00000200:16.0:1616513082.720132:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.79@tcp1 routed destination 00000400:00000200:16.0:1616513082.720160:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.79@tcp1(10.31.138.79@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513082.720163:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662539230016 00000800:00000200:16.0:1616513082.720165:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.720170:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.720172:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.720174:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:4.0:1616513082.720175:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cec800 x1688662539230016/t0(0) o400->151152d3-86fc-187d-c4c9-f8713d579f28@10.31.138.79@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.720178:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.720183:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.720184:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:4.0:1616513082.720189:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662539230016, offset 192 00000400:00000200:4.0:1616513082.720195:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.79@tcp1 00000400:00000200:4.0:1616513082.720202:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.79@tcp1 routed destination 00000400:00000200:4.0:1616513082.720216:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.79@tcp1(10.31.138.79@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.720222:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513082.720227:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.720230:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.720232:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.720236:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513082.720238:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513082.720242:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513082.720245:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.720248:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.720249:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.720254:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.720259:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43e48 00000800:00000200:24.0:1616513082.720266:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513082.720297:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.720304:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.720306:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.720313:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.720317:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba2a8 00000800:00000200:22.0:1616513082.720321:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.720324:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.725250:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.725305:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.725314:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.725323:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.14@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.725331:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.14@tcp1 of length 224 into portal 26 MB=0x5f9105c97bc40 00000400:00000200:14.0:1616513082.725338:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.14@tcp1 of length 224/224 into md 0xf97ca059 [1] + 0 00000400:00000200:14.0:1616513082.725343:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70550 00000400:00000200:14.0:1616513082.725345:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.725348:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513082.725352:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70550 00000800:00000200:14.0:1616513082.725355:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.725357:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.725358:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.725362:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.725413:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681223551794240 00010000:00000200:16.0:1616513082.725422:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1681223551794240/t0(0) o400->46a39b78-ce07-280c-1022-3d584c9de891@10.31.132.14@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.725445:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681223551794240, offset 192 00000400:00000200:16.0:1616513082.725450:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.14@tcp1 00000400:00000200:16.0:1616513082.725457:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.14@tcp1 routed destination 00000400:00000200:16.0:1616513082.725470:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.14@tcp1(10.31.132.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.725475:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.725480:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:11.2:1616513082.725482:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.725483:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.725485:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.725489:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:16.0:1616513082.725495:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513082.725553:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.725564:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.725573:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.14@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.725581:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.14@tcp1 of length 224 into portal 12 MB=0x5f9105c97bcc0 00000400:00000200:20.0:1616513082.725587:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.14@tcp1 of length 224/224 into md 0x865bbec1 [1] + 55216 00000400:00000200:20.0:1616513082.725593:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.725595:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513082.725602:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513082.725605:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513082.725606:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513082.725609:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.725617:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513082.725620:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513082.725627:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513082.725632:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43e48 00000800:00000200:0.0:1616513082.725636:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513082.725639:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513082.725668:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681223551794368 00010000:00000200:4.0:1616513082.725680:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceec00 x1681223551794368/t0(0) o400->4d2e5fb6-3c02-bc23-5614-019d1377a4c9@10.31.132.14@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.725694:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681223551794368, offset 192 00000400:00000200:4.0:1616513082.725700:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.14@tcp1 00000400:00000200:4.0:1616513082.725706:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.14@tcp1 routed destination 00000400:00000200:4.0:1616513082.725720:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.14@tcp1(10.31.132.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.725728:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513082.725733:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.725736:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.725737:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.725742:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.725748:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.725801:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.725808:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.725811:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.725817:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.725821:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00000800:00000200:22.0:1616513082.725825:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.725828:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.729164:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513082.729218:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513082.729220:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513082.729227:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.729236:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.46@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.729246:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.46@tcp1 of length 224 into portal 12 MB=0x6042a3bb14780 00000400:00000200:26.0:1616513082.729252:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.46@tcp1 of length 224/224 into md 0xf964d519 [1] + 24856 00000400:00000200:26.0:1616513082.729258:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.729260:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.729267:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.729270:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.729272:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.729275:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513082.729291:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.729300:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.729309:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.46@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.729318:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.46@tcp1 of length 224 into portal 26 MB=0x6042a3bb14640 00000400:00000200:24.0:1616513082.729324:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.46@tcp1 of length 224/224 into md 0xf97ca079 [1] + 0 00000400:00000200:24.0:1616513082.729329:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71f58 00000100:00000200:18.0:1616513082.729330:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693429296875392 00000400:00000200:24.0:1616513082.729332:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.729334:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.729338:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71f58 00000800:00000200:24.0:1616513082.729341:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:18.0:1616513082.729342:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57cc80 x1693429296875392/t0(0) o400->4591a5ae-5f8b-18db-a857-ba8bd5e690bc@10.31.138.46@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513082.729344:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.729346:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.729350:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513082.729355:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693429296875392, offset 192 00000400:00000200:18.0:1616513082.729361:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.46@tcp1 00000400:00000200:18.0:1616513082.729367:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.46@tcp1 routed destination 00000400:00000200:18.0:1616513082.729381:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.46@tcp1(10.31.138.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.729388:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513082.729393:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.729395:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513082.729397:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513082.729402:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.729406:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693429296875072 00000800:00000200:11.2:1616513082.729407:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:16.0:1616513082.729416:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1693429296875072/t0(0) o400->519ac1a5-5940-54d9-8c2f-51730344be8e@10.31.138.46@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.729429:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693429296875072, offset 192 00000400:00000200:16.0:1616513082.729446:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.46@tcp1 00000400:00000200:16.0:1616513082.729453:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.46@tcp1 routed destination 00000400:00000200:16.0:1616513082.729467:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.46@tcp1(10.31.138.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513082.729471:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.729473:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513082.729478:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513082.729479:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:0.0:1616513082.729480:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513082.729481:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.729483:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513082.729486:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:16.0:1616513082.729488:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513082.729490:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd6d8 00000800:00000200:7.2:1616513082.729493:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.729494:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513082.729494:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513082.729497:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513082.729505:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.729512:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.729515:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.729522:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.729527:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde5d8 00000800:00000200:22.0:1616513082.729531:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.729534:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.733136:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513082.733152:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513082.733189:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.733199:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.733208:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.97@tcp1 : PUT - for me 00000800:00000200:20.0:1616513082.733209:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513082.733217:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.97@tcp1 of length 224 into portal 12 MB=0x601955b5ecb80 00000800:00000200:20.0:1616513082.733218:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513082.733223:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.97@tcp1 of length 224/224 into md 0xf964d519 [1] + 25080 00000400:00000200:20.0:1616513082.733226:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.97@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.733229:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.733231:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513082.733235:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.97@tcp1 of length 224 into portal 26 MB=0x601955b5ecac0 00000800:00000200:26.0:1616513082.733240:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513082.733242:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.97@tcp1 of length 224/224 into md 0xf97ca089 [1] + 0 00000800:00000200:26.0:1616513082.733243:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.733243:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513082.733246:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71b18 00000800:00000200:26.0:1616513082.733247:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513082.733249:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.733251:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.733256:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71b18 00000800:00000200:20.0:1616513082.733259:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.733261:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.733263:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.733266:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513082.733307:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590854957952 00010000:00000200:18.0:1616513082.733319:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc579200 x1690590854957952/t0(0) o400->67b5a6ea-4762-4324-6c58-45b5e8cd6dc3@10.31.140.97@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513082.733322:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590854957760 00010000:00000200:16.0:1616513082.733332:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1690590854957760/t0(0) o400->c7a1e804-1fc4-f711-13b4-133f7a6c717c@10.31.140.97@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513082.733333:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690590854957952, offset 192 00000400:00000200:18.0:1616513082.733339:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.97@tcp1 00000100:00000200:16.0:1616513082.733345:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690590854957760, offset 192 00000400:00000200:18.0:1616513082.733346:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.97@tcp1 routed destination 00000400:00000200:16.0:1616513082.733351:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.97@tcp1 00000400:00000200:18.0:1616513082.733361:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.97@tcp1(10.31.140.97@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.733368:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513082.733370:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.97@tcp1 routed destination 00000800:00000200:18.0:1616513082.733374:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.733378:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513082.733380:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513082.733384:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.97@tcp1(10.31.140.97@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.733385:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513082.733390:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513082.733390:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.733395:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.733398:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.733400:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.733404:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.733410:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.733411:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513082.733422:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.733429:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.733432:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513082.733447:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513082.733451:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513082.733454:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513082.733457:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.733458:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdec38 00000800:00000200:22.0:1616513082.733462:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513082.733463:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513082.733465:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513082.733466:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc7f8 00000800:00000200:0.0:1616513082.733470:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513082.733473:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.738909:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.738965:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.738974:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.738983:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.250@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.738992:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.250@tcp1 of length 224 into portal 26 MB=0x603ff53800000 00000400:00000200:24.0:1616513082.738999:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.250@tcp1 of length 224/224 into md 0xf97ca0a9 [1] + 0 00000400:00000200:24.0:1616513082.739004:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f320 00000400:00000200:24.0:1616513082.739006:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.739009:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.739014:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f320 00000800:00000200:24.0:1616513082.739017:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.739017:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.739020:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.739022:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.739074:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245012705280 00010000:00000200:16.0:1616513082.739084:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1693245012705280/t0(0) o400->95e24934-9fb2-c022-d9e7-6a161fd53995@10.31.132.250@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.739098:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245012705280, offset 192 00000400:00000200:16.0:1616513082.739105:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.250@tcp1 00000400:00000200:16.0:1616513082.739112:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.250@tcp1 routed destination 00000400:00000200:16.0:1616513082.739126:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.250@tcp1(10.31.132.250@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.739132:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.739137:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.739139:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.739142:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.739146:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.739151:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.739152:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513082.739203:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.739212:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513082.739214:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513082.739220:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513082.739226:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde4c8 00000800:00000200:0.0:1616513082.739229:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513082.739232:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.739582:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.739593:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.739600:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513082.739602:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.739609:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.250@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.739617:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.250@tcp1 of length 224 into portal 12 MB=0x603ff53800100 00000400:00000200:20.0:1616513082.739622:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.250@tcp1 of length 224/224 into md 0x865bbec1 [1] + 55440 00000400:00000200:20.0:1616513082.739627:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.739630:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513082.739636:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.739640:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.739642:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513082.739698:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245012705536 00010000:00000200:4.0:1616513082.739710:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8d80 x1693245012705536/t0(0) o400->5a09be02-a13b-793d-10af-cc533c713d15@10.31.132.250@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.739723:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245012705536, offset 192 00000400:00000200:4.0:1616513082.739729:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.250@tcp1 00000400:00000200:4.0:1616513082.739736:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.250@tcp1 routed destination 00000400:00000200:4.0:1616513082.739750:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.250@tcp1(10.31.132.250@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.739756:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513082.739762:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.739764:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.739766:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.739771:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.739777:0:49738:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.739829:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.739835:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.739838:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.739844:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.739848:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bad48 00000800:00000200:14.0:1616513082.739851:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.739854:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.755162:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.755188:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.755197:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513082.755204:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513082.755206:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.12@tcp1 : PUT - for me 00000800:00000200:28.0:1616513082.755214:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513082.755215:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.12@tcp1 of length 224 into portal 26 MB=0x6000866bb4680 00000400:00000200:0.0:1616513082.755222:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.12@tcp1 of length 224/224 into md 0xf97ca0b9 [1] + 0 00000400:00000200:28.0:1616513082.755223:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.12@tcp1 : PUT - for me 00000400:00000200:0.0:1616513082.755227:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fb18 00000400:00000200:0.0:1616513082.755229:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.755232:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.12@tcp1 of length 224 into portal 12 MB=0x6000866bb4700 00000100:00000200:0.0:1616513082.755232:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513082.755237:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fb18 00000400:00000200:28.0:1616513082.755238:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.12@tcp1 of length 224/224 into md 0x865bbec1 [1] + 55664 00000800:00000200:0.0:1616513082.755240:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513082.755243:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513082.755244:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513082.755245:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:28.0:1616513082.755246:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513082.755248:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.755253:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.755256:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.755258:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.755321:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688885943551616 00000100:00000200:4.0:1616513082.755321:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688885943551744 00010000:00000200:16.0:1616513082.755332:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688885943551616/t0(0) o400->5320a032-f461-53d9-5442-431f4b637ce9@10.31.136.12@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:4.0:1616513082.755334:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedd580 x1688885943551744/t0(0) o400->83eeca1e-ff71-b044-b987-4dd20461b00b@10.31.136.12@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513082.755346:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688885943551616, offset 192 00000100:00000200:4.0:1616513082.755348:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688885943551744, offset 192 00000400:00000200:16.0:1616513082.755352:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.12@tcp1 00000400:00000200:4.0:1616513082.755358:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.12@tcp1 00000400:00000200:16.0:1616513082.755360:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.12@tcp1 routed destination 00000400:00000200:16.0:1616513082.755375:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.12@tcp1(10.31.136.12@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.755381:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:4.0:1616513082.755384:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.12@tcp1 routed destination 00000800:00000200:16.0:1616513082.755392:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.755395:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.755397:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:4.0:1616513082.755399:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.12@tcp1(10.31.136.12@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.755402:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513082.755405:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:16.0:1616513082.755408:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513082.755411:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.755414:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.755416:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513082.755417:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513082.755421:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.755428:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513082.755429:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.755445:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.755457:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.755460:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.755467:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.755473:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf4b8 00000800:00000200:20.0:1616513082.755476:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513082.755495:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.755502:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.755505:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.755511:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.755515:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baee0 00000800:00000200:22.0:1616513082.755519:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.755522:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.765708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.765764:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.765773:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.765782:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.136@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.765792:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.136@tcp1 of length 224 into portal 26 MB=0x60123d53d6540 00000400:00000200:24.0:1616513082.765799:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.136@tcp1 of length 224/224 into md 0xf97ca0d9 [1] + 0 00000400:00000200:24.0:1616513082.765804:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e4c8 00000400:00000200:24.0:1616513082.765808:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.765811:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513082.765816:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e4c8 00000800:00000200:28.0:1616513082.765818:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.765819:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.765822:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.765823:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.765856:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513082.765880:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690103273317696 00010000:00000200:16.0:1616513082.765891:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1690103273317696/t0(0) o400->e15cf158-41a0-f13e-aaa3-414dc0dd8812@10.31.143.136@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.765905:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690103273317696, offset 192 00000800:00000200:14.0:1616513082.765911:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513082.765912:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.136@tcp1 00000400:00000200:16.0:1616513082.765920:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.136@tcp1 routed destination 00000800:00000200:14.0:1616513082.765921:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.765930:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.136@tcp1 : PUT - for me 00000400:00000200:16.0:1616513082.765934:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.136@tcp1(10.31.143.136@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513082.765939:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.136@tcp1 of length 224 into portal 12 MB=0x60123d53d66c0 00000800:00000200:16.0:1616513082.765945:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513082.765945:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.136@tcp1 of length 224/224 into md 0xf964d519 [1] + 25304 00000800:00000200:16.0:1616513082.765951:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513082.765951:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.765953:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513082.765954:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.765956:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.765960:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513082.765960:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.765963:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513082.765964:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513082.765966:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513082.765967:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513082.765968:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513082.766020:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690103273318080 00000800:00000200:0.0:1616513082.766027:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:18.0:1616513082.766031:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74ddba0480 x1690103273318080/t0(0) o400->b2c148c1-4c6e-515b-ebfd-d23f26dd592b@10.31.143.136@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513082.766034:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513082.766037:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513082.766043:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:18.0:1616513082.766044:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690103273318080, offset 192 00000400:00000200:0.0:1616513082.766049:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf7e8 00000400:00000200:18.0:1616513082.766050:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.136@tcp1 00000800:00000200:0.0:1616513082.766052:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513082.766056:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513082.766057:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.136@tcp1 routed destination 00000400:00000200:18.0:1616513082.766071:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.136@tcp1(10.31.143.136@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.766077:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513082.766082:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.766085:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513082.766087:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513082.766091:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.766096:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.766107:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.766114:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.766117:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.766124:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.766127:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc000 00000800:00000200:22.0:1616513082.766131:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.766134:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.778860:0:49742:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513082.778915:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.778924:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513082.778927:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.778933:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.117@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.778942:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.117@tcp1 of length 224 into portal 26 MB=0x5ffdbc380cbc0 00000400:00000200:26.0:1616513082.778948:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.117@tcp1 of length 224/224 into md 0xf97ca0e9 [1] + 0 00000400:00000200:26.0:1616513082.778953:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43f58 00000400:00000200:26.0:1616513082.778955:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.778958:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513082.778962:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43f58 00000800:00000200:26.0:1616513082.778965:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.778968:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.778970:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.779027:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694226471872 00010000:00000200:16.0:1616513082.779038:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688694226471872/t0(0) o400->98be7c57-7bdb-7a01-3124-c51ea3434a66@10.31.140.117@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.779052:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694226471872, offset 192 00000400:00000200:16.0:1616513082.779059:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.117@tcp1 00000400:00000200:16.0:1616513082.779066:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.117@tcp1 routed destination 00000400:00000200:16.0:1616513082.779081:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.117@tcp1(10.31.140.117@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.779088:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.779093:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.779096:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.779098:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.779103:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.779109:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.779149:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.779206:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.779213:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.779216:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513082.779222:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.779228:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdee58 00000800:00000200:28.0:1616513082.779231:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.779234:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.779240:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.779248:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.117@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.779255:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.117@tcp1 of length 224 into portal 12 MB=0x5ffdbc380ce40 00000800:00000200:24.0:1616513082.779260:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513082.779262:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.117@tcp1 of length 224/224 into md 0x865bbec1 [1] + 55888 00000400:00000200:28.0:1616513082.779267:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.779269:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.779275:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.779277:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.779279:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513082.779344:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694226472512 00010000:00000200:4.0:1616513082.779357:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed8480 x1688694226472512/t0(0) o400->26094484-4f88-1f70-49d3-9931d51eef7f@10.31.140.117@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.779371:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694226472512, offset 192 00000400:00000200:4.0:1616513082.779377:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.117@tcp1 00000400:00000200:4.0:1616513082.779385:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.117@tcp1 routed destination 00000400:00000200:4.0:1616513082.779400:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.117@tcp1(10.31.140.117@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.779406:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513082.779412:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.779415:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.779417:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.779422:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.779431:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.779485:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.779492:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.779495:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.779502:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.779505:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba198 00000800:00000200:22.0:1616513082.779510:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.779512:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.807882:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.807939:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.807948:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.807958:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.54@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.807968:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.54@tcp1 of length 224 into portal 26 MB=0x60035fe54dcc0 00000400:00000200:20.0:1616513082.807975:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.54@tcp1 of length 224/224 into md 0xf97ca0f9 [1] + 0 00000400:00000200:20.0:1616513082.807980:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43760 00000400:00000200:20.0:1616513082.807982:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.807985:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.807990:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43760 00000800:00000200:24.0:1616513082.807993:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.807993:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.807996:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.807998:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.808055:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689081760505024 00010000:00000200:16.0:1616513082.808067:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1689081760505024/t0(0) o400->4b097f34-24c2-0753-76a9-d8bf46c17d07@10.31.133.54@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.808081:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689081760505024, offset 192 00000400:00000200:16.0:1616513082.808088:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.54@tcp1 00000400:00000200:16.0:1616513082.808095:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.54@tcp1 routed destination 00000800:00000200:11.2:1616513082.808110:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:16.0:1616513082.808111:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.54@tcp1(10.31.133.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.808117:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.808123:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.808126:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.808130:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513082.808136:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:16.0:1616513082.808142:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513082.808165:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.808177:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.808186:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.54@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.808194:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.54@tcp1 of length 224 into portal 12 MB=0x60035fe54dd80 00000400:00000200:28.0:1616513082.808201:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.54@tcp1 of length 224/224 into md 0x865bbec1 [1] + 56112 00000400:00000200:28.0:1616513082.808206:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.808209:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.808215:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513082.808218:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513082.808220:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513082.808223:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.808231:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513082.808234:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513082.808235:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513082.808241:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513082.808247:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde088 00000800:00000200:0.0:1616513082.808250:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513082.808280:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689081760505216 00010000:00000200:4.0:1616513082.808293:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed8d80 x1689081760505216/t0(0) o400->2d8e795b-51c9-2f76-abc4-f3d25b886fbc@10.31.133.54@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.808307:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689081760505216, offset 192 00000400:00000200:4.0:1616513082.808314:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.54@tcp1 00000400:00000200:4.0:1616513082.808321:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.54@tcp1 routed destination 00000400:00000200:4.0:1616513082.808336:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.54@tcp1(10.31.133.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.808343:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513082.808349:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.808352:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.808354:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.808359:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.808366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.808420:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.808427:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.808430:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.808448:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.808452:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb540 00000800:00000200:14.0:1616513082.808456:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.808459:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.809171:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513082.809225:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.809235:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513082.809244:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.809253:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x60399133ead00 00000400:00000200:22.0:1616513082.809259:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0xf964d519 [1] + 25528 00000400:00000200:22.0:1616513082.809265:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.809268:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513082.809276:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.809279:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513082.809281:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.809286:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513082.809362:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805848018176 00010000:00000200:18.0:1616513082.809431:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dec201b00 x1692805848018176/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:708/0 lens 336/336 e 0 to 0 dl 1616513088 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513082.809459:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805848018176, offset 192 00000400:00000200:18.0:1616513082.809466:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:18.0:1616513082.809473:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:18.0:1616513082.809487:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.809494:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513082.809500:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.809502:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513082.809504:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513082.809509:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.809515:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.809569:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.809575:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.809578:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.809585:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.809589:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccff0 00000800:00000200:20.0:1616513082.809593:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.809596:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.824224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513082.824281:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.824290:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513082.824298:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.10@tcp1 : PUT - for me 00000400:00000200:14.0:1616513082.824308:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.10@tcp1 of length 224 into portal 26 MB=0x6001c0ad42600 00000400:00000200:14.0:1616513082.824314:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.10@tcp1 of length 224/224 into md 0xf97ca129 [1] + 0 00000400:00000200:14.0:1616513082.824319:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a424c8 00000400:00000200:14.0:1616513082.824321:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513082.824324:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513082.824329:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a424c8 00000800:00000200:14.0:1616513082.824332:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.824333:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.824335:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513082.824338:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513082.824392:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688970301023744 00010000:00000200:16.0:1616513082.824403:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688970301023744/t0(0) o400->9066cc35-a29b-a7b8-a62e-5e1ae9057f09@10.31.133.10@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.824418:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688970301023744, offset 192 00000400:00000200:16.0:1616513082.824424:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.10@tcp1 00000400:00000200:16.0:1616513082.824431:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.10@tcp1 routed destination 00000400:00000200:16.0:1616513082.824460:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.10@tcp1(10.31.133.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.824466:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.824472:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.824475:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.824477:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.824482:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.824487:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.824489:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513082.824540:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.824547:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.824550:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.824557:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.824562:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf430 00000800:00000200:22.0:1616513082.824566:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.824569:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.825020:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513082.825031:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.825039:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513082.825040:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.825047:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.10@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.825056:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.10@tcp1 of length 224 into portal 12 MB=0x6001c0ad42740 00000400:00000200:26.0:1616513082.825063:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.10@tcp1 of length 224/224 into md 0xf964d519 [1] + 25864 00000400:00000200:26.0:1616513082.825068:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.825070:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.825077:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.825080:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.825082:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513082.825147:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688970301024064 00010000:00000200:18.0:1616513082.825159:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc579b00 x1688970301024064/t0(0) o400->20b8130b-1a19-4471-14a4-ba95a73a1989@10.31.133.10@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513082.825186:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688970301024064, offset 192 00000400:00000200:18.0:1616513082.825192:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.10@tcp1 00000400:00000200:18.0:1616513082.825200:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.10@tcp1 routed destination 00000400:00000200:18.0:1616513082.825214:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.10@tcp1(10.31.133.10@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.825221:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513082.825227:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.825229:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513082.825231:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513082.825236:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.825242:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.825296:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.825303:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.825306:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513082.825311:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.825312:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.825316:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc6e8 00000800:00000200:24.0:1616513082.825320:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.826723:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.826778:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.826787:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513082.826796:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.5@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.826805:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.5@tcp1 of length 224 into portal 26 MB=0x5ffd4c2437c40 00000400:00000200:20.0:1616513082.826811:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.5@tcp1 of length 224/224 into md 0xf97ca149 [1] + 0 00000400:00000200:20.0:1616513082.826816:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43298 00000400:00000200:20.0:1616513082.826820:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.826823:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.826827:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43298 00000800:00000200:20.0:1616513082.826830:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.826831:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.826833:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513082.826835:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.826890:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664140905536 00010000:00000200:16.0:1616513082.826901:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688664140905536/t0(0) o400->49636c3a-b96c-d4c0-4b86-40777956e92a@10.31.136.5@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.826916:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664140905536, offset 192 00000400:00000200:16.0:1616513082.826922:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.5@tcp1 00000400:00000200:16.0:1616513082.826929:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.5@tcp1 routed destination 00000400:00000200:16.0:1616513082.826944:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.5@tcp1(10.31.136.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.826951:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.826956:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.826959:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.826961:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.826966:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.826971:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.826973:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.827005:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513082.827025:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.827032:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.827035:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513082.827041:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.827047:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde550 00000800:00000200:22.0:1616513082.827050:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.827053:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513082.827060:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.827069:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513082.827078:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.196@tcp1 : PUT - for me 00000400:00000200:0.0:1616513082.827086:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.196@tcp1 of length 224 into portal 26 MB=0x6000e4b0f46c0 00000400:00000200:0.0:1616513082.827093:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.196@tcp1 of length 224/224 into md 0xf97ca169 [1] + 0 00000400:00000200:0.0:1616513082.827098:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a435c8 00000400:00000200:0.0:1616513082.827101:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513082.827104:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513082.827108:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a435c8 00000800:00000200:0.0:1616513082.827111:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.827115:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513082.827115:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:16.0:1616513082.827121:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688911249098432 00000800:00000200:0.0:1616513082.827122:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513082.827125:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.827126:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00010000:00000200:16.0:1616513082.827126:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688911249098432/t0(0) o400->9e48258e-7ebb-ffd8-184a-c7cf6464cc61@10.31.142.196@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:24.0:1616513082.827134:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.196@tcp1 : PUT - for me 00000100:00000200:16.0:1616513082.827139:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688911249098432, offset 192 00000400:00000200:24.0:1616513082.827142:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.196@tcp1 of length 224 into portal 12 MB=0x6000e4b0f4a00 00000400:00000200:16.0:1616513082.827144:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.196@tcp1 00000400:00000200:24.0:1616513082.827149:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.196@tcp1 of length 224/224 into md 0x865bbec1 [1] + 56336 00000400:00000200:16.0:1616513082.827150:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.196@tcp1 routed destination 00000400:00000200:24.0:1616513082.827154:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.827157:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513082.827164:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.196@tcp1(10.31.142.196@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513082.827169:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.827170:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513082.827172:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.827174:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513082.827174:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.827177:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513082.827178:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.827182:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513082.827186:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513082.827187:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.827198:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.827202:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.827204:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513082.827207:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.827210:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.827214:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf8f8 00000800:00000200:20.0:1616513082.827217:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513082.827227:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688911249099264 00010000:00000200:4.0:1616513082.827240:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd704c80 x1688911249099264/t0(0) o400->74ac560a-0e8f-02b6-09e2-4cb3ea9ea877@10.31.142.196@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.827254:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688911249099264, offset 192 00000400:00000200:4.0:1616513082.827260:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.196@tcp1 00000400:00000200:4.0:1616513082.827268:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.196@tcp1 routed destination 00000400:00000200:4.0:1616513082.827285:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.196@tcp1(10.31.142.196@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.827291:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513082.827297:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.827300:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513082.827302:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513082.827307:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.827312:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513082.827365:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.827372:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.827375:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513082.827380:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513082.827381:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.827385:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbcb0 00000800:00000200:14.0:1616513082.827389:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513082.827795:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513082.827805:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.827811:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513082.827814:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.827819:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.5@tcp1 : PUT - for me 00000400:00000200:22.0:1616513082.827826:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.5@tcp1 of length 224 into portal 12 MB=0x5ffd4c2437d40 00000400:00000200:22.0:1616513082.827833:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.5@tcp1 of length 224/224 into md 0xf964d519 [1] + 26088 00000400:00000200:22.0:1616513082.827838:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513082.827840:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513082.827847:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.827850:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513082.827852:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513082.827860:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664140905792 00010000:00000200:18.0:1616513082.827866:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2ec00 x1688664140905792/t0(0) o400->75698f7e-17a0-f66d-4807-50b31674fd31@10.31.136.5@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513082.827876:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664140905792, offset 192 00000400:00000200:18.0:1616513082.827881:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.5@tcp1 00000400:00000200:18.0:1616513082.827886:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.5@tcp1 routed destination 00000400:00000200:18.0:1616513082.827898:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.5@tcp1(10.31.136.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.827903:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513082.827908:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.827910:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513082.827911:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513082.827914:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.827922:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.827932:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.827938:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513082.827940:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513082.827942:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513082.827947:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513082.827950:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc220 00000800:00000200:28.0:1616513082.827954:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.830188:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513082.830241:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.830250:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513082.830260:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.217@tcp1 : PUT - for me 00000400:00000200:0.0:1616513082.830269:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.217@tcp1 of length 224 into portal 26 MB=0x5ffe6336eb300 00000400:00000200:0.0:1616513082.830275:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.217@tcp1 of length 224/224 into md 0xf97ca179 [1] + 0 00000400:00000200:0.0:1616513082.830280:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43980 00000400:00000200:0.0:1616513082.830282:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513082.830285:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513082.830289:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43980 00000800:00000200:0.0:1616513082.830292:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.830294:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513082.830295:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513082.830298:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.830354:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739054007040 00010000:00000200:16.0:1616513082.830365:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688739054007040/t0(0) o400->3f82143d-3d19-e4a8-66fd-bfc705c7ecd2@10.31.142.217@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.830379:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688739054007040, offset 192 00000400:00000200:16.0:1616513082.830386:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.217@tcp1 00000400:00000200:16.0:1616513082.830393:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.217@tcp1 routed destination 00000400:00000200:16.0:1616513082.830408:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.217@tcp1(10.31.142.217@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.830415:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.830421:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.830423:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.830426:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.830430:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.830435:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.830447:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513082.830488:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.830495:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513082.830497:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513082.830504:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513082.830509:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdff58 00000800:00000200:26.0:1616513082.830515:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.830518:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.830523:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.830531:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.217@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.830539:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.217@tcp1 of length 224 into portal 12 MB=0x5ffe6336eb5c0 00000800:00000200:14.0:1616513082.830540:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513082.830546:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.217@tcp1 of length 224/224 into md 0xf964d519 [1] + 26312 00000400:00000200:26.0:1616513082.830551:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.830553:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513082.830559:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.830562:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.830564:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513082.830569:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739054007744 00010000:00000200:18.0:1616513082.830574:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57f080 x1688739054007744/t0(0) o400->478bc439-a809-2d9e-e51e-5400d9ba93c9@10.31.142.217@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513082.830583:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688739054007744, offset 192 00000400:00000200:18.0:1616513082.830588:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.217@tcp1 00000400:00000200:18.0:1616513082.830593:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.217@tcp1 routed destination 00000400:00000200:18.0:1616513082.830605:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.217@tcp1(10.31.142.217@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513082.830611:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513082.830615:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513082.830617:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513082.830618:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513082.830621:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.830626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.830686:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.830693:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.830696:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513082.830702:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.830706:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc550 00000800:00000200:24.0:1616513082.830710:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.830713:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.830940:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513082.830951:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.830959:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513082.830961:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.830969:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.89@tcp1 : PUT - for me 00000400:00000200:20.0:1616513082.830977:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.89@tcp1 of length 224 into portal 26 MB=0x600204627a540 00000400:00000200:20.0:1616513082.830984:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.89@tcp1 of length 224/224 into md 0xf97ca189 [1] + 0 00000400:00000200:20.0:1616513082.830989:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42220 00000400:00000200:20.0:1616513082.830991:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513082.830994:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513082.830999:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42220 00000800:00000200:20.0:1616513082.831002:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.831005:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.831006:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.831011:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688988476220736 00010000:00000200:16.0:1616513082.831017:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688988476220736/t0(0) o400->8becba12-3aab-9367-07f5-35a59101e910@10.31.142.89@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.831029:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688988476220736, offset 192 00000400:00000200:16.0:1616513082.831034:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.89@tcp1 00000400:00000200:16.0:1616513082.831040:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.89@tcp1 routed destination 00000400:00000200:16.0:1616513082.831054:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.89@tcp1(10.31.142.89@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.831060:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.831065:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.831067:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.831069:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.831072:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513082.831076:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.831077:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513082.831130:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513082.831137:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513082.831140:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513082.831140:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513082.831146:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513082.831152:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf320 00000800:00000200:22.0:1616513082.831156:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.831840:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.831917:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.831928:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.831936:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.89@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.831944:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.89@tcp1 of length 224 into portal 12 MB=0x600204627b300 00000400:00000200:24.0:1616513082.831951:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.89@tcp1 of length 224/224 into md 0x865bbec1 [1] + 56560 00000400:00000200:24.0:1616513082.831956:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.831958:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.831965:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.831979:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.831980:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.831983:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513082.832032:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688988476224256 00010000:00000200:4.0:1616513082.832044:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd705100 x1688988476224256/t0(0) o400->db56d55f-1a67-e7cc-1bdf-737e3f64c01b@10.31.142.89@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.832058:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688988476224256, offset 192 00000400:00000200:4.0:1616513082.832065:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.89@tcp1 00000400:00000200:4.0:1616513082.832072:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.89@tcp1 routed destination 00000400:00000200:4.0:1616513082.832087:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.89@tcp1(10.31.142.89@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.832093:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513082.832099:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.832102:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513082.832104:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513082.832108:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.832113:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513082.832123:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.832127:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513082.832130:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513082.832131:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513082.832135:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513082.832138:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baff0 00000800:00000200:0.0:1616513082.832141:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:27.0:1616513082.871194:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:27.0:1616513082.871202:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:7.2:1616513082.897269:0:49753:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513082.897329:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513082.897340:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513082.897350:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.193@tcp1 : PUT - for me 00000400:00000200:26.0:1616513082.897361:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.193@tcp1 of length 224 into portal 26 MB=0x604b7e5113b00 00000400:00000200:26.0:1616513082.897368:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.193@tcp1 of length 224/224 into md 0xf97ca199 [1] + 0 00000400:00000200:26.0:1616513082.897373:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42088 00000400:00000200:26.0:1616513082.897376:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513082.897379:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513082.897384:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42088 00000800:00000200:2.0:1616513082.897385:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.897388:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513082.897391:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513082.897393:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.897461:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513082.897467:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694037728901888 00010000:00000200:16.0:1616513082.897480:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1694037728901888/t0(0) o400->8333f7c3-6f91-907d-3cd3-1cd8c760952f@10.31.140.193@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.897495:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694037728901888, offset 192 00000400:00000200:16.0:1616513082.897502:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.193@tcp1 00000400:00000200:16.0:1616513082.897511:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.193@tcp1 routed destination 00000400:00000200:16.0:1616513082.897527:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.193@tcp1(10.31.140.193@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513082.897531:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513082.897534:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513082.897541:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513082.897541:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.897544:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.897546:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513082.897551:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.193@tcp1 : PUT - for me 00000800:00000200:16.0:1616513082.897551:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513082.897556:0:49753:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.897559:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513082.897560:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.193@tcp1 of length 224 into portal 12 MB=0x604b7e5113f00 00000400:00000200:24.0:1616513082.897566:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.193@tcp1 of length 224/224 into md 0x865bbec1 [1] + 56784 00000400:00000200:24.0:1616513082.897572:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.897575:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.897582:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.897585:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.897586:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.897590:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513082.897614:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.897621:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.897624:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.897630:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.897636:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdeee0 00000800:00000200:14.0:1616513082.897640:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.897645:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513082.897651:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694037728902912 00010000:00000200:4.0:1616513082.897665:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd700480 x1694037728902912/t0(0) o400->0f132103-9cd1-1ebd-dca8-776cdeaa84c4@10.31.140.193@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.897680:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694037728902912, offset 192 00000400:00000200:4.0:1616513082.897687:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.193@tcp1 00000400:00000200:4.0:1616513082.897694:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.193@tcp1 routed destination 00000400:00000200:4.0:1616513082.897710:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.193@tcp1(10.31.140.193@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.897716:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513082.897722:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.897725:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513082.897729:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513082.897734:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.897739:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.897752:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.897760:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.897763:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513082.897769:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.897773:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baff0 00000800:00000200:20.0:1616513082.897777:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513082.897780:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.904961:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513082.905018:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.905028:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.905037:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.59@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.905046:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.59@tcp1 of length 224 into portal 26 MB=0x60478e7f53d00 00000400:00000200:28.0:1616513082.905052:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.59@tcp1 of length 224/224 into md 0xf97ca1b9 [1] + 0 00000400:00000200:28.0:1616513082.905057:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a427f8 00000400:00000200:28.0:1616513082.905060:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.905062:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513082.905067:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a427f8 00000800:00000200:28.0:1616513082.905070:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513082.905073:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513082.905073:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513082.905080:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.905083:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.905084:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.905092:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.59@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.905101:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.59@tcp1 of length 224 into portal 12 MB=0x60478e7f54100 00000400:00000200:24.0:1616513082.905107:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.59@tcp1 of length 224/224 into md 0x865bbec1 [1] + 57008 00000400:00000200:24.0:1616513082.905112:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.905115:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513082.905121:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.905125:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.905126:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.905131:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693767194459392 00010000:00000200:16.0:1616513082.905142:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1693767194459392/t0(0) o400->ffdc357b-c3dd-8310-d62d-d271ae0983b4@10.31.129.59@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.905156:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693767194459392, offset 192 00000400:00000200:16.0:1616513082.905162:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.59@tcp1 00000400:00000200:16.0:1616513082.905169:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.59@tcp1 routed destination 00000400:00000200:16.0:1616513082.905184:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.59@tcp1(10.31.129.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513082.905184:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693767194460416 00000800:00000200:16.0:1616513082.905191:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:4.0:1616513082.905195:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd707080 x1693767194460416/t0(0) o400->04471d2c-5530-c8b0-6642-bb8681c0ca0a@10.31.129.59@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513082.905197:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.905202:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.905204:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.905209:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513082.905209:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693767194460416, offset 192 00000400:00000200:4.0:1616513082.905215:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.59@tcp1 00000100:00000200:16.0:1616513082.905216:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.905217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513082.905222:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.59@tcp1 routed destination 00000400:00000200:4.0:1616513082.905236:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.59@tcp1(10.31.129.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.905242:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513082.905247:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.905250:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513082.905252:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513082.905256:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.905261:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513082.905289:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.905296:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513082.905299:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513082.905306:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513082.905312:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfcb0 00000800:00000200:2.0:1616513082.905315:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513082.905315:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513082.905318:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513082.905323:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513082.905326:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513082.905327:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513082.905332:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513082.905336:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbcb0 00000800:00000200:0.0:1616513082.905340:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513082.960036:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513082.960093:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.960104:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513082.960115:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.199@tcp1 : PUT - for me 00000400:00000200:24.0:1616513082.960126:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.199@tcp1 of length 224 into portal 26 MB=0x5ffd4e0e60f80 00000400:00000200:24.0:1616513082.960133:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.199@tcp1 of length 224/224 into md 0xf97ca1c9 [1] + 0 00000400:00000200:24.0:1616513082.960138:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43dc0 00000400:00000200:24.0:1616513082.960141:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513082.960145:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513082.960148:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.960150:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43dc0 00000800:00000200:24.0:1616513082.960153:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.960156:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513082.960158:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513082.960218:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664654876544 00010000:00000200:16.0:1616513082.960231:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688664654876544/t0(0) o400->8c0e4b27-fd85-cf18-9370-4e6df5cc922d@10.31.128.199@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.960246:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664654876544, offset 192 00000400:00000200:16.0:1616513082.960253:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.199@tcp1 00000400:00000200:16.0:1616513082.960261:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.199@tcp1 routed destination 00000400:00000200:16.0:1616513082.960277:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.199@tcp1(10.31.128.199@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.960284:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513082.960291:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513082.960294:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.960296:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513082.960302:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513082.960309:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513082.960309:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513082.960355:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513082.960365:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.960372:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.960376:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.960383:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.960389:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfe48 00000800:00000200:14.0:1616513082.960392:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.960395:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513082.960410:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513082.960421:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513082.960451:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.199@tcp1 : PUT - for me 00000400:00000200:28.0:1616513082.960461:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.199@tcp1 of length 224 into portal 12 MB=0x5ffd4e0e61340 00000400:00000200:28.0:1616513082.960467:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.199@tcp1 of length 224/224 into md 0x865bbec1 [1] + 57232 00000400:00000200:28.0:1616513082.960472:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513082.960475:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513082.960482:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.960482:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.960486:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513082.960488:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513082.960508:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664654877504 00010000:00000200:4.0:1616513082.960521:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed9200 x1688664654877504/t0(0) o400->155c051e-0497-8743-6705-6cca0dfb2b2a@10.31.128.199@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.960536:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664654877504, offset 192 00000400:00000200:4.0:1616513082.960542:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.199@tcp1 00000400:00000200:4.0:1616513082.960550:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.199@tcp1 routed destination 00000400:00000200:4.0:1616513082.960565:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.199@tcp1(10.31.128.199@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.960572:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513082.960578:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.960581:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513082.960583:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513082.960588:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.960592:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513082.960603:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513082.960608:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513082.960611:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513082.960613:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513082.960618:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513082.960622:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb540 00000800:00000200:20.0:1616513082.960626:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513082.991880:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513082.991938:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513082.991948:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513082.991957:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.78@tcp1 : PUT - for me 00000400:00000200:2.0:1616513082.991968:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.78@tcp1 of length 224 into portal 26 MB=0x5ffdd7c4d2380 00000400:00000200:2.0:1616513082.991974:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.78@tcp1 of length 224/224 into md 0xf97ca1d9 [1] + 0 00000400:00000200:2.0:1616513082.991979:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a438f8 00000400:00000200:2.0:1616513082.991982:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513082.991985:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513082.991990:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a438f8 00000800:00000200:26.0:1616513082.991993:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513082.991993:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513082.991996:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513082.991998:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513082.992035:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513082.992056:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701621838720 00010000:00000200:16.0:1616513082.992068:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688701621838720/t0(0) o400->6d83b3c6-1e0d-60f1-1ca6-46f9ab7a8238@10.31.136.78@tcp1:39/0 lens 224/224 e 0 to 0 dl 1616513174 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513082.992082:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701621838720, offset 192 00000400:00000200:16.0:1616513082.992089:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.78@tcp1 00000800:00000200:0.0:1616513082.992093:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513082.992097:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.78@tcp1 routed destination 00000800:00000200:0.0:1616513082.992103:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513082.992112:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.78@tcp1 : PUT - for me 00000400:00000200:16.0:1616513082.992113:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.78@tcp1(10.31.136.78@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513082.992119:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513082.992122:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.78@tcp1 of length 224 into portal 12 MB=0x5ffdd7c4d2640 00000800:00000200:16.0:1616513082.992126:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:0.0:1616513082.992128:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.78@tcp1 of length 224/224 into md 0x865bbec1 [1] + 57456 00000800:00000200:16.0:1616513082.992129:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513082.992131:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513082.992133:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513082.992136:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513082.992136:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513082.992141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513082.992143:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513082.992144:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513082.992147:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513082.992148:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513082.992152:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513082.992154:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513082.992161:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513082.992163:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513082.992170:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513082.992175:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfd38 00000800:00000200:14.0:1616513082.992179:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513082.992182:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513082.992206:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701621839424 00010000:00000200:4.0:1616513082.992218:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86c380 x1688701621839424/t0(0) o400->15f8bef5-dd7d-8a69-65b4-cc17b23a4735@10.31.136.78@tcp1:708/0 lens 224/224 e 0 to 0 dl 1616513088 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513082.992232:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701621839424, offset 192 00000400:00000200:4.0:1616513082.992239:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.78@tcp1 00000400:00000200:4.0:1616513082.992246:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.78@tcp1 routed destination 00000400:00000200:4.0:1616513082.992260:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.78@tcp1(10.31.136.78@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513082.992266:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513082.992272:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513082.992274:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513082.992276:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513082.992281:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513082.992287:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513082.992342:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513082.992348:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513082.992351:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513082.992354:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513082.992358:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513082.992362:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba198 00000800:00000200:24.0:1616513082.992366:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.020095:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.020160:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.020170:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.020180:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.165@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.020189:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.165@tcp1 of length 224 into portal 26 MB=0x5ffd531cbbf80 00000400:00000200:26.0:1616513083.020196:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.165@tcp1 of length 224/224 into md 0xf97ca1f9 [1] + 0 00000400:00000200:26.0:1616513083.020201:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43188 00000400:00000200:26.0:1616513083.020204:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.020207:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.020213:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43188 00000800:00000200:2.0:1616513083.020217:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.020218:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.020221:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.020223:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.020247:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513083.020282:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666012106624 00010000:00000200:16.0:1616513083.020294:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688666012106624/t0(0) o400->197740b1-2bbb-3243-e570-ac429d407540@10.31.129.165@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513083.020304:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.020309:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666012106624, offset 192 00000800:00000200:14.0:1616513083.020314:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513083.020315:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.165@tcp1 00000400:00000200:16.0:1616513083.020323:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.165@tcp1 routed destination 00000400:00000200:14.0:1616513083.020323:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.165@tcp1 : PUT - for me 00000400:00000200:16.0:1616513083.020340:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.165@tcp1(10.31.129.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.020347:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513083.020347:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.165@tcp1 of length 224 into portal 12 MB=0x5ffd531cbc300 00000800:00000200:16.0:1616513083.020354:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513083.020354:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.165@tcp1 of length 224/224 into md 0xf964d519 [1] + 26536 00000800:00000200:16.0:1616513083.020357:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513083.020359:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513083.020359:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513083.020359:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.020362:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513083.020365:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:7.2:1616513083.020370:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:16.0:1616513083.020372:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513083.020372:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513083.020375:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513083.020377:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513083.020380:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.020384:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513083.020387:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513083.020389:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513083.020392:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.020397:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdef68 00000100:00000200:18.0:1616513083.020399:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666012107520 00000800:00000200:14.0:1616513083.020400:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:18.0:1616513083.020412:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464624c80 x1688666012107520/t0(0) o400->eb23823d-8c6b-6eda-26fc-93dc6fbcd9d2@10.31.129.165@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.020427:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666012107520, offset 192 00000400:00000200:18.0:1616513083.020447:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.165@tcp1 00000400:00000200:18.0:1616513083.020455:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.165@tcp1 routed destination 00000400:00000200:18.0:1616513083.020470:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.165@tcp1(10.31.129.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.020477:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.020483:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.020485:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.020487:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.020492:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.020501:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.020519:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.020526:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.020529:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.020535:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.020539:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdb18 00000800:00000200:28.0:1616513083.020543:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.020546:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.046220:0:49768:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513083.046278:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513083.046289:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513083.046300:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.112@tcp1 : PUT - for me 00000400:00000200:22.0:1616513083.046310:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.112@tcp1 of length 224 into portal 26 MB=0x6018388871280 00000400:00000200:22.0:1616513083.046317:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.112@tcp1 of length 224/224 into md 0xf97ca209 [1] + 0 00000400:00000200:22.0:1616513083.046322:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43ed0 00000400:00000200:22.0:1616513083.046325:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513083.046328:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513083.046332:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ed0 00000800:00000200:2.0:1616513083.046334:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.046336:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513083.046339:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.046341:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.046399:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514303160960 00010000:00000200:16.0:1616513083.046411:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1690514303160960/t0(0) o400->d2bd37c8-b8ed-10fb-96c0-9c4131707869@10.31.142.112@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.046426:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514303160960, offset 192 00000400:00000200:16.0:1616513083.046446:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.112@tcp1 00000400:00000200:16.0:1616513083.046455:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.112@tcp1 routed destination 00000400:00000200:16.0:1616513083.046470:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.112@tcp1(10.31.142.112@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.046477:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.046484:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.046487:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.046489:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.046494:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.046499:0:49768:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.046502:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513083.046557:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.046564:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.046567:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.046574:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.046579:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde220 00000800:00000200:26.0:1616513083.046583:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.046586:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.046621:0:49768:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513083.046633:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.046642:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513083.046644:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513083.046652:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.112@tcp1 : PUT - for me 00000400:00000200:2.0:1616513083.046660:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.112@tcp1 of length 224 into portal 12 MB=0x6018388871700 00000400:00000200:2.0:1616513083.046667:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.112@tcp1 of length 224/224 into md 0xf964d519 [1] + 26760 00000400:00000200:2.0:1616513083.046672:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513083.046675:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513083.046683:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.046686:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.046687:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.046751:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514303162112 00010000:00000200:18.0:1616513083.046764:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d7500 x1690514303162112/t0(0) o400->d1271297-62f1-5b22-3ce3-0c7913dacef4@10.31.142.112@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.046778:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514303162112, offset 192 00000400:00000200:18.0:1616513083.046784:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.112@tcp1 00000400:00000200:18.0:1616513083.046792:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.112@tcp1 routed destination 00000400:00000200:18.0:1616513083.046806:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.112@tcp1(10.31.142.112@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.046813:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.046819:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.046821:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.046823:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.046829:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.046836:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.046892:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.046899:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.046902:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513083.046909:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.046913:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc908 00000800:00000200:20.0:1616513083.046917:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.046920:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.049895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.049951:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.049960:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.049968:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.142@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.049978:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.142@tcp1 of length 224 into portal 26 MB=0x5ffe94db66200 00000400:00000200:26.0:1616513083.049984:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.142@tcp1 of length 224/224 into md 0xf97ca239 [1] + 0 00000400:00000200:26.0:1616513083.049989:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42dd0 00000400:00000200:26.0:1616513083.049992:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.049994:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.049998:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42dd0 00000800:00000200:26.0:1616513083.050001:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.050004:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513083.050004:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.050011:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513083.050014:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.050014:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513083.050023:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.142@tcp1 : PUT - for me 00000400:00000200:14.0:1616513083.050033:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.142@tcp1 of length 224 into portal 12 MB=0x5ffe94db66280 00000400:00000200:14.0:1616513083.050040:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.142@tcp1 of length 224/224 into md 0xf964d519 [1] + 26984 00000400:00000200:14.0:1616513083.050045:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.050048:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513083.050054:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.050057:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.050059:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.050063:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688752379814400 00010000:00000200:16.0:1616513083.050074:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688752379814400/t0(0) o400->67ee6db6-d7a4-4008-1764-76a5af260781@10.31.141.142@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.050088:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688752379814400, offset 192 00000400:00000200:16.0:1616513083.050096:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.142@tcp1 00000400:00000200:16.0:1616513083.050104:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.142@tcp1 routed destination 00000100:00000200:18.0:1616513083.050118:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688752379814528 00000400:00000200:16.0:1616513083.050119:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.142@tcp1(10.31.141.142@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.050125:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:18.0:1616513083.050130:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464626c00 x1688752379814528/t0(0) o400->385b7e58-1e6c-408b-d66a-21cdb6fd7b30@10.31.141.142@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.050131:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.050134:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.050136:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.050141:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:18.0:1616513083.050145:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688752379814528, offset 192 00000800:00000200:7.2:1616513083.050146:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.050148:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513083.050151:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.142@tcp1 00000400:00000200:18.0:1616513083.050158:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.142@tcp1 routed destination 00000400:00000200:18.0:1616513083.050173:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.142@tcp1(10.31.141.142@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.050180:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.050185:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.050188:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.050190:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.050195:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.050201:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513083.050201:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.050208:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.050211:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513083.050212:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513083.050218:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.050223:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf078 00000800:00000200:2.0:1616513083.050227:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513083.050255:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.050262:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.050265:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.050271:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.050275:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccd48 00000800:00000200:28.0:1616513083.050280:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.050282:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.050288:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513083.050290:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.050292:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.050294:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.050297:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:27.0:1616513083.050622:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926160, portal 4 00000100:00000200:27.0:1616513083.050628:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926160, offset 0 00000400:00000200:27.0:1616513083.050637:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:27.0:1616513083.050644:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:27.0:1616513083.050659:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513083.050666:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:27.0:1616513083.050674:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513083.050677:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:27.0:1616513083.050679:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513083.050686:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513083.050733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513083.050792:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513083.050809:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:24.0:1616513083.050812:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513083.050813:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:24.0:1616513083.050819:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:24.0:1616513083.050825:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bfc380 x1693067725926160/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513083.050837:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248908 00000800:00000200:24.0:1616513083.050842:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513083.050951:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:0.0:1616513083.050964:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513083.050974:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513083.050977:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:0.0:1616513083.050983:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:0.0:1616513083.050991:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626f10 00000400:00000200:0.0:1616513083.050998:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d7311 [1] + 192 00000400:00000200:0.0:1616513083.051004:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.051009:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bfc380 x1693067725926160/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:0.0:1616513083.051024:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513083.051027:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513083.051029:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:27.0:1616513083.051035:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bfc380 x1693067725926160/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513083.051049:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bfc380 x1693067725926160/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513083.051060:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248088 00000800:00000200:7.2:1616513083.067254:0:49770:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513083.067311:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.067321:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513083.067330:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.88@tcp1 : PUT - for me 00000400:00000200:14.0:1616513083.067340:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.88@tcp1 of length 224 into portal 26 MB=0x5ffdf83e2cc00 00000400:00000200:14.0:1616513083.067347:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.88@tcp1 of length 224/224 into md 0xf97ca249 [1] + 0 00000400:00000200:14.0:1616513083.067352:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42ee0 00000400:00000200:14.0:1616513083.067354:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.067357:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513083.067361:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ee0 00000800:00000200:14.0:1616513083.067365:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513083.067366:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.067368:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513083.067372:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513083.067375:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.067376:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513083.067385:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.88@tcp1 : PUT - for me 00000400:00000200:22.0:1616513083.067395:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.88@tcp1 of length 224 into portal 12 MB=0x5ffdf83e2cc80 00000400:00000200:22.0:1616513083.067401:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.88@tcp1 of length 224/224 into md 0xf964d519 [1] + 27208 00000400:00000200:22.0:1616513083.067407:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513083.067410:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513083.067417:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513083.067420:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.067422:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.067426:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688710339021824 00010000:00000200:16.0:1616513083.067451:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688710339021824/t0(0) o400->5d5a2c42-4e67-d78d-af11-17decbe82ee4@10.31.128.88@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.067466:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688710339021824, offset 192 00000400:00000200:16.0:1616513083.067488:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.88@tcp1 00000100:00000200:18.0:1616513083.067489:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688710339021952 00000400:00000200:16.0:1616513083.067496:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.88@tcp1 routed destination 00010000:00000200:18.0:1616513083.067502:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a28480 x1688710339021952/t0(0) o400->8b50665c-ed1d-2680-f2a7-dda9fb646e96@10.31.128.88@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513083.067510:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.88@tcp1(10.31.128.88@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513083.067516:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688710339021952, offset 192 00000800:00000200:16.0:1616513083.067516:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.067522:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513083.067523:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.88@tcp1 00000800:00000200:16.0:1616513083.067525:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.067527:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513083.067531:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.88@tcp1 routed destination 00000800:00000200:16.0:1616513083.067532:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.067537:0:49770:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.067538:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513083.067545:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.88@tcp1(10.31.128.88@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.067552:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.067558:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.067560:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.067572:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.067577:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.067597:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513083.067601:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.067608:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.067611:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513083.067612:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513083.067618:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.067623:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fded48 00000800:00000200:2.0:1616513083.067627:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513083.067653:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.067660:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.067663:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.067669:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.067673:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd980 00000800:00000200:28.0:1616513083.067680:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.067683:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.077890:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513083.077904:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513083.077913:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513083.077922:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.152@tcp1 : PUT - for me 00000400:00000200:22.0:1616513083.077932:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.152@tcp1 of length 224 into portal 26 MB=0x603e114d9d3c0 00000400:00000200:22.0:1616513083.077938:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.152@tcp1 of length 224/224 into md 0xf97ca259 [1] + 0 00000400:00000200:22.0:1616513083.077943:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43c28 00000400:00000200:22.0:1616513083.077945:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513083.077948:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513083.077952:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43c28 00000800:00000200:22.0:1616513083.077955:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.077957:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.077958:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513083.077962:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.078029:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115112608704 00010000:00000200:16.0:1616513083.078041:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1693115112608704/t0(0) o400->45cd9c61-d4fa-091b-f3b0-6f886a6cec84@10.31.136.152@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.078055:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693115112608704, offset 192 00000400:00000200:16.0:1616513083.078062:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.152@tcp1 00000400:00000200:16.0:1616513083.078069:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.152@tcp1 routed destination 00000400:00000200:16.0:1616513083.078084:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.152@tcp1(10.31.136.152@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.078095:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.078101:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.078104:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.078106:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.078111:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.078116:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.078118:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513083.078170:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.078177:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513083.078180:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513083.078187:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.078192:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdedd0 00000800:00000200:14.0:1616513083.078196:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.078199:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.079273:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.079336:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.079345:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.079354:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.152@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.079363:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.152@tcp1 of length 224 into portal 12 MB=0x603e114d9d500 00000400:00000200:20.0:1616513083.079370:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.152@tcp1 of length 224/224 into md 0x865bbec1 [1] + 57680 00000400:00000200:20.0:1616513083.079375:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.079378:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513083.079384:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.079387:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.079389:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.079392:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513083.079473:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115112609024 00010000:00000200:4.0:1616513083.079486:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cea880 x1693115112609024/t0(0) o400->cb305ad4-579e-e2db-0f02-30336b571e6b@10.31.136.152@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513083.079501:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693115112609024, offset 192 00000400:00000200:4.0:1616513083.079507:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.152@tcp1 00000400:00000200:4.0:1616513083.079514:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.152@tcp1 routed destination 00000400:00000200:4.0:1616513083.079529:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.152@tcp1(10.31.136.152@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513083.079536:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513083.079545:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513083.079548:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513083.079550:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513083.079555:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.079559:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.079614:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.079621:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.079624:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.079630:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.079634:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baee0 00000800:00000200:28.0:1616513083.079638:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.079641:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513083.114676:0:6487:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926176, portal 4 00000100:00000200:20.0:1616513083.114681:0:6487:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926176, offset 0 00000400:00000200:20.0:1616513083.114688:0:6487:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:20.0:1616513083.114695:0:6487:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:20.0:1616513083.114708:0:6487:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513083.114715:0:6487:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:20.0:1616513083.114721:0:6487:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513083.114726:0:6487:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513083.114729:0:6487:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:7.2:1616513083.114730:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513083.114734:0:6487:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513083.114779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:26.0:1616513083.114786:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513083.114794:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:26.0:1616513083.114797:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513083.114800:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513083.114802:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.114806:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.151@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.114808:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513083.114813:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cbeede300 x1693067725926176/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513083.114816:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.151@tcp1 of length 224 into portal 26 MB=0x5ffd53eda0ec0 00000400:00000200:26.0:1616513083.114823:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.151@tcp1 of length 224/224 into md 0xf97ca279 [1] + 0 00000400:00000200:20.0:1616513083.114825:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bba0 00000400:00000200:26.0:1616513083.114828:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42000 00000800:00000200:20.0:1616513083.114828:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513083.114831:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000400:00000200:26.0:1616513083.114832:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.114835:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:11.2:1616513083.114839:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513083.114840:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42000 00000800:00000200:22.0:1616513083.114841:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.114844:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.114847:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.114849:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513083.114854:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.114862:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.114863:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513083.114872:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.151@tcp1 : PUT - for me 00000400:00000200:28.0:1616513083.114881:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.151@tcp1 of length 224 into portal 12 MB=0x5ffd53eda0fc0 00000400:00000200:28.0:1616513083.114887:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.151@tcp1 of length 224/224 into md 0x865bbec1 [1] + 57904 00000400:00000200:28.0:1616513083.114893:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.114896:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513083.114902:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.114905:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.114907:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.114908:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666231148224 00010000:00000200:16.0:1616513083.114919:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688666231148224/t0(0) o400->7c7038af-b1e1-d0b6-0c46-f883039dda8f@10.31.132.151@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513083.114925:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666231148480 00000100:00000200:16.0:1616513083.114933:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666231148224, offset 192 00000400:00000200:16.0:1616513083.114939:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.151@tcp1 00010000:00000200:4.0:1616513083.114939:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedb600 x1688666231148480/t0(0) o400->788725c2-e25a-954a-6503-c7e1acfc221b@10.31.132.151@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513083.114946:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.151@tcp1 routed destination 00000800:00000200:5.2:1616513083.114954:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000100:00000200:4.0:1616513083.114954:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666231148480, offset 192 00000400:00000200:16.0:1616513083.114960:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.151@tcp1(10.31.132.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:4.0:1616513083.114960:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.151@tcp1 00000800:00000200:20.0:1616513083.114963:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513083.114966:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:4.0:1616513083.114967:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.151@tcp1 routed destination 00000800:00000200:20.0:1616513083.114969:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:0.0:1616513083.114972:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:4.0:1616513083.114983:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.151@tcp1(10.31.132.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.114990:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513083.114990:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.114992:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:16.0:1616513083.114993:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.114995:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513083.114996:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513083.114998:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626f20 00000800:00000200:16.0:1616513083.115000:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513083.115000:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513083.115003:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513083.115004:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d7351 [1] + 192 00000800:00000200:7.2:1616513083.115005:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.115007:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513083.115007:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.115009:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:11.2:1616513083.115012:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:20.0:1616513083.115013:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cbeede300 x1693067725926176/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513083.115026:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513083.115029:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513083.115031:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:28.0:1616513083.115033:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.115038:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:24.0:1616513083.115039:0:6487:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cbeede300 x1693067725926176/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513083.115040:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.115042:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.115046:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:24.0:1616513083.115053:0:6487:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cbeede300 x1693067725926176/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513083.115059:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513083.115064:0:6487:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79b8f8 00000400:00000200:28.0:1616513083.115067:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bad48 00000800:00000200:14.0:1616513083.115067:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513083.115070:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513083.115070:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513083.115077:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.115082:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde2a8 00000800:00000200:14.0:1616513083.115088:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.115091:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.117546:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513083.117598:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513083.117607:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513083.117616:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.134@tcp1 : PUT - for me 00000400:00000200:22.0:1616513083.117625:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.134@tcp1 of length 224 into portal 12 MB=0x6042a8acf3e00 00000400:00000200:22.0:1616513083.117632:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.134@tcp1 of length 224/224 into md 0xf964d519 [1] + 27432 00000400:00000200:22.0:1616513083.117637:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513083.117640:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513083.117647:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513083.117650:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513083.117652:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513083.117656:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.117718:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693430624239104 00010000:00000200:18.0:1616513083.117730:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2f500 x1693430624239104/t0(0) o400->eea839f3-1e11-dac8-7b23-bbec7553467e@10.31.129.134@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513083.117730:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:18.0:1616513083.117744:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693430624239104, offset 192 00000400:00000200:18.0:1616513083.117751:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.134@tcp1 00000400:00000200:18.0:1616513083.117758:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.134@tcp1 routed destination 00000400:00000200:18.0:1616513083.117775:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.134@tcp1(10.31.129.134@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.117781:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513083.117785:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.117787:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.117790:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513083.117792:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513083.117794:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.117797:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.117801:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513083.117805:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.134@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.117814:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.134@tcp1 of length 224 into portal 26 MB=0x6042a8acf3ac0 00000400:00000200:0.0:1616513083.117820:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.134@tcp1 of length 224/224 into md 0xf97ca299 [1] + 0 00000400:00000200:0.0:1616513083.117825:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a425d8 00000400:00000200:0.0:1616513083.117827:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.117830:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.117834:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a425d8 00000800:00000200:0.0:1616513083.117837:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.117838:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.117840:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.117842:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513083.117861:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.117868:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513083.117870:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513083.117877:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.117881:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc330 00000800:00000200:14.0:1616513083.117885:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.117888:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.117902:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693430624238272 00010000:00000200:16.0:1616513083.117912:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1693430624238272/t0(0) o400->da603688-da6b-543a-45e4-f841c2f88df4@10.31.129.134@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.117934:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693430624238272, offset 192 00000400:00000200:16.0:1616513083.117940:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.134@tcp1 00000400:00000200:16.0:1616513083.117947:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.134@tcp1 routed destination 00000400:00000200:16.0:1616513083.117962:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.134@tcp1(10.31.129.134@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.117969:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.117974:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.117977:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.117981:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.117986:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.117990:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513083.117993:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513083.118001:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.118004:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513083.118006:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513083.118012:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.118017:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf5c8 00000800:00000200:0.0:1616513083.118019:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.118022:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.143250:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.143306:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.143317:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.143326:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.18@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.143336:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.18@tcp1 of length 224 into portal 26 MB=0x60194fefa31c0 00000400:00000200:26.0:1616513083.143342:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.18@tcp1 of length 224/224 into md 0xf97ca2a9 [1] + 0 00000400:00000200:26.0:1616513083.143347:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43870 00000400:00000200:26.0:1616513083.143350:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.143353:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.143358:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43870 00000800:00000200:22.0:1616513083.143360:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.143361:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.143364:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.143366:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.143424:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589304861120 00010000:00000200:16.0:1616513083.143449:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1690589304861120/t0(0) o400->557ca56f-d434-8193-4b79-e08f2da4ab61@10.31.129.18@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513083.143459:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513083.143464:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690589304861120, offset 192 00000400:00000200:16.0:1616513083.143470:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.18@tcp1 00000400:00000200:16.0:1616513083.143478:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.18@tcp1 routed destination 00000400:00000200:16.0:1616513083.143494:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.18@tcp1(10.31.129.18@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.143500:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.143507:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.143511:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.143513:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.143515:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:16.0:1616513083.143518:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:16.0:1616513083.143525:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513083.143525:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513083.143534:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.18@tcp1 : PUT - for me 00000400:00000200:14.0:1616513083.143543:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.18@tcp1 of length 224 into portal 12 MB=0x60194fefa3580 00000400:00000200:14.0:1616513083.143549:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.18@tcp1 of length 224/224 into md 0xf964d519 [1] + 27656 00000400:00000200:14.0:1616513083.143555:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.143558:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513083.143565:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513083.143569:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513083.143571:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513083.143578:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513083.143579:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513083.143581:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.143582:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.143589:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:18.0:1616513083.143590:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589304862080 00000400:00000200:2.0:1616513083.143594:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde330 00000800:00000200:2.0:1616513083.143598:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:18.0:1616513083.143601:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623f00 x1690589304862080/t0(0) o400->19410a33-191f-ad7f-07bf-82e87d1c766c@10.31.129.18@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.143615:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690589304862080, offset 192 00000400:00000200:18.0:1616513083.143621:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.18@tcp1 00000400:00000200:18.0:1616513083.143628:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.18@tcp1 routed destination 00000400:00000200:18.0:1616513083.143642:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.18@tcp1(10.31.129.18@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.143648:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.143653:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.143656:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.143658:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.143662:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.143671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.143727:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.143734:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.143737:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.143743:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.143744:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.143748:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cce58 00000800:00000200:20.0:1616513083.143752:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.151129:0:49779:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.151184:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.151194:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.151203:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.153@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.151214:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.153@tcp1 of length 224 into portal 26 MB=0x5ffd4ab9364c0 00000400:00000200:26.0:1616513083.151220:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.153@tcp1 of length 224/224 into md 0xf97ca2b9 [1] + 0 00000400:00000200:26.0:1616513083.151225:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba330 00000400:00000200:26.0:1616513083.151228:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.151231:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.151235:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba330 00000800:00000200:14.0:1616513083.151238:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.151239:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.151242:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.151243:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.151301:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663760266432 00010000:00000200:16.0:1616513083.151312:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688663760266432/t0(0) o400->a5ae12ab-6a24-9b5f-4a9f-9ddbb6e13634@10.31.132.153@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.151326:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663760266432, offset 192 00000400:00000200:16.0:1616513083.151332:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.153@tcp1 00000400:00000200:16.0:1616513083.151340:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.153@tcp1 routed destination 00000400:00000200:16.0:1616513083.151355:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.153@tcp1(10.31.132.153@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.151362:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.151368:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.151371:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.151373:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.151378:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.151383:0:49779:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.151384:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513083.151449:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513083.151456:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513083.151459:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513083.151465:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513083.151471:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfa90 00000800:00000200:22.0:1616513083.151475:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.151477:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513083.151484:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513083.151485:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513083.151492:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.153@tcp1 : PUT - for me 00000400:00000200:22.0:1616513083.151500:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.153@tcp1 of length 224 into portal 12 MB=0x5ffd4ab9365c0 00000400:00000200:22.0:1616513083.151507:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.153@tcp1 of length 224/224 into md 0xf964d519 [1] + 27880 00000400:00000200:22.0:1616513083.151512:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513083.151515:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513083.151521:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513083.151523:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.151525:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.151587:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663760266688 00010000:00000200:18.0:1616513083.151599:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2ba80 x1688663760266688/t0(0) o400->aa608938-a473-a0d4-004d-1dff81da4959@10.31.132.153@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.151613:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663760266688, offset 192 00000400:00000200:18.0:1616513083.151620:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.153@tcp1 00000400:00000200:18.0:1616513083.151627:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.153@tcp1 routed destination 00000400:00000200:18.0:1616513083.151642:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.153@tcp1(10.31.132.153@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.151649:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.151655:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.151658:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.151662:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.151667:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.151674:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.151728:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.151734:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513083.151737:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513083.151743:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.151746:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccb28 00000800:00000200:0.0:1616513083.151750:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.151753:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.167101:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.167156:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.167165:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.167176:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.122@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.167186:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.122@tcp1 of length 224 into portal 26 MB=0x6018326ae9440 00000400:00000200:26.0:1616513083.167193:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.122@tcp1 of length 224/224 into md 0xf97ca2c9 [1] + 0 00000400:00000200:26.0:1616513083.167197:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a433a8 00000400:00000200:26.0:1616513083.167200:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.167203:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.167207:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a433a8 00000800:00000200:14.0:1616513083.167207:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.167210:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.167213:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.167215:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.167271:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512661582912 00010000:00000200:16.0:1616513083.167283:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1690512661582912/t0(0) o400->d66078e0-6441-426c-e4cd-280995bb76dd@10.31.136.122@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.167299:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512661582912, offset 192 00000400:00000200:16.0:1616513083.167305:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.122@tcp1 00000400:00000200:16.0:1616513083.167313:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.122@tcp1 routed destination 00000400:00000200:16.0:1616513083.167329:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.122@tcp1(10.31.136.122@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.167335:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.167341:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.167344:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.167346:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.167351:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.167357:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.167358:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513083.167411:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.167418:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.167421:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.167427:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.167453:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde000 00000800:00000200:2.0:1616513083.167456:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.167459:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.167607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.167661:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.167670:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.167678:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.122@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.167686:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.122@tcp1 of length 224 into portal 12 MB=0x6018326ae95c0 00000400:00000200:0.0:1616513083.167692:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.122@tcp1 of length 224/224 into md 0x865bbec1 [1] + 58128 00000400:00000200:0.0:1616513083.167697:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.167700:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.167708:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.167712:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.167714:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.167717:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513083.167727:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512661583296 00010000:00000200:8.0:1616513083.167738:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86f500 x1690512661583296/t0(0) o400->3aa258c7-5600-5a4f-327e-096fec7233aa@10.31.136.122@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513083.167752:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512661583296, offset 192 00000400:00000200:8.0:1616513083.167758:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.122@tcp1 00000400:00000200:8.0:1616513083.167765:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.122@tcp1 routed destination 00000400:00000200:8.0:1616513083.167779:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.122@tcp1(10.31.136.122@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.167785:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513083.167791:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513083.167794:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513083.167796:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513083.167801:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.167805:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.167816:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.167819:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513083.167821:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513083.167826:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.167829:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71320 00000800:00000200:0.0:1616513083.167833:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.167835:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.171562:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.171617:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.171627:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513083.171630:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.171636:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.126@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.171652:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.126@tcp1 of length 224 into portal 26 MB=0x603fc13246ec0 00000400:00000200:20.0:1616513083.171658:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.126@tcp1 of length 224/224 into md 0xf97ca2d9 [1] + 0 00000400:00000200:20.0:1616513083.171663:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42aa0 00000400:00000200:20.0:1616513083.171666:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.171668:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513083.171673:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42aa0 00000800:00000200:20.0:1616513083.171676:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.171678:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.171680:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.171697:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693231048060608 00010000:00000200:16.0:1616513083.171707:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1693231048060608/t0(0) o400->2b2e661c-ac06-c7d3-456e-a09993f39016@10.31.138.126@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.171720:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693231048060608, offset 192 00000400:00000200:16.0:1616513083.171726:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.126@tcp1 00000400:00000200:16.0:1616513083.171733:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.126@tcp1 routed destination 00000800:00000200:7.2:1616513083.171735:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513083.171747:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.126@tcp1(10.31.138.126@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.171753:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.171759:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.171762:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.171764:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513083.171768:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:16.0:1616513083.171774:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513083.171790:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.171799:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513083.171807:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.126@tcp1 : PUT - for me 00000400:00000200:14.0:1616513083.171815:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.126@tcp1 of length 224 into portal 12 MB=0x603fc13246f40 00000400:00000200:14.0:1616513083.171821:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.126@tcp1 of length 224/224 into md 0xf964d519 [1] + 28104 00000400:00000200:14.0:1616513083.171826:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.171828:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513083.171835:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513083.171838:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513083.171840:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513083.171844:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.171847:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513083.171849:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513083.171854:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.171858:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf650 00000100:00000200:18.0:1616513083.171860:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693231048060736 00000800:00000200:14.0:1616513083.171861:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.171863:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:18.0:1616513083.171872:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464626300 x1693231048060736/t0(0) o400->deaa864d-154a-8abc-71d1-4b57d896b9a5@10.31.138.126@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.171885:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693231048060736, offset 192 00000400:00000200:18.0:1616513083.171891:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.126@tcp1 00000400:00000200:18.0:1616513083.171898:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.126@tcp1 routed destination 00000400:00000200:18.0:1616513083.171912:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.126@tcp1(10.31.138.126@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.171918:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.171923:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.171925:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.171927:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.171932:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.171936:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.171946:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.171950:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513083.171952:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513083.171959:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513083.171959:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.171962:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd430 00000800:00000200:0.0:1616513083.171966:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.181863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.181919:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.181928:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.181936:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.8@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.181945:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.8@tcp1 of length 224 into portal 26 MB=0x5ffdc899ab900 00000400:00000200:0.0:1616513083.181951:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.8@tcp1 of length 224/224 into md 0xf97ca2e9 [1] + 0 00000400:00000200:0.0:1616513083.181955:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42b28 00000400:00000200:0.0:1616513083.181957:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.181960:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.181964:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42b28 00000800:00000200:0.0:1616513083.181967:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.181969:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.181971:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.181975:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.182026:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697550059776 00010000:00000200:16.0:1616513083.182037:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688697550059776/t0(0) o400->934f07c1-7378-a6e3-74a9-0c75264dba0f@10.31.136.8@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.182050:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697550059776, offset 192 00000400:00000200:16.0:1616513083.182056:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.8@tcp1 00000400:00000200:16.0:1616513083.182063:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.8@tcp1 routed destination 00000400:00000200:16.0:1616513083.182078:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.8@tcp1(10.31.136.8@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.182085:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.182090:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.182092:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.182094:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.182099:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513083.182105:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.182111:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513083.182166:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513083.182173:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513083.182175:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513083.182182:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513083.182189:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde880 00000800:00000200:22.0:1616513083.182193:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.182196:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.182266:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513083.182319:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.182329:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513083.182330:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.182337:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.8@tcp1 : PUT - for me 00000400:00000200:28.0:1616513083.182345:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.8@tcp1 of length 224 into portal 12 MB=0x5ffdc899aba00 00000400:00000200:28.0:1616513083.182352:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.8@tcp1 of length 224/224 into md 0x865bbec1 [1] + 58352 00000400:00000200:28.0:1616513083.182357:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.182360:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513083.182377:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.182381:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.182383:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513083.182429:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697550060032 00010000:00000200:8.0:1616513083.182451:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeede780 x1688697550060032/t0(0) o400->2da10a7b-3942-7dc5-3e56-9fedcb78df5a@10.31.136.8@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513083.182464:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697550060032, offset 192 00000400:00000200:8.0:1616513083.182470:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.8@tcp1 00000400:00000200:8.0:1616513083.182476:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.8@tcp1 routed destination 00000400:00000200:8.0:1616513083.182490:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.8@tcp1(10.31.136.8@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.182496:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513083.182502:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513083.182513:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513083.182515:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513083.182519:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.182524:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.182535:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.182540:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.182542:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.182543:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.182549:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.182553:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71ed0 00000800:00000200:20.0:1616513083.182557:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.196788:0:49784:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513083.196844:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.196852:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513083.196859:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.159@tcp1 : PUT - for me 00000400:00000200:14.0:1616513083.196867:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.159@tcp1 of length 224 into portal 26 MB=0x5ffd83d9934c0 00000400:00000200:14.0:1616513083.196873:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.159@tcp1 of length 224/224 into md 0xf97ca2f9 [1] + 0 00000400:00000200:14.0:1616513083.196877:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a422a8 00000400:00000200:14.0:1616513083.196879:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.196881:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513083.196885:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a422a8 00000800:00000200:14.0:1616513083.196888:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.196890:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513083.196891:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.196894:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.196946:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688679095022784 00010000:00000200:16.0:1616513083.196956:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688679095022784/t0(0) o400->640978e9-0cdd-35c1-8f0c-e1bce5060452@10.31.129.159@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.196968:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688679095022784, offset 192 00000400:00000200:16.0:1616513083.196973:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.159@tcp1 00000400:00000200:16.0:1616513083.196980:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.159@tcp1 routed destination 00000400:00000200:16.0:1616513083.196993:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.159@tcp1(10.31.129.159@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.196998:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.197003:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.197006:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.197008:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.197012:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.197016:0:49784:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.197017:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513083.197039:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513083.197070:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.197077:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.197080:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.197086:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.197092:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfed0 00000800:00000200:0.0:1616513083.197094:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513083.197095:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.197098:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513083.197102:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.197110:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.159@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.197117:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.159@tcp1 of length 224 into portal 12 MB=0x5ffd83d9938c0 00000400:00000200:0.0:1616513083.197124:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.159@tcp1 of length 224/224 into md 0x865bbec1 [1] + 58576 00000400:00000200:0.0:1616513083.197129:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.197132:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.197137:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.197140:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.197142:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.197144:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513083.197155:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688679095023808 00010000:00000200:8.0:1616513083.197166:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86ad00 x1688679095023808/t0(0) o400->8a509f05-69af-e197-d4d6-5540985c4f12@10.31.129.159@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513083.197178:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688679095023808, offset 192 00000400:00000200:8.0:1616513083.197184:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.159@tcp1 00000400:00000200:8.0:1616513083.197190:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.159@tcp1 routed destination 00000400:00000200:8.0:1616513083.197203:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.159@tcp1(10.31.129.159@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.197209:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513083.197214:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513083.197216:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513083.197218:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513083.197223:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.197228:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.197237:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.197240:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513083.197242:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513083.197246:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513083.197247:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.197250:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71650 00000800:00000200:0.0:1616513083.197253:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.225782:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.225839:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.225849:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.225858:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.101@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.225868:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.101@tcp1 of length 224 into portal 12 MB=0x603e23dd8e0c0 00000400:00000200:0.0:1616513083.225874:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.101@tcp1 of length 224/224 into md 0x865bbec1 [1] + 58800 00000400:00000200:0.0:1616513083.225880:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.225882:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.225889:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.225892:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.225894:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.225898:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513083.225910:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693120095379648 00010000:00000200:8.0:1616513083.225923:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86de80 x1693120095379648/t0(0) o400->8caca2c3-fd93-799c-6895-9d574cd32bbe@10.31.138.101@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513083.225946:0:49787:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:8.0:1616513083.225947:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693120095379648, offset 192 00000400:00000200:8.0:1616513083.225953:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.101@tcp1 00000800:00000200:14.0:1616513083.225960:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513083.225961:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.101@tcp1 routed destination 00000800:00000200:14.0:1616513083.225969:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:8.0:1616513083.225976:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.101@tcp1(10.31.138.101@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.225983:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513083.225986:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.101@tcp1 : PUT - for me 00000800:00000200:8.0:1616513083.225990:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513083.225993:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513083.225995:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.101@tcp1 of length 224 into portal 26 MB=0x603e23dd8de00 00000800:00000200:8.0:1616513083.225995:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:8.0:1616513083.226000:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513083.226002:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.101@tcp1 of length 224/224 into md 0xf97ca309 [1] + 0 00000400:00000200:14.0:1616513083.226007:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42990 00000400:00000200:14.0:1616513083.226009:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.226012:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513083.226013:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513083.226016:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42990 00000800:00000200:14.0:1616513083.226020:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513083.226022:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:14.0:1616513083.226023:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:14.0:1616513083.226024:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.226025:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513083.226027:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513083.226032:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.226036:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f717e8 00000800:00000200:2.0:1616513083.226040:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.226083:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693120095378944 00010000:00000200:16.0:1616513083.226094:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1693120095378944/t0(0) o400->433f0235-5bc3-4163-30c4-48b90f7871dd@10.31.138.101@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.226108:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693120095378944, offset 192 00000400:00000200:16.0:1616513083.226115:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.101@tcp1 00000400:00000200:16.0:1616513083.226123:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.101@tcp1 routed destination 00000400:00000200:16.0:1616513083.226138:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.101@tcp1(10.31.138.101@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.226144:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.226150:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.226154:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.226157:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.226162:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.226167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513083.226169:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513083.226178:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.226186:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.226188:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.226190:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.226195:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.226201:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdebb0 00000800:00000200:20.0:1616513083.226204:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.270194:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513083.270251:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513083.270260:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513083.270269:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.130@tcp1 : PUT - for me 00000400:00000200:22.0:1616513083.270279:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.130@tcp1 of length 224 into portal 26 MB=0x604b7e1a2e8c0 00000400:00000200:22.0:1616513083.270286:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.130@tcp1 of length 224/224 into md 0xf97ca329 [1] + 0 00000400:00000200:22.0:1616513083.270291:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43a90 00000400:00000200:22.0:1616513083.270294:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513083.270296:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513083.270301:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a90 00000800:00000200:14.0:1616513083.270303:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.270304:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513083.270307:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.270308:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.270366:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694037671340224 00010000:00000200:16.0:1616513083.270378:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1694037671340224/t0(0) o400->96adbe40-5472-cd9b-6952-244e94b9861f@10.31.140.130@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.270392:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694037671340224, offset 192 00000400:00000200:16.0:1616513083.270398:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.130@tcp1 00000400:00000200:16.0:1616513083.270406:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.130@tcp1 routed destination 00000400:00000200:16.0:1616513083.270421:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.130@tcp1(10.31.140.130@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.270428:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.270434:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.270437:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.270451:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.270456:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513083.270464:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.270468:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.270521:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.270528:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.270531:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.270537:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.270543:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde880 00000800:00000200:2.0:1616513083.270546:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.270549:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.270556:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513083.270558:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513083.270563:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.130@tcp1 : PUT - for me 00000400:00000200:2.0:1616513083.270571:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.130@tcp1 of length 224 into portal 12 MB=0x604b7e1a2ea80 00000400:00000200:2.0:1616513083.270578:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.130@tcp1 of length 224/224 into md 0xf964d519 [1] + 28328 00000400:00000200:2.0:1616513083.270583:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513083.270585:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513083.270592:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.270594:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.270596:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.270656:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694037671340672 00010000:00000200:18.0:1616513083.270668:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d3180 x1694037671340672/t0(0) o400->f2eaa600-2835-7d2f-9ce7-436d9ac30922@10.31.140.130@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.270681:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694037671340672, offset 192 00000400:00000200:18.0:1616513083.270687:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.130@tcp1 00000400:00000200:18.0:1616513083.270694:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.130@tcp1 routed destination 00000400:00000200:18.0:1616513083.270708:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.130@tcp1(10.31.140.130@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.270714:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.270719:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.270721:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.270723:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.270728:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.270773:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.270830:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.270837:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.270840:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.270847:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.270851:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cde48 00000800:00000200:28.0:1616513083.270855:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.270858:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.277141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.277198:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.277207:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.277216:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.48@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.277225:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.48@tcp1 of length 224 into portal 26 MB=0x6018528622b40 00000400:00000200:0.0:1616513083.277232:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.48@tcp1 of length 224/224 into md 0xf97ca339 [1] + 0 00000400:00000200:0.0:1616513083.277237:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43650 00000400:00000200:0.0:1616513083.277239:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.277242:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.277249:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43650 00000800:00000200:20.0:1616513083.277252:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.277252:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.277255:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.277257:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.277312:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690521280064320 00010000:00000200:16.0:1616513083.277322:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1690521280064320/t0(0) o400->820cb84c-48dc-0cd1-46a3-eb15c528f396@10.31.141.48@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.277337:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690521280064320, offset 192 00000400:00000200:16.0:1616513083.277343:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.48@tcp1 00000400:00000200:16.0:1616513083.277350:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.48@tcp1 routed destination 00000400:00000200:16.0:1616513083.277365:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.48@tcp1(10.31.141.48@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.277372:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.277377:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.277380:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.277382:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.277387:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513083.277390:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513083.277394:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.277395:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513083.277407:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513083.277409:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513083.277415:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513083.277416:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513083.277418:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.277418:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.277425:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.48@tcp1 : PUT - for me 00000400:00000200:22.0:1616513083.277425:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513083.277430:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde000 00000400:00000200:28.0:1616513083.277446:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.48@tcp1 of length 224 into portal 12 MB=0x6018528622f40 00000800:00000200:22.0:1616513083.277447:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.277450:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513083.277452:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.48@tcp1 of length 224/224 into md 0x865bbec1 [1] + 59024 00000400:00000200:28.0:1616513083.277458:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.277460:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513083.277467:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.277471:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.277473:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513083.277484:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690521280065344 00010000:00000200:8.0:1616513083.277494:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed9f80 x1690521280065344/t0(0) o400->1b8c2160-1326-7b84-0fbb-3e0ff2e14b43@10.31.141.48@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513083.277506:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690521280065344, offset 192 00000400:00000200:8.0:1616513083.277512:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.48@tcp1 00000400:00000200:8.0:1616513083.277518:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.48@tcp1 routed destination 00000400:00000200:8.0:1616513083.277531:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.48@tcp1(10.31.141.48@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.277536:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513083.277541:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513083.277543:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513083.277545:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513083.277549:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.277555:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.277566:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.277572:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.277574:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.277575:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.277581:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.277585:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71298 00000800:00000200:20.0:1616513083.277589:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.280733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.280792:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.280802:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.280811:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.181@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.280819:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.181@tcp1 of length 224 into portal 26 MB=0x6059f1028b140 00000400:00000200:0.0:1616513083.280826:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.181@tcp1 of length 224/224 into md 0xf97ca359 [1] + 0 00000400:00000200:0.0:1616513083.280831:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a437e8 00000400:00000200:0.0:1616513083.280833:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.280836:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.280840:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a437e8 00000800:00000200:0.0:1616513083.280843:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.280846:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.280847:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513083.280855:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.280858:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513083.280858:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.280867:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.181@tcp1 : PUT - for me 00000400:00000200:28.0:1616513083.280876:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.181@tcp1 of length 224 into portal 12 MB=0x6059f1028b1c0 00000400:00000200:28.0:1616513083.280882:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.181@tcp1 of length 224/224 into md 0x865bbec1 [1] + 59248 00000400:00000200:28.0:1616513083.280887:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.280890:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513083.280897:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.280900:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.280902:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.280904:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695030589305152 00000100:00000200:8.0:1616513083.280907:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695030589305280 00010000:00000200:8.0:1616513083.280913:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeeda880 x1695030589305280/t0(0) o400->06bf3ac7-50fe-d7ad-2436-cd654c6cde30@10.31.129.181@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:16.0:1616513083.280915:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1695030589305152/t0(0) o400->48e91b2b-db3c-7189-75ce-229e44c54395@10.31.129.181@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513083.280924:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695030589305280, offset 192 00000400:00000200:8.0:1616513083.280930:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.181@tcp1 00000100:00000200:16.0:1616513083.280932:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695030589305152, offset 192 00000400:00000200:8.0:1616513083.280936:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.181@tcp1 routed destination 00000400:00000200:16.0:1616513083.280939:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.181@tcp1 00000400:00000200:16.0:1616513083.280946:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.181@tcp1 routed destination 00000400:00000200:8.0:1616513083.280949:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.181@tcp1(10.31.129.181@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.280956:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513083.280961:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.181@tcp1(10.31.129.181@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.280967:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.280968:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513083.280970:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513083.280971:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.280974:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513083.280975:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513083.280976:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.280978:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513083.280981:0:49793:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.280983:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.280988:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513083.280989:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513083.280999:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.281007:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.281009:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.281011:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.281016:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.281023:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde330 00000800:00000200:20.0:1616513083.281027:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513083.281036:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.281042:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.281045:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.281052:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.281056:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70cc0 00000800:00000200:26.0:1616513083.281060:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.281063:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.291205:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513083.291211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513083.291261:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.291271:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513083.291279:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513083.291281:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.107@tcp1 : PUT - for me 00000800:00000200:22.0:1616513083.291289:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513083.291291:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.107@tcp1 of length 224 into portal 26 MB=0x5ffdd7bdf8440 00000800:00000200:14.0:1616513083.291292:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513083.291298:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.107@tcp1 of length 224/224 into md 0xf97ca379 [1] + 0 00000400:00000200:22.0:1616513083.291298:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.107@tcp1 : PUT - for me 00000400:00000200:28.0:1616513083.291302:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42660 00000400:00000200:28.0:1616513083.291305:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513083.291307:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.107@tcp1 of length 224 into portal 12 MB=0x5ffdd7bdf8840 00000100:00000200:28.0:1616513083.291308:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513083.291312:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42660 00000800:00000200:28.0:1616513083.291315:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513083.291315:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513083.291317:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.107@tcp1 of length 224/224 into md 0xf964d519 [1] + 28552 00000800:00000200:28.0:1616513083.291318:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.291320:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513083.291322:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513083.291325:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513083.291331:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513083.291334:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513083.291336:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.291376:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701614654528 00010000:00000200:16.0:1616513083.291386:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688701614654528/t0(0) o400->025c27fe-c2ec-d05d-47cc-2883724ad557@10.31.141.107@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513083.291400:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701614655552 00000100:00000200:16.0:1616513083.291401:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701614654528, offset 192 00000400:00000200:16.0:1616513083.291407:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.107@tcp1 00010000:00000200:18.0:1616513083.291412:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2d580 x1688701614655552/t0(0) o400->3e3155be-f371-ad88-cb22-9661d76ef73e@10.31.141.107@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.291426:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701614655552, offset 192 00000400:00000200:16.0:1616513083.291427:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.107@tcp1 routed destination 00000400:00000200:18.0:1616513083.291448:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.107@tcp1 00000400:00000200:16.0:1616513083.291454:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.107@tcp1(10.31.141.107@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.291460:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513083.291462:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.107@tcp1 routed destination 00000800:00000200:16.0:1616513083.291466:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.291468:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.291470:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.291475:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513083.291478:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.107@tcp1(10.31.141.107@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513083.291481:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.291481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513083.291484:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.291490:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.291493:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.291495:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.291499:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.291510:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.291532:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513083.291534:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513083.291538:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513083.291541:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:0.0:1616513083.291541:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.291543:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513083.291547:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.291550:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.291551:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd210 00000800:00000200:0.0:1616513083.291555:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513083.291556:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde2a8 00000800:00000200:0.0:1616513083.291557:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513083.291559:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.291562:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.360140:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513083.360197:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513083.360208:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513083.360214:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513083.360217:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.138@tcp1 : PUT - for me 00000400:00000200:4.0:1616513083.360227:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.138@tcp1 of length 224 into portal 12 MB=0x6039422ead080 00000400:00000200:4.0:1616513083.360233:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.138@tcp1 of length 224/224 into md 0x865bbec1 [1] + 59472 00000400:00000200:4.0:1616513083.360239:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513083.360242:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513083.360249:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513083.360252:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513083.360254:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513083.360269:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692784636121216 00010000:00000200:8.0:1616513083.360293:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb239f80 x1692784636121216/t0(0) o400->94a117c8-380b-1110-0669-bbf8bd9c2d32@10.31.140.138@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513083.360306:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692784636121216, offset 192 00000400:00000200:8.0:1616513083.360312:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.138@tcp1 00000400:00000200:8.0:1616513083.360320:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.138@tcp1 routed destination 00000400:00000200:8.0:1616513083.360334:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.138@tcp1(10.31.140.138@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.360340:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513083.360346:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513083.360348:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513083.360350:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513083.360356:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.360364:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.360420:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.360427:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513083.360430:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513083.360449:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.360453:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70a18 00000800:00000200:14.0:1616513083.360457:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.360460:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.361921:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.361981:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.361991:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.362000:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.138@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.362011:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.138@tcp1 of length 224 into portal 26 MB=0x6039422eacf00 00000400:00000200:26.0:1616513083.362018:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.138@tcp1 of length 224/224 into md 0xf97ca399 [1] + 0 00000400:00000200:26.0:1616513083.362022:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43b18 00000400:00000200:26.0:1616513083.362025:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.362028:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.362032:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43b18 00000800:00000200:26.0:1616513083.362035:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.362035:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.362038:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513083.362040:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.362098:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692784636120832 00010000:00000200:16.0:1616513083.362109:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1692784636120832/t0(0) o400->85296513-79fc-81bc-80e0-e6c5ed7988cf@10.31.140.138@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.362124:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692784636120832, offset 192 00000400:00000200:16.0:1616513083.362132:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.138@tcp1 00000400:00000200:16.0:1616513083.362140:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.138@tcp1 routed destination 00000400:00000200:16.0:1616513083.362154:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.138@tcp1(10.31.140.138@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.362161:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.362167:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.362169:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.362172:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.362177:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513083.362183:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513083.362223:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.362277:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.362283:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513083.362285:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513083.362291:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.362296:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fded48 00000800:00000200:0.0:1616513083.362299:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.362302:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513083.370559:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926192, portal 4 00000100:00000200:30.0:1616513083.370564:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926192, offset 0 00000400:00000200:30.0:1616513083.370570:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513083.370575:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513083.370587:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513083.370592:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513083.370597:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513083.370600:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513083.370602:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513083.370606:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513083.370653:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513083.370708:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513083.370715:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513083.370718:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.370724:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513083.370731:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c8a2cc80 x1693067725926192/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513083.370743:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee220 00000800:00000200:28.0:1616513083.370747:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513083.370750:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513083.370779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:0.0:1616513083.370833:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513083.370844:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:0.0:1616513083.370853:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:0.0:1616513083.370861:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626f30 00000400:00000200:0.0:1616513083.370868:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d74d1 [1] + 192 00000400:00000200:0.0:1616513083.370873:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.370879:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c8a2cc80 x1693067725926192/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513083.370889:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513083.370898:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513083.370901:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513083.370903:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513083.370904:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c8a2cc80 x1693067725926192/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513083.370915:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c8a2cc80 x1693067725926192/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513083.370924:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eecc0 00000100:00000200:23.0:1616513083.370929:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926208, portal 4 00000100:00000200:23.0:1616513083.370935:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926208, offset 0 00000400:00000200:23.0:1616513083.370943:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:23.0:1616513083.370950:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:23.0:1616513083.370964:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513083.370971:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:23.0:1616513083.370979:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513083.370982:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:23.0:1616513083.370985:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:23.0:1616513083.370991:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513083.370996:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513083.371006:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513083.371010:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513083.371013:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513083.371015:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513083.371018:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513083.371023:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be43600 x1693067725926208/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513083.371033:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3edd0 00000800:00000200:28.0:1616513083.371036:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513083.371051:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513083.371062:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513083.371070:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:0.0:1616513083.371072:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:20.0:1616513083.371079:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513083.371087:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626f40 00000400:00000200:20.0:1616513083.371093:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d74e1 [1] + 192 00000400:00000200:20.0:1616513083.371098:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.371104:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be43600 x1693067725926208/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513083.371119:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513083.371121:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513083.371123:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:23.0:1616513083.371129:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be43600 x1693067725926208/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513083.371142:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be43600 x1693067725926208/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513083.371153:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eee0 00000800:00000200:7.2:1616513083.377548:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513083.377562:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.377572:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513083.377578:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513083.377581:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.73@tcp1 : PUT - for me 00000400:00000200:14.0:1616513083.377590:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.73@tcp1 of length 224 into portal 26 MB=0x60217c92cac80 00000400:00000200:14.0:1616513083.377597:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.73@tcp1 of length 224/224 into md 0xf97ca3a9 [1] + 0 00000400:00000200:14.0:1616513083.377601:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42f68 00000400:00000200:14.0:1616513083.377604:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.377607:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513083.377611:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42f68 00000800:00000200:14.0:1616513083.377614:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.377617:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.377619:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.377675:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691151042915456 00010000:00000200:16.0:1616513083.377687:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1691151042915456/t0(0) o400->99cb6009-6bf4-0c1b-cb72-f3cf55230b0d@10.31.141.73@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.377702:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691151042915456, offset 192 00000400:00000200:16.0:1616513083.377708:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.73@tcp1 00000400:00000200:16.0:1616513083.377716:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.73@tcp1 routed destination 00000400:00000200:16.0:1616513083.377731:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.73@tcp1(10.31.141.73@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.377738:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.377744:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.377747:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.377749:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.377754:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.377759:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.377761:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513083.377813:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.377820:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513083.377823:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513083.377823:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.377829:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.377835:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde220 00000800:00000200:2.0:1616513083.377839:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.378239:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.378251:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.378259:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513083.378260:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513083.378268:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.73@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.378277:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.73@tcp1 of length 224 into portal 12 MB=0x60217c92cb340 00000400:00000200:26.0:1616513083.378283:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.73@tcp1 of length 224/224 into md 0xf964d519 [1] + 28776 00000400:00000200:26.0:1616513083.378288:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.378291:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513083.378298:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.378301:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.378303:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.378361:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691151042917184 00010000:00000200:18.0:1616513083.378372:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57ec00 x1691151042917184/t0(0) o400->02dd1dbd-9648-fc8d-e836-904a2384f0d6@10.31.141.73@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.378386:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691151042917184, offset 192 00000400:00000200:18.0:1616513083.378391:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.73@tcp1 00000400:00000200:18.0:1616513083.378398:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.73@tcp1 routed destination 00000400:00000200:18.0:1616513083.378412:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.73@tcp1(10.31.141.73@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.378418:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.378424:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.378426:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.378428:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.378432:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.378480:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513083.378537:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513083.378544:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513083.378546:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513083.378552:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513083.378553:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513083.378557:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd540 00000800:00000200:4.0:1616513083.378561:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.380799:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.380856:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.380865:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.380874:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.117@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.380883:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.117@tcp1 of length 224 into portal 26 MB=0x6018446ea8240 00000400:00000200:0.0:1616513083.380889:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.117@tcp1 of length 224/224 into md 0xf97ca3d9 [1] + 0 00000400:00000200:0.0:1616513083.380894:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42440 00000400:00000200:0.0:1616513083.380897:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.380900:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.380904:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42440 00000800:00000200:0.0:1616513083.380907:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.380910:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.380910:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.380914:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.380967:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517497348672 00010000:00000200:16.0:1616513083.380978:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1690517497348672/t0(0) o400->04d3fab5-8af8-c652-368b-f5115e030498@10.31.142.117@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.380992:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690517497348672, offset 192 00000800:00000200:11.2:1616513083.380998:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:16.0:1616513083.380999:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.117@tcp1 00000400:00000200:16.0:1616513083.381006:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.117@tcp1 routed destination 00000400:00000200:16.0:1616513083.381021:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.117@tcp1(10.31.142.117@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.381032:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.381038:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.381040:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.381043:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.381048:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513083.381053:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513083.381053:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.381055:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513083.381062:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513083.381071:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.117@tcp1 : PUT - for me 00000400:00000200:28.0:1616513083.381079:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.117@tcp1 of length 224 into portal 12 MB=0x6018446ea84c0 00000400:00000200:28.0:1616513083.381087:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.117@tcp1 of length 224/224 into md 0x865bbec1 [1] + 59696 00000400:00000200:28.0:1616513083.381092:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.381095:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513083.381102:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.381105:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.381107:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513083.381107:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513083.381110:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513083.381114:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.381117:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513083.381118:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517497349312 00000400:00000200:26.0:1616513083.381124:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:8.0:1616513083.381128:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedc380 x1690517497349312/t0(0) o400->cb58fd5e-ca8f-9e2e-36e6-e66b3d5557b4@10.31.142.117@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:26.0:1616513083.381129:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfd38 00000800:00000200:26.0:1616513083.381132:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.381135:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513083.381140:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690517497349312, offset 192 00000400:00000200:8.0:1616513083.381146:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.117@tcp1 00000400:00000200:8.0:1616513083.381152:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.117@tcp1 routed destination 00000400:00000200:8.0:1616513083.381165:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.117@tcp1(10.31.142.117@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.381171:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513083.381176:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513083.381179:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513083.381181:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513083.381185:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.381191:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.381201:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.381206:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.381209:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.381221:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.381225:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.381229:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71210 00000800:00000200:20.0:1616513083.381232:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.406083:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513083.406139:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.406148:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513083.406157:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.156@tcp1 : PUT - for me 00000400:00000200:28.0:1616513083.406165:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.156@tcp1 of length 224 into portal 26 MB=0x5fff0454c4600 00000400:00000200:28.0:1616513083.406172:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.156@tcp1 of length 224/224 into md 0xf97ca3e9 [1] + 0 00000400:00000200:28.0:1616513083.406176:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42908 00000400:00000200:28.0:1616513083.406179:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.406181:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513083.406185:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42908 00000800:00000200:28.0:1616513083.406188:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.406190:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.406192:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513083.406194:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.406253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513083.406257:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688782303413760 00010000:00000200:16.0:1616513083.406268:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688782303413760/t0(0) o400->95a64802-e5ee-44f0-551c-60f1e8394ade@10.31.131.156@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513083.406269:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513083.406282:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688782303413760, offset 192 00000400:00000200:16.0:1616513083.406289:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.156@tcp1 00000800:00000200:0.0:1616513083.406293:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513083.406296:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.156@tcp1 routed destination 00000400:00000200:0.0:1616513083.406306:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.156@tcp1 : PUT - for me 00000400:00000200:16.0:1616513083.406312:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.156@tcp1(10.31.131.156@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513083.406314:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.156@tcp1 of length 224 into portal 12 MB=0x5fff0454c4a00 00000800:00000200:16.0:1616513083.406318:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513083.406320:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.156@tcp1 of length 224/224 into md 0x865bbec1 [1] + 59920 00000800:00000200:16.0:1616513083.406325:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:0.0:1616513083.406325:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513083.406326:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513083.406328:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513083.406328:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513083.406330:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.406334:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513083.406336:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.406339:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513083.406341:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513083.406341:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513083.406352:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688782303414784 00010000:00000200:8.0:1616513083.406362:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86a400 x1688782303414784/t0(0) o400->a97b0d4e-a318-c89a-3a1f-1be738c11d2c@10.31.131.156@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513083.406375:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688782303414784, offset 192 00000400:00000200:8.0:1616513083.406380:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.156@tcp1 00000800:00000200:7.2:1616513083.406384:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513083.406387:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.156@tcp1 routed destination 00000400:00000200:8.0:1616513083.406400:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.156@tcp1(10.31.131.156@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.406406:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513083.406411:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513083.406413:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513083.406415:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513083.406419:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.406424:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.406446:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.406450:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513083.406451:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513083.406453:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513083.406455:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.406458:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513083.406458:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513083.406461:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f716d8 00000400:00000200:14.0:1616513083.406461:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513083.406464:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513083.406468:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.406473:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfcb0 00000800:00000200:14.0:1616513083.406477:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.406480:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.413278:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.413337:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.413347:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.413356:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.187@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.413364:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.187@tcp1 of length 224 into portal 26 MB=0x60194e9beafc0 00000400:00000200:26.0:1616513083.413371:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.187@tcp1 of length 224/224 into md 0xf97ca419 [1] + 0 00000400:00000200:26.0:1616513083.413377:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a426e8 00000400:00000200:26.0:1616513083.413380:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.413382:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.413387:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a426e8 00000800:00000200:26.0:1616513083.413390:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.413392:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513083.413392:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.413396:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.413462:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588948639680 00010000:00000200:16.0:1616513083.413474:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1690588948639680/t0(0) o400->f90acd6a-de49-6cf1-d723-d2a5b6dc52d2@10.31.140.187@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.413488:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588948639680, offset 192 00000400:00000200:16.0:1616513083.413494:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.187@tcp1 00000400:00000200:16.0:1616513083.413501:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.187@tcp1 routed destination 00000400:00000200:16.0:1616513083.413517:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.187@tcp1(10.31.140.187@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.413523:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.413529:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.413532:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.413536:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.413541:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.413547:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.413548:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513083.413599:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.413606:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513083.413609:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513083.413611:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513083.413616:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.413624:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf320 00000800:00000200:14.0:1616513083.413628:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.414266:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.414322:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.414332:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.414341:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.187@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.414349:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.187@tcp1 of length 224 into portal 12 MB=0x60194e9beb740 00000400:00000200:20.0:1616513083.414355:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.187@tcp1 of length 224/224 into md 0x865bbec1 [1] + 60144 00000400:00000200:20.0:1616513083.414361:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.414363:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513083.414370:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.414373:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.414375:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.414379:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513083.414388:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588948641600 00010000:00000200:8.0:1616513083.414398:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cee300 x1690588948641600/t0(0) o400->77550ced-fbd1-9ad8-0e2f-604158c83c10@10.31.140.187@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513083.414412:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588948641600, offset 192 00000400:00000200:8.0:1616513083.414417:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.187@tcp1 00000400:00000200:8.0:1616513083.414424:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.187@tcp1 routed destination 00000400:00000200:8.0:1616513083.414444:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.187@tcp1(10.31.140.187@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513083.414451:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513083.414456:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513083.414459:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513083.414461:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513083.414465:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.414470:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513083.414524:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513083.414531:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513083.414534:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513083.414541:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513083.414545:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70e58 00000800:00000200:4.0:1616513083.414549:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513083.414552:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.420061:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513083.420117:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513083.420119:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513083.420127:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513083.420133:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513083.420136:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.188@tcp1 : PUT - for me 00000400:00000200:2.0:1616513083.420145:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.188@tcp1 of length 224 into portal 26 MB=0x5f93834bac400 00000400:00000200:2.0:1616513083.420154:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.188@tcp1 of length 224/224 into md 0xf97ca429 [1] + 0 00000400:00000200:2.0:1616513083.420159:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a423b8 00000400:00000200:2.0:1616513083.420162:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513083.420164:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513083.420169:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a423b8 00000800:00000200:2.0:1616513083.420172:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.420174:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513083.420175:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513083.420176:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513083.420185:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.420194:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.188@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.420202:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.188@tcp1 of length 224 into portal 12 MB=0x5f93834bac480 00000400:00000200:20.0:1616513083.420209:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.188@tcp1 of length 224/224 into md 0x865bbec1 [1] + 60368 00000400:00000200:20.0:1616513083.420214:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.420217:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513083.420224:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.420227:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.420229:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.420232:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.420235:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681394681693184 00010000:00000200:16.0:1616513083.420246:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1681394681693184/t0(0) o400->4cf364e1-a3ea-5e0f-2fa8-dcbbb169ce28@10.31.138.188@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.420260:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681394681693184, offset 192 00000400:00000200:16.0:1616513083.420266:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.188@tcp1 00000400:00000200:16.0:1616513083.420274:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.188@tcp1 routed destination 00000100:00000200:12.0:1616513083.420288:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681394681693312 00000400:00000200:16.0:1616513083.420289:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.188@tcp1(10.31.138.188@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.420295:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:12.0:1616513083.420300:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cec380 x1681394681693312/t0(0) o400->61bf3c5f-61bd-d177-87e2-3bdc8421b664@10.31.138.188@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.420301:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.420304:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.420306:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.420311:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513083.420314:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681394681693312, offset 192 00000800:00000200:7.2:1616513083.420316:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.420317:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513083.420321:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.188@tcp1 00000800:00000200:26.0:1616513083.420326:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513083.420328:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.188@tcp1 routed destination 00000800:00000200:26.0:1616513083.420331:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.420334:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.420341:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513083.420343:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.188@tcp1(10.31.138.188@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513083.420346:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf8f8 00000800:00000200:12.0:1616513083.420349:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513083.420350:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.420352:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513083.420355:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513083.420358:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.420360:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.420365:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.420370:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513083.420385:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513083.420392:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513083.420394:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513083.420401:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513083.420407:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f210 00000800:00000200:4.0:1616513083.420411:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513083.420414:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.424463:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.424519:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.424528:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.424537:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.81@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.424546:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.81@tcp1 of length 224 into portal 26 MB=0x5ffdca2da8880 00000400:00000200:20.0:1616513083.424553:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.81@tcp1 of length 224/224 into md 0xf97ca449 [1] + 0 00000400:00000200:20.0:1616513083.424558:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42110 00000400:00000200:20.0:1616513083.424560:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.424563:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513083.424567:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42110 00000800:00000200:20.0:1616513083.424570:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.424572:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.424573:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.424579:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.424582:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.424583:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.424592:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.81@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.424601:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.81@tcp1 of length 224 into portal 12 MB=0x5ffdca2da8900 00000400:00000200:0.0:1616513083.424607:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.81@tcp1 of length 224/224 into md 0x865bbec1 [1] + 60592 00000400:00000200:0.0:1616513083.424612:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.424615:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.424622:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.424625:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.424627:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.424631:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697973672064 00010000:00000200:16.0:1616513083.424643:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688697973672064/t0(0) o400->a4ade200-6acd-cdeb-c28e-b4a9d6dbad06@10.31.142.81@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.424657:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697973672064, offset 192 00000400:00000200:16.0:1616513083.424663:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.81@tcp1 00000400:00000200:16.0:1616513083.424671:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.81@tcp1 routed destination 00000400:00000200:16.0:1616513083.424685:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.81@tcp1(10.31.142.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:12.0:1616513083.424687:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697973672192 00000800:00000200:16.0:1616513083.424692:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.424698:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:12.0:1616513083.424699:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86f080 x1688697973672192/t0(0) o400->fbdf6f05-84f1-5a95-ab63-edcc4cf9d42e@10.31.142.81@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.424700:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.424702:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.424707:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513083.424713:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697973672192, offset 192 00000800:00000200:7.2:1616513083.424713:0:49809:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.424714:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513083.424720:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.81@tcp1 00000400:00000200:12.0:1616513083.424727:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.81@tcp1 routed destination 00000400:00000200:12.0:1616513083.424742:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.81@tcp1(10.31.142.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513083.424749:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513083.424754:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513083.424759:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.424761:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.424766:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513083.424767:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513083.424771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513083.424774:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.424776:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.424783:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.424788:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf430 00000800:00000200:26.0:1616513083.424792:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.424795:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513083.424843:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.424850:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.424853:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513083.424853:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.424859:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.424863:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ecc0 00000800:00000200:28.0:1616513083.424867:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.428098:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.428156:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.428165:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.428175:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.215@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.428184:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.215@tcp1 of length 224 into portal 26 MB=0x5ffe1eff8e340 00000400:00000200:0.0:1616513083.428190:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.215@tcp1 of length 224/224 into md 0xf97ca469 [1] + 0 00000400:00000200:0.0:1616513083.428195:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42bb0 00000400:00000200:0.0:1616513083.428198:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.428200:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.428205:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42bb0 00000800:00000200:0.0:1616513083.428208:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513083.428209:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.428211:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.428216:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.428219:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513083.428220:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513083.428228:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.215@tcp1 : PUT - for me 00000400:00000200:4.0:1616513083.428236:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.215@tcp1 of length 224 into portal 12 MB=0x5ffe1eff8ed00 00000400:00000200:4.0:1616513083.428242:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.215@tcp1 of length 224/224 into md 0x865bbec1 [1] + 60816 00000400:00000200:4.0:1616513083.428247:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513083.428249:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513083.428256:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513083.428259:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513083.428261:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.428284:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688720742343488 00010000:00000200:16.0:1616513083.428294:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688720742343488/t0(0) o400->4be11901-1587-6723-3966-5e1d03f17149@10.31.142.215@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.428309:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688720742343488, offset 192 00000400:00000200:16.0:1616513083.428315:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.215@tcp1 00000100:00000200:12.0:1616513083.428316:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688720742345984 00000400:00000200:16.0:1616513083.428321:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.215@tcp1 routed destination 00010000:00000200:12.0:1616513083.428327:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb238480 x1688720742345984/t0(0) o400->8abf9dae-a6fc-5577-b219-2582305b649f@10.31.142.215@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513083.428335:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.215@tcp1(10.31.142.215@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.428341:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:12.0:1616513083.428341:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688720742345984, offset 192 00000800:00000200:16.0:1616513083.428347:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513083.428347:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.215@tcp1 00000800:00000200:16.0:1616513083.428349:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.428351:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513083.428354:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.215@tcp1 routed destination 00000800:00000200:16.0:1616513083.428356:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513083.428362:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.428363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513083.428368:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.215@tcp1(10.31.142.215@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513083.428374:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513083.428379:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513083.428382:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.428384:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.428388:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.428393:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513083.428416:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.428422:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.428424:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513083.428428:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513083.428431:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513083.428450:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.428457:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513083.428457:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdee58 00000800:00000200:26.0:1616513083.428460:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513083.428461:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.428462:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.428469:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.428472:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e330 00000800:00000200:28.0:1616513083.428476:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.466649:0:49813:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.466705:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.466715:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.466724:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.85@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.466735:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.85@tcp1 of length 224 into portal 26 MB=0x601844e582b40 00000400:00000200:26.0:1616513083.466741:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.85@tcp1 of length 224/224 into md 0xf97ca479 [1] + 0 00000400:00000200:26.0:1616513083.466748:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42880 00000400:00000200:26.0:1616513083.466751:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.466755:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.466759:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42880 00000800:00000200:2.0:1616513083.466761:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.466762:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.466765:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.466767:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.466779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513083.466827:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517621975872 00010000:00000200:16.0:1616513083.466845:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1690517621975872/t0(0) o400->a995b855-0aae-4e8f-7e3a-5f0d3f914e54@10.31.131.85@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:4.0:1616513083.466851:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513083.466860:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690517621975872, offset 192 00000800:00000200:4.0:1616513083.466877:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513083.466878:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.85@tcp1 00000400:00000200:4.0:1616513083.466886:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.85@tcp1 : PUT - for me 00000400:00000200:16.0:1616513083.466887:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.85@tcp1 routed destination 00000400:00000200:4.0:1616513083.466897:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.85@tcp1 of length 224 into portal 12 MB=0x601844e582bc0 00000400:00000200:16.0:1616513083.466903:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.85@tcp1(10.31.131.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:4.0:1616513083.466903:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.85@tcp1 of length 224/224 into md 0x865bbec1 [1] + 61040 00000400:00000200:4.0:1616513083.466908:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513083.466910:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:4.0:1616513083.466911:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513083.466912:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513083.466917:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513083.466918:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.466920:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513083.466920:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513083.466922:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513083.466922:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513083.466928:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.466932:0:49813:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.466935:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513083.466982:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517621976000 00000800:00000200:14.0:1616513083.466988:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.466996:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:12.0:1616513083.466996:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb239680 x1690517621976000/t0(0) o400->36a3c387-bd3f-73a6-7ca4-a504e4767cb7@10.31.131.85@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513083.466999:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513083.467000:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513083.467006:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.467011:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf4b8 00000800:00000200:14.0:1616513083.467015:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513083.467015:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690517621976000, offset 192 00000400:00000200:12.0:1616513083.467021:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.85@tcp1 00000400:00000200:12.0:1616513083.467029:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.85@tcp1 routed destination 00000400:00000200:12.0:1616513083.467044:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.85@tcp1(10.31.131.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513083.467051:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513083.467057:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513083.467059:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.467061:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.467067:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.467072:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.467127:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.467134:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513083.467137:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513083.467143:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.467147:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eaa0 00000800:00000200:0.0:1616513083.467153:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.467157:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.473504:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513083.473559:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.473569:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513083.473577:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.222@tcp1 : PUT - for me 00000400:00000200:2.0:1616513083.473587:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.222@tcp1 of length 224 into portal 12 MB=0x601cfbcc28f00 00000400:00000200:2.0:1616513083.473594:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.222@tcp1 of length 224/224 into md 0xf964d519 [1] + 29000 00000400:00000200:2.0:1616513083.473599:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513083.473602:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513083.473610:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.473613:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513083.473613:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513083.473620:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513083.473623:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.473624:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.473632:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.222@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.473642:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.222@tcp1 of length 224 into portal 26 MB=0x601cfbcc28e00 00000400:00000200:26.0:1616513083.473648:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.222@tcp1 of length 224/224 into md 0xf97ca489 [1] + 0 00000400:00000200:26.0:1616513083.473653:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43ba0 00000400:00000200:26.0:1616513083.473655:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.473658:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.473662:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ba0 00000800:00000200:26.0:1616513083.473665:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.473668:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.473670:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.473690:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690841596989184 00010000:00000200:18.0:1616513083.473702:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d1200 x1690841596989184/t0(0) o400->a3b71b0c-3f1c-c7a0-e3f4-06388f03985b@10.31.136.222@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.473717:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690841596989184, offset 192 00000400:00000200:18.0:1616513083.473723:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.222@tcp1 00000100:00000200:16.0:1616513083.473724:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690841596988928 00000400:00000200:18.0:1616513083.473730:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.222@tcp1 routed destination 00010000:00000200:16.0:1616513083.473734:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1690841596988928/t0(0) o400->206a61df-4881-7665-2e73-b8c6fd076fb2@10.31.136.222@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513083.473745:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.222@tcp1(10.31.136.222@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513083.473748:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690841596988928, offset 192 00000800:00000200:18.0:1616513083.473751:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513083.473753:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.222@tcp1 00000800:00000200:18.0:1616513083.473757:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513083.473761:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.222@tcp1 routed destination 00000800:00000200:18.0:1616513083.473762:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513083.473764:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513083.473769:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.473773:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513083.473776:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.222@tcp1(10.31.136.222@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.473782:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.473788:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.473790:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.473792:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.473797:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513083.473803:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513083.473803:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513083.473826:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.473833:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513083.473836:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513083.473837:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513083.473843:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.473847:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc660 00000800:00000200:14.0:1616513083.473851:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513083.473858:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.473865:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.473867:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513083.473874:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.473879:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdec38 00000800:00000200:20.0:1616513083.473883:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.473886:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:23.2:1616513083.494129:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f49000] (20)++ 00000800:00000200:23.0:1616513083.494152:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f49000] (21)++ 00000800:00000200:23.0:1616513083.494161:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.202@o2ib1 00000400:00000200:23.0:1616513083.494171:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.202@o2ib1 : PUT - for me 00000400:00000200:23.0:1616513083.494181:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.202@o2ib1 of length 224 into portal 26 MB=0x60381910fafc0 00000400:00000200:23.0:1616513083.494188:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.202@o2ib1 of length 224/224 into md 0xf97ca4a9 [1] + 0 00000400:00000200:23.0:1616513083.494193:0:6474:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43100 00000400:00000200:23.0:1616513083.494197:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:23.0:1616513083.494200:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:23.0:1616513083.494206:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43100 00000800:00000200:23.0:1616513083.494213:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f49000] (22)++ 00000800:00000200:23.0:1616513083.494216:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f49000] (23)-- 00000800:00000200:23.0:1616513083.494217:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f49000] (22)-- 00000800:00000200:27.0:1616513083.494220:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f49000] (21)++ 00000800:00000200:27.0:1616513083.494232:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.202@o2ib1 00000400:00000200:27.0:1616513083.494241:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.202@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513083.494249:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.202@o2ib1 of length 224 into portal 12 MB=0x60381910fb040 00000400:00000200:27.0:1616513083.494256:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.202@o2ib1 of length 224/224 into md 0x3657608d [1] + 91616 00000400:00000200:27.0:1616513083.494262:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513083.494265:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:16.0:1616513083.494270:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692704879652800 00000800:00000200:27.0:1616513083.494274:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f49000] (22)++ 00000800:00000200:27.0:1616513083.494277:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f49000] (23)-- 00000800:00000200:31.0:1616513083.494278:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f49000] (23)-- 00010000:00000200:16.0:1616513083.494281:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1692704879652800/t0(0) o400->50293889-9be1-2abd-9fe1-5e284c974829@10.242.46.202@o2ib1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:27.0:1616513083.494282:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f49000] (21)-- 00000100:00000200:16.0:1616513083.494294:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692704879652800, offset 192 00000400:00000200:16.0:1616513083.494300:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.202@o2ib1 00000100:00000200:23.0:1616513083.494302:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692704879652928 00000400:00000200:16.0:1616513083.494307:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.202@o2ib1 local destination 00010000:00000200:23.0:1616513083.494317:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d841ded5e80 x1692704879652928/t0(0) o400->ba4666a3-d447-eca3-b72f-7fe53c0ca012@10.242.46.202@o2ib1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513083.494321:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.202@o2ib1(10.242.46.202@o2ib1:10.242.46.202@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.494328:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.202@o2ib1 00000100:00000200:23.0:1616513083.494332:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692704879652928, offset 192 00000800:00000200:16.0:1616513083.494334:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91ec100] -> 10.242.46.202@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.494337:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f49000] (20)++ 00000400:00000200:23.0:1616513083.494338:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.202@o2ib1 00000800:00000200:16.0:1616513083.494339:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f49000] (21)++ 00000800:00000200:16.0:1616513083.494343:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f49000] (22)-- 00000400:00000200:23.0:1616513083.494345:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.202@o2ib1 local destination 00000800:00000200:23.2:1616513083.494349:0:170661:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f49000] (21)++ 00000100:00000200:16.0:1616513083.494350:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:27.0:1616513083.494357:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f49000] (22)++ 00000800:00000200:27.0:1616513083.494363:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f49000] (23)-- 00000400:00000200:27.0:1616513083.494366:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513083.494367:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f49000] (22)-- 00000400:00000200:23.0:1616513083.494368:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.202@o2ib1(10.242.46.202@o2ib1:10.242.46.202@o2ib1) : PUT try# 0 00000400:00000200:27.0:1616513083.494372:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.202@o2ib1: PUT: OK 00000800:00000200:23.0:1616513083.494374:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.202@o2ib1 00000400:00000200:27.0:1616513083.494378:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdec38 00000800:00000200:27.0:1616513083.494381:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f49000] (21)-- 00000800:00000200:23.0:1616513083.494381:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91ec100] -> 10.242.46.202@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513083.494383:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f49000] (20)++ 00000800:00000200:23.0:1616513083.494385:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f49000] (21)++ 00000800:00000200:23.0:1616513083.494390:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f49000] (22)-- 00000800:00000200:23.2:1616513083.494394:0:170661:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f49000] (21)++ 00000800:00000200:31.0:1616513083.494402:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f49000] (22)++ 00000800:00000200:31.0:1616513083.494408:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f49000] (23)-- 00000400:00000200:31.0:1616513083.494411:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:27.0:1616513083.494412:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f49000] (22)-- 00000400:00000200:31.0:1616513083.494417:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.202@o2ib1: PUT: OK 00000400:00000200:31.0:1616513083.494422:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eee0 00000800:00000200:31.0:1616513083.494428:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f49000] (21)-- 00000800:00000200:7.2:1616513083.506086:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513083.506099:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513083.506108:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513083.506116:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.226@tcp1 : PUT - for me 00000400:00000200:30.0:1616513083.506125:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.226@tcp1 of length 224 into portal 26 MB=0x5ffdc63a77f40 00000400:00000200:30.0:1616513083.506131:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.226@tcp1 of length 224/224 into md 0xf97ca4b9 [1] + 0 00000400:00000200:30.0:1616513083.506135:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42c38 00000400:00000200:30.0:1616513083.506138:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513083.506141:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513083.506145:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42c38 00000800:00000200:11.2:1616513083.506145:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513083.506148:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513083.506151:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513083.506153:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513083.506157:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.506209:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696913362752 00000800:00000200:0.0:1616513083.506214:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:16.0:1616513083.506221:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688696913362752/t0(0) o400->67553fbf-ca88-941c-3ed8-b5f1a91993c1@10.31.141.226@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513083.506223:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.506231:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.226@tcp1 : PUT - for me 00000100:00000200:16.0:1616513083.506236:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696913362752, offset 192 00000400:00000200:0.0:1616513083.506239:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.226@tcp1 of length 224 into portal 12 MB=0x5ffdc63a781c0 00000400:00000200:16.0:1616513083.506243:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.226@tcp1 00000400:00000200:0.0:1616513083.506248:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.226@tcp1 of length 224/224 into md 0x865bbec1 [1] + 61264 00000400:00000200:16.0:1616513083.506251:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.226@tcp1 routed destination 00000400:00000200:0.0:1616513083.506253:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.506255:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.506261:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.506264:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513083.506266:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.226@tcp1(10.31.141.226@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513083.506266:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.506269:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513083.506272:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.506279:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.506284:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.506286:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.506291:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.506296:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.506297:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513083.506325:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696913363392 00010000:00000200:12.0:1616513083.506337:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde869b00 x1688696913363392/t0(0) o400->5396dc29-5cca-9200-03d6-1a1d04dc566b@10.31.141.226@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513083.506351:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.506358:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.506361:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:12.0:1616513083.506363:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696913363392, offset 192 00000400:00000200:2.0:1616513083.506368:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513083.506369:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.226@tcp1 00000400:00000200:2.0:1616513083.506374:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdee58 00000400:00000200:12.0:1616513083.506376:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.226@tcp1 routed destination 00000800:00000200:2.0:1616513083.506377:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.506380:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513083.506391:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.226@tcp1(10.31.141.226@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513083.506398:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513083.506404:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513083.506407:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.506409:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.506414:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.506419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.506432:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.506453:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.506456:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513083.506456:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.506463:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.506467:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fdc0 00000800:00000200:20.0:1616513083.506471:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.517272:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513083.517275:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513083.517330:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.517340:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513083.517347:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513083.517349:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.234@tcp1 : PUT - for me 00000800:00000200:26.0:1616513083.517357:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513083.517358:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.234@tcp1 of length 224 into portal 12 MB=0x5ffe585868f00 00000800:00000200:30.0:1616513083.517359:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513083.517365:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.234@tcp1 of length 224/224 into md 0x865bbec1 [1] + 61488 00000400:00000200:26.0:1616513083.517366:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.234@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.517370:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.517373:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:26.0:1616513083.517375:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.234@tcp1 of length 224 into portal 26 MB=0x5ffe585868dc0 00000800:00000200:0.0:1616513083.517379:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513083.517382:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.234@tcp1 of length 224/224 into md 0xf97ca4c9 [1] + 0 00000800:00000200:0.0:1616513083.517382:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.517383:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.517386:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513083.517387:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42e58 00000400:00000200:26.0:1616513083.517389:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.517392:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.517396:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42e58 00000800:00000200:26.0:1616513083.517399:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.517402:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.517404:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513083.517455:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688736136335104 00000100:00000200:16.0:1616513083.517465:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688736136334784 00010000:00000200:12.0:1616513083.517467:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86f980 x1688736136335104/t0(0) o400->da24cc7c-62cf-de69-2135-2a79191aeff2@10.31.140.234@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:16.0:1616513083.517476:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688736136334784/t0(0) o400->963fce11-6989-52c4-95af-0904326bf06a@10.31.140.234@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513083.517482:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688736136335104, offset 192 00000400:00000200:12.0:1616513083.517488:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.234@tcp1 00000100:00000200:16.0:1616513083.517491:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688736136334784, offset 192 00000400:00000200:12.0:1616513083.517495:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.234@tcp1 routed destination 00000400:00000200:16.0:1616513083.517497:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.234@tcp1 00000400:00000200:16.0:1616513083.517505:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.234@tcp1 routed destination 00000400:00000200:12.0:1616513083.517511:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.234@tcp1(10.31.140.234@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513083.517518:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513083.517520:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.234@tcp1(10.31.140.234@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.517527:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513083.517528:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513083.517530:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.517533:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513083.517533:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.517536:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.517537:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513083.517538:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513083.517542:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.517543:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.517548:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513083.517549:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513083.517559:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513083.517565:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513083.517567:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513083.517570:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513083.517574:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513083.517579:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf8f8 00000800:00000200:8.0:1616513083.517583:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513083.517596:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.517603:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.517605:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.517612:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.517616:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f870 00000800:00000200:2.0:1616513083.517622:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.517625:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.526453:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513083.526464:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513083.526472:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513083.526481:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.111@tcp1 : PUT - for me 00000400:00000200:30.0:1616513083.526490:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.111@tcp1 of length 224 into portal 26 MB=0x60195c57b7740 00000400:00000200:30.0:1616513083.526496:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.111@tcp1 of length 224/224 into md 0xf97ca4f9 [1] + 0 00000400:00000200:30.0:1616513083.526501:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43320 00000400:00000200:30.0:1616513083.526505:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513083.526508:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513083.526519:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43320 00000800:00000200:30.0:1616513083.526522:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.526522:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513083.526524:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513083.526529:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513083.526532:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.526533:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.526541:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.111@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.526550:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.111@tcp1 of length 224 into portal 12 MB=0x60195c57b77c0 00000400:00000200:26.0:1616513083.526557:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.111@tcp1 of length 224/224 into md 0xf964d519 [1] + 29224 00000400:00000200:26.0:1616513083.526562:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.526565:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513083.526573:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.526576:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.526578:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.526585:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690592635221824 00010000:00000200:16.0:1616513083.526596:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1690592635221824/t0(0) o400->31ea3c46-264a-8427-1b34-9d268361b59d@10.31.140.111@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.526610:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690592635221824, offset 192 00000400:00000200:16.0:1616513083.526617:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.111@tcp1 00000400:00000200:16.0:1616513083.526624:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.111@tcp1 routed destination 00000100:00000200:18.0:1616513083.526639:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690592635221952 00000400:00000200:16.0:1616513083.526639:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.111@tcp1(10.31.140.111@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.526645:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.526651:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:18.0:1616513083.526652:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc578000 x1690592635221952/t0(0) o400->662329b5-dcdc-5c5a-cbad-c2f3852f92b9@10.31.140.111@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.526654:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.526656:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.526661:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:18.0:1616513083.526666:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690592635221952, offset 192 00000800:00000200:7.2:1616513083.526667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.526668:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513083.526672:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.111@tcp1 00000400:00000200:18.0:1616513083.526680:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.111@tcp1 routed destination 00000800:00000200:14.0:1616513083.526690:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513083.526695:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.111@tcp1(10.31.140.111@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513083.526697:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513083.526700:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513083.526700:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513083.526702:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513083.526706:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513083.526708:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.526710:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.526712:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513083.526712:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfcb0 00000800:00000200:14.0:1616513083.526715:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513083.526717:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.526724:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.526775:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.526781:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513083.526783:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513083.526789:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.526792:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccee0 00000800:00000200:0.0:1616513083.526796:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.526798:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.532889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.532943:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.532953:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513083.532955:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513083.532962:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.234@tcp1 : PUT - for me 00000800:00000200:8.0:1616513083.532963:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513083.532966:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513083.532971:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.234@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.532972:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.234@tcp1 of length 224 into portal 26 MB=0x5ffe33f90e800 00000400:00000200:20.0:1616513083.532979:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.234@tcp1 of length 224/224 into md 0xf97ca519 [1] + 0 00000400:00000200:8.0:1616513083.532979:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.234@tcp1 of length 224 into portal 12 MB=0x5ffe33f90eac0 00000400:00000200:20.0:1616513083.532984:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43a08 00000400:00000200:8.0:1616513083.532985:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.234@tcp1 of length 224/224 into md 0x865bbec1 [1] + 61712 00000400:00000200:20.0:1616513083.532987:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.532990:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513083.532990:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513083.532992:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513083.532995:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a08 00000800:00000200:20.0:1616513083.532998:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513083.532999:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.533001:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513083.533002:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513083.533003:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513083.533005:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.533059:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726372673536 00010000:00000200:16.0:1616513083.533070:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688726372673536/t0(0) o400->269d4760-6a8b-f409-d17e-15fdaa9e44d8@10.31.141.234@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513083.533078:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726372674240 00000100:00000200:16.0:1616513083.533085:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726372673536, offset 192 00010000:00000200:12.0:1616513083.533090:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d1680 x1688726372674240/t0(0) o400->71b252d8-47f4-4464-15e8-119c0ab1663c@10.31.141.234@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513083.533091:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.234@tcp1 00000400:00000200:16.0:1616513083.533098:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.234@tcp1 routed destination 00000100:00000200:12.0:1616513083.533104:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726372674240, offset 192 00000400:00000200:12.0:1616513083.533111:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.234@tcp1 00000400:00000200:16.0:1616513083.533113:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.234@tcp1(10.31.141.234@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513083.533118:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.234@tcp1 routed destination 00000800:00000200:16.0:1616513083.533120:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513083.533133:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.234@tcp1(10.31.141.234@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513083.533141:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.533142:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.533145:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.533147:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513083.533147:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513083.533150:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.533152:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513083.533152:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.533157:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513083.533158:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.533160:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513083.533162:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.533218:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513083.533219:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513083.533225:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513083.533226:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513083.533227:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.533227:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513083.533229:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.533234:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.533235:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.533242:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde220 00000400:00000200:28.0:1616513083.533245:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e2a8 00000800:00000200:26.0:1616513083.533245:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.533248:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513083.533249:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.543534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.543592:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.543602:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513083.543603:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513083.543611:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.172@tcp1 : PUT - for me 00000800:00000200:8.0:1616513083.543611:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513083.543619:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.172@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.543620:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.172@tcp1 of length 224 into portal 26 MB=0x60392e25c3000 00000400:00000200:20.0:1616513083.543626:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.172@tcp1 of length 224/224 into md 0xf97ca539 [1] + 0 00000400:00000200:8.0:1616513083.543627:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.172@tcp1 of length 224 into portal 12 MB=0x60392e25c3080 00000400:00000200:20.0:1616513083.543631:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42330 00000400:00000200:8.0:1616513083.543632:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.172@tcp1 of length 224/224 into md 0x865bbec1 [1] + 61936 00000400:00000200:20.0:1616513083.543633:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.543636:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513083.543637:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513083.543639:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513083.543641:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42330 00000800:00000200:20.0:1616513083.543644:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513083.543645:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513083.543646:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513083.543648:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:8.0:1616513083.543648:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:8.0:1616513083.543650:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.543652:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.543702:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692779258064896 00010000:00000200:16.0:1616513083.543713:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1692779258064896/t0(0) o400->c535fcdb-2ca7-1015-9546-9e22c4b2e755@10.31.140.172@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513083.543723:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692779258065024 00000100:00000200:16.0:1616513083.543726:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692779258064896, offset 192 00000400:00000200:16.0:1616513083.543731:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.172@tcp1 00010000:00000200:12.0:1616513083.543735:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d1b00 x1692779258065024/t0(0) o400->9ab3ba33-1a89-8944-ce09-4a86761933c8@10.31.140.172@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513083.543738:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.172@tcp1 routed destination 00000100:00000200:12.0:1616513083.543748:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692779258065024, offset 192 00000400:00000200:16.0:1616513083.543752:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.172@tcp1(10.31.140.172@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513083.543753:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.172@tcp1 00000800:00000200:16.0:1616513083.543759:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513083.543760:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.172@tcp1 routed destination 00000400:00000200:12.0:1616513083.543774:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.172@tcp1(10.31.140.172@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513083.543780:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.543781:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.543783:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.543785:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513083.543785:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513083.543788:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.543789:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513083.543790:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.543794:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513083.543795:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.543797:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513083.543799:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513083.543808:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513083.543812:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513083.543814:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513083.543819:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513083.543824:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde2a8 00000800:00000200:30.0:1616513083.543827:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513083.543829:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513083.543853:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.543860:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.543863:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513083.543863:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.543869:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.543873:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f6d8 00000800:00000200:28.0:1616513083.543877:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.556086:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.556146:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.556158:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.556167:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.74@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.556176:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.74@tcp1 of length 224 into portal 26 MB=0x5ffdc16df5840 00000400:00000200:20.0:1616513083.556183:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.74@tcp1 of length 224/224 into md 0xf97ca559 [1] + 0 00000400:00000200:20.0:1616513083.556188:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42550 00000400:00000200:20.0:1616513083.556191:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.556194:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513083.556198:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42550 00000800:00000200:0.0:1616513083.556200:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.556201:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.556204:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.556208:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.556211:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.556211:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.556219:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.74@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.556227:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.74@tcp1 of length 224 into portal 12 MB=0x5ffdc16df59c0 00000400:00000200:0.0:1616513083.556232:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.74@tcp1 of length 224/224 into md 0x865bbec1 [1] + 62160 00000400:00000200:0.0:1616513083.556237:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.556239:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.556245:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.556248:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.556249:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.556259:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695625177152 00010000:00000200:16.0:1616513083.556270:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688695625177152/t0(0) o400->425310cd-ed2d-a8e6-0094-0789fba5aebe@10.31.141.74@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.556283:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695625177152, offset 192 00000400:00000200:16.0:1616513083.556290:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.74@tcp1 00000400:00000200:16.0:1616513083.556297:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.74@tcp1 routed destination 00000100:00000200:12.0:1616513083.556306:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695625177536 00000400:00000200:16.0:1616513083.556311:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.74@tcp1(10.31.141.74@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.556317:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:12.0:1616513083.556317:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde869200 x1688695625177536/t0(0) o400->c140913b-4045-754e-aff5-fa63883d8ba9@10.31.141.74@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.556323:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.556325:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.556327:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513083.556331:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695625177536, offset 192 00000800:00000200:16.0:1616513083.556332:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513083.556337:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.74@tcp1 00000100:00000200:16.0:1616513083.556338:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513083.556344:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.74@tcp1 routed destination 00000800:00000200:7.2:1616513083.556349:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513083.556358:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.74@tcp1(10.31.141.74@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513083.556364:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513083.556369:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513083.556372:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.556373:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.556377:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.556383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513083.556403:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.556409:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.556412:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.556418:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.556423:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde000 00000800:00000200:26.0:1616513083.556427:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.556429:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513083.556442:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513083.556449:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513083.556449:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513083.556451:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.556453:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.556455:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513083.556456:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.556458:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513083.556459:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.556465:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.556469:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f980 00000800:00000200:28.0:1616513083.556473:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513083.562476:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926224, portal 4 00000100:00000200:30.0:1616513083.562481:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926224, offset 0 00000400:00000200:30.0:1616513083.562486:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:30.0:1616513083.562492:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:30.0:1616513083.562504:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513083.562510:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:30.0:1616513083.562516:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513083.562518:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:30.0:1616513083.562520:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:30.0:1616513083.562524:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513083.562571:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.562591:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513083.562598:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513083.562602:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513083.562609:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513083.562617:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d7464623600 x1693067725926224/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513083.562630:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee110 00000800:00000200:1.0:1616513083.562635:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513083.562638:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513083.562749:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513083.562759:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.562765:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513083.562769:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513083.562774:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513083.562783:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626f50 00000400:00000200:1.0:1616513083.562791:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffca5 [1] + 192 00000400:00000200:1.0:1616513083.562796:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513083.562801:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d7464623600 x1693067725926224/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513083.562816:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.562819:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513083.562821:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:30.0:1616513083.562822:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d7464623600 x1693067725926224/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513083.562834:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d7464623600 x1693067725926224/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513083.562843:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eecc0 00000800:00000200:11.2:1616513083.608410:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.608468:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.608478:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.608489:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.29@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.608499:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.29@tcp1 of length 224 into portal 26 MB=0x5ffe785a45780 00000400:00000200:20.0:1616513083.608506:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.29@tcp1 of length 224/224 into md 0xf97ca569 [1] + 0 00000400:00000200:20.0:1616513083.608511:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a434b8 00000400:00000200:20.0:1616513083.608514:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.608517:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513083.608523:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a434b8 00000800:00000200:0.0:1616513083.608524:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.608526:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.608529:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.608531:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.608590:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688744728221568 00010000:00000200:16.0:1616513083.608602:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688744728221568/t0(0) o400->fd546ca5-fd89-3e1c-fd37-0b175548b3a5@10.31.142.29@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.608618:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688744728221568, offset 192 00000400:00000200:16.0:1616513083.608624:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.29@tcp1 00000400:00000200:16.0:1616513083.608632:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.29@tcp1 routed destination 00000800:00000200:7.2:1616513083.608641:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513083.608647:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.29@tcp1(10.31.142.29@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.608654:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.608661:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.608664:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.608666:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513083.608672:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:16.0:1616513083.608680:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513083.608698:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.608708:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513083.608709:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513083.608717:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513083.608717:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.29@tcp1 : PUT - for me 00000400:00000200:30.0:1616513083.608721:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513083.608725:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.29@tcp1 of length 224 into portal 12 MB=0x5ffe785a459c0 00000400:00000200:30.0:1616513083.608728:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.608732:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.29@tcp1 of length 224/224 into md 0xf964d519 [1] + 29448 00000400:00000200:30.0:1616513083.608734:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdebb0 00000800:00000200:30.0:1616513083.608737:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513083.608738:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513083.608740:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513083.608741:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513083.608749:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.608752:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.608754:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.608816:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688744728222144 00010000:00000200:18.0:1616513083.608829:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464624800 x1688744728222144/t0(0) o400->5eeca1a2-ebc1-ea7e-8a6a-37d4da70a49a@10.31.142.29@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.608844:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688744728222144, offset 192 00000400:00000200:18.0:1616513083.608850:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.29@tcp1 00000400:00000200:18.0:1616513083.608857:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.29@tcp1 routed destination 00000400:00000200:18.0:1616513083.608872:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.29@tcp1(10.31.142.29@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.608879:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.608885:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.608888:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.608890:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.608895:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.608900:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513083.608920:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513083.608926:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513083.608928:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513083.608934:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513083.608938:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd188 00000800:00000200:8.0:1616513083.608941:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513083.608944:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513083.610574:0:6480:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926240, portal 4 00000100:00000200:0.0:1616513083.610580:0:6480:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926240, offset 0 00000400:00000200:0.0:1616513083.610586:0:6480:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:0.0:1616513083.610592:0:6480:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:0.0:1616513083.610606:0:6480:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513083.610612:0:6480:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:0.0:1616513083.610618:0:6480:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513083.610621:0:6480:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:0.0:1616513083.610624:0:6480:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:0.0:1616513083.610629:0:6480:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513083.610676:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513083.610734:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513083.610742:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000800:00000200:30.0:1616513083.610744:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513083.610744:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.610751:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513083.610758:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4557ceb600 x1693067725926240/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513083.610770:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:2.0:1616513083.610773:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513083.610855:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513083.610910:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513083.610919:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513083.610928:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513083.610938:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626f60 00000400:00000200:14.0:1616513083.610945:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cab29 [1] + 192 00000400:00000200:14.0:1616513083.610950:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.610955:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4557ceb600 x1693067725926240/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513083.610965:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513083.610970:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513083.610973:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513083.610975:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:0.0:1616513083.610979:0:6480:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4557ceb600 x1693067725926240/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:0.0:1616513083.610992:0:6480:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4557ceb600 x1693067725926240/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513083.611002:0:6480:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb055c8 00000800:00000200:11.2:1616513083.625627:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.625691:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.625701:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.625710:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.93@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.625720:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.93@tcp1 of length 224 into portal 26 MB=0x5ffdd8e791940 00000400:00000200:0.0:1616513083.625727:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.93@tcp1 of length 224/224 into md 0xf97ca579 [1] + 0 00000400:00000200:0.0:1616513083.625731:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43cb0 00000400:00000200:0.0:1616513083.625736:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.625739:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.625744:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43cb0 00000800:00000200:20.0:1616513083.625746:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.625747:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.625750:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513083.625753:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.625756:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.625757:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.625766:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.93@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.625775:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.93@tcp1 of length 224 into portal 12 MB=0x5ffdd8e791ac0 00000400:00000200:20.0:1616513083.625781:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.93@tcp1 of length 224/224 into md 0x865bbec1 [1] + 62384 00000400:00000200:20.0:1616513083.625786:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.625789:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513083.625796:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.625801:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.625803:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.625807:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701926709568 00010000:00000200:16.0:1616513083.625818:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688701926709568/t0(0) o400->388baf13-852a-68bc-a851-a5b0b5c5e5c7@10.31.142.93@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.625833:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701926709568, offset 192 00000400:00000200:16.0:1616513083.625839:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.93@tcp1 00000400:00000200:16.0:1616513083.625846:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.93@tcp1 routed destination 00000100:00000200:12.0:1616513083.625860:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701926709952 00000400:00000200:16.0:1616513083.625861:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.93@tcp1(10.31.142.93@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.625868:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:12.0:1616513083.625872:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce9680 x1688701926709952/t0(0) o400->b52c5804-ce87-1afb-95ca-7ebea012e3b6@10.31.142.93@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.625876:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.625878:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.625880:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.625886:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513083.625887:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701926709952, offset 192 00000100:00000200:16.0:1616513083.625893:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513083.625893:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.93@tcp1 00000800:00000200:7.2:1616513083.625894:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513083.625901:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.93@tcp1 routed destination 00000800:00000200:30.0:1616513083.625907:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513083.625912:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513083.625914:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513083.625916:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.93@tcp1(10.31.142.93@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513083.625921:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513083.625922:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513083.625928:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513083.625932:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde2a8 00000800:00000200:12.0:1616513083.625932:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.625934:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513083.625936:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513083.625938:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513083.625939:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.625944:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.625960:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.625968:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.625971:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513083.625973:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.625977:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.625982:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e908 00000800:00000200:28.0:1616513083.625985:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513083.626479:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926256, portal 4 00000100:00000200:28.0:1616513083.626484:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926256, offset 0 00000400:00000200:28.0:1616513083.626490:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:28.0:1616513083.626495:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:28.0:1616513083.626508:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513083.626517:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:28.0:1616513083.626523:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513083.626526:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:28.0:1616513083.626528:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:28.0:1616513083.626533:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513083.626580:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513083.626603:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513083.626609:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513083.626615:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513083.626622:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513083.626629:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4557cebf00 x1693067725926256/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513083.626641:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccf68 00000800:00000200:27.0:1616513083.626646:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513083.626649:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513083.626715:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513083.626725:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513083.626731:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513083.626735:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513083.626739:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513083.626747:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626f70 00000400:00000200:27.0:1616513083.626754:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166c5d [1] + 192 00000400:00000200:27.0:1616513083.626760:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513083.626764:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4557cebf00 x1693067725926256/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513083.626780:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513083.626783:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513083.626785:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:28.0:1616513083.626788:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4557cebf00 x1693067725926256/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513083.626801:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4557cebf00 x1693067725926256/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513083.626811:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd078 00000800:00000200:7.2:1616513083.643623:0:49831:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513083.643645:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513083.643680:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.643690:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.643699:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.153@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.643709:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.153@tcp1 of length 224 into portal 12 MB=0x5ffff93dd7ec0 00000800:00000200:0.0:1616513083.643715:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513083.643716:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.153@tcp1 of length 224/224 into md 0xf964d519 [1] + 29672 00000400:00000200:26.0:1616513083.643723:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.643726:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.643741:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513083.643742:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.643745:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513083.643747:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.643750:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513083.643750:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.153@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.643758:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.153@tcp1 of length 224 into portal 26 MB=0x5ffff93dd7980 00000400:00000200:0.0:1616513083.643765:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.153@tcp1 of length 224/224 into md 0xf97ca589 [1] + 0 00000400:00000200:0.0:1616513083.643769:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43430 00000400:00000200:0.0:1616513083.643772:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.643774:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.643779:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43430 00000800:00000200:0.0:1616513083.643781:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.643784:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.643786:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.643789:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513083.643812:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688848046063296 00010000:00000200:18.0:1616513083.643825:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc578480 x1688848046063296/t0(0) o400->3d5aae49-7ef4-f445-e377-a634c02c93db@10.31.131.153@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513083.643843:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688848046061952 00000100:00000200:18.0:1616513083.643850:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688848046063296, offset 192 00010000:00000200:16.0:1616513083.643855:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688848046061952/t0(0) o400->55c1e30b-276e-bbd0-9a23-b9f0aedf6d18@10.31.131.153@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513083.643857:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.153@tcp1 00000400:00000200:18.0:1616513083.643864:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.153@tcp1 routed destination 00000100:00000200:16.0:1616513083.643869:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688848046061952, offset 192 00000400:00000200:16.0:1616513083.643875:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.153@tcp1 00000400:00000200:18.0:1616513083.643879:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.153@tcp1(10.31.131.153@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513083.643883:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.153@tcp1 routed destination 00000800:00000200:18.0:1616513083.643886:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513083.643892:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.643895:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513083.643897:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513083.643898:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.153@tcp1(10.31.131.153@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.643902:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513083.643905:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513083.643906:0:49831:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.643910:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.643913:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.643915:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513083.643917:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513083.643920:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513083.643923:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513083.643925:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513083.643925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513083.643927:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:30.0:1616513083.643931:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513083.643934:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd078 00000800:00000200:30.0:1616513083.643937:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513083.643940:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513083.643989:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.643996:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513083.643999:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513083.644005:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513083.644011:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfcb0 00000800:00000200:12.0:1616513083.644014:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.644017:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:5.0:1616513083.644580:0:6494:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926272, portal 4 00000100:00000200:5.0:1616513083.644586:0:6494:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926272, offset 0 00000400:00000200:5.0:1616513083.644592:0:6494:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:5.0:1616513083.644598:0:6494:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:5.0:1616513083.644611:0:6494:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:5.0:1616513083.644617:0:6494:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513083.644624:0:6494:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513083.644627:0:6494:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:5.0:1616513083.644629:0:6494:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513083.644634:0:6494:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513083.644643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.644657:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513083.644662:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513083.644665:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513083.644666:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513083.644672:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513083.644678:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1396c00 x1693067725926272/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513083.644696:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55fed0 00000800:00000200:1.0:1616513083.644701:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513083.644888:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513083.644897:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.644903:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513083.644905:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513083.644912:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513083.644920:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626f80 00000400:00000200:1.0:1616513083.644926:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffcb5 [1] + 192 00000400:00000200:1.0:1616513083.644931:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513083.644936:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1396c00 x1693067725926272/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513083.644949:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.644952:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513083.644954:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:5.0:1616513083.644955:0:6494:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1396c00 x1693067725926272/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:5.0:1616513083.644967:0:6494:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1396c00 x1693067725926272/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513083.644976:0:6494:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55e7f8 00000800:00000200:11.2:1616513083.645223:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.645275:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.645284:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.645292:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.67@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.645301:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.67@tcp1 of length 224 into portal 26 MB=0x601950b79d140 00000400:00000200:0.0:1616513083.645307:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.67@tcp1 of length 224/224 into md 0xf97ca599 [1] + 0 00000400:00000200:0.0:1616513083.645311:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a436d8 00000400:00000200:0.0:1616513083.645314:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.645317:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.645321:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a436d8 00000800:00000200:0.0:1616513083.645324:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.645326:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.645328:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.645332:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.645385:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589514551616 00010000:00000200:16.0:1616513083.645396:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1690589514551616/t0(0) o400->fffe811d-c1d7-8388-5ae5-9b34652260ff@10.31.141.67@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.645411:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690589514551616, offset 192 00000400:00000200:16.0:1616513083.645417:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.67@tcp1 00000400:00000200:16.0:1616513083.645425:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.67@tcp1 routed destination 00000400:00000200:16.0:1616513083.645453:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.67@tcp1(10.31.141.67@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.645460:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.645466:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.645469:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.645471:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.645476:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.645482:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.645485:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513083.645537:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.645543:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.645546:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.645553:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.645558:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdee58 00000800:00000200:26.0:1616513083.645562:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.645565:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.646248:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.646302:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.646312:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513083.646320:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.67@tcp1 : PUT - for me 00000400:00000200:12.0:1616513083.646329:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.67@tcp1 of length 224 into portal 12 MB=0x601950b79dd40 00000400:00000200:12.0:1616513083.646335:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.67@tcp1 of length 224/224 into md 0x865bbec1 [1] + 62608 00000400:00000200:12.0:1616513083.646340:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513083.646343:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513083.646351:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.646354:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.646356:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.646361:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.646366:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589514554688 00010000:00000200:16.0:1616513083.646374:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0df080 x1690589514554688/t0(0) o400->1665f9a1-97bb-777b-7329-36f1479a97b4@10.31.141.67@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513083.646386:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690589514554688, offset 192 00000400:00000200:16.0:1616513083.646391:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.67@tcp1 00000400:00000200:16.0:1616513083.646397:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.67@tcp1 routed destination 00000400:00000200:16.0:1616513083.646410:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.67@tcp1(10.31.141.67@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.646415:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.646420:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.646422:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.646424:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.646427:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.646432:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.646446:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.646452:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.646454:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.646457:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.646461:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.646465:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdee58 00000800:00000200:20.0:1616513083.646469:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.647837:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513083.647847:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513083.647854:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513083.647863:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.213@tcp1 : PUT - for me 00000400:00000200:30.0:1616513083.647884:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.213@tcp1 of length 224 into portal 26 MB=0x60016f69e71c0 00000400:00000200:30.0:1616513083.647890:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.213@tcp1 of length 224/224 into md 0xf97ca5a9 [1] + 0 00000400:00000200:30.0:1616513083.647894:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42ff0 00000400:00000200:30.0:1616513083.647896:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513083.647899:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513083.647902:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ff0 00000800:00000200:30.0:1616513083.647905:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513083.647907:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513083.647909:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513083.647911:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.647966:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688948487123392 00010000:00000200:16.0:1616513083.647977:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688948487123392/t0(0) o400->c2c31fe5-066f-5224-b7a7-79eaaa974560@10.31.136.213@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513083.647988:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513083.647991:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688948487123392, offset 192 00000400:00000200:16.0:1616513083.647997:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.213@tcp1 00000400:00000200:16.0:1616513083.648005:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.213@tcp1 routed destination 00000400:00000200:16.0:1616513083.648021:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.213@tcp1(10.31.136.213@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.648027:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.648033:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.648036:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.648038:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.648042:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:16.0:1616513083.648043:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:16.0:1616513083.648050:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513083.648051:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.648060:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.213@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.648068:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.213@tcp1 of length 224 into portal 12 MB=0x60016f69e7280 00000400:00000200:26.0:1616513083.648075:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.213@tcp1 of length 224/224 into md 0xf964d519 [1] + 29896 00000400:00000200:26.0:1616513083.648080:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.648083:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513083.648089:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513083.648092:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513083.648094:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513083.648097:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.648105:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513083.648108:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513083.648108:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513083.648115:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.648120:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfcb0 00000800:00000200:14.0:1616513083.648124:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.648152:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688948487123584 00010000:00000200:18.0:1616513083.648164:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57ad00 x1688948487123584/t0(0) o400->f278cccd-610e-1d4a-2b27-25704bb01edf@10.31.136.213@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.648179:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688948487123584, offset 192 00000400:00000200:18.0:1616513083.648185:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.213@tcp1 00000400:00000200:18.0:1616513083.648192:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.213@tcp1 routed destination 00000400:00000200:18.0:1616513083.648207:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.213@tcp1(10.31.136.213@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.648214:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.648220:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.648222:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.648224:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.648229:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.648237:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.648290:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.648297:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.648300:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.648307:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.648310:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc088 00000800:00000200:28.0:1616513083.648314:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.648317:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.658488:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:7.0:1616513083.658532:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926288, portal 4 00000100:00000200:7.0:1616513083.658537:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926288, offset 0 00000800:00000200:0.0:1616513083.658542:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:7.0:1616513083.658544:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:7.0:1616513083.658550:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:0.0:1616513083.658551:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.658561:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.108@tcp1 : PUT - for me 00000400:00000200:7.0:1616513083.658564:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513083.658570:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.108@tcp1 of length 224 into portal 26 MB=0x5ffffed418680 00000800:00000200:7.0:1616513083.658572:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:0.0:1616513083.658576:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.108@tcp1 of length 224/224 into md 0xf97ca5c9 [1] + 0 00000800:00000200:7.0:1616513083.658578:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513083.658581:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:0.0:1616513083.658581:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43210 00000800:00000200:7.0:1616513083.658583:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:0.0:1616513083.658583:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.658586:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:7.0:1616513083.658588:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:0.0:1616513083.658591:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43210 00000800:00000200:0.0:1616513083.658593:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.658596:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.658597:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513083.658604:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.658606:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.658608:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.658616:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.108@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.658625:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.108@tcp1 of length 224 into portal 12 MB=0x5ffffed418740 00000400:00000200:20.0:1616513083.658631:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.108@tcp1 of length 224/224 into md 0x865bbec1 [1] + 62832 00000800:00000200:1.2:1616513083.658633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:20.0:1616513083.658636:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.658639:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513083.658648:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.658651:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:1.0:1616513083.658651:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:20.0:1616513083.658653:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.658657:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688849545791104 00000800:00000200:1.0:1616513083.658657:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513083.658659:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513083.658661:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513083.658667:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00010000:00000200:16.0:1616513083.658668:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688849545791104/t0(0) o400->13cab8de-5ffc-c8dc-2d66-9a69422ed531@10.31.138.108@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:1.0:1616513083.658674:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600900 x1693067725926288/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513083.658682:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688849545791104, offset 192 00000400:00000200:1.0:1616513083.658686:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637ebb0 00000800:00000200:1.0:1616513083.658690:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000400:00000200:16.0:1616513083.658691:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.108@tcp1 00000400:00000200:16.0:1616513083.658698:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.108@tcp1 routed destination 00000100:00000200:24.0:1616513083.658713:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688849545791296 00000400:00000200:16.0:1616513083.658713:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.108@tcp1(10.31.138.108@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.658720:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:24.0:1616513083.658726:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cecc80 x1688849545791296/t0(0) o400->61f3a512-8416-5ce5-7c27-821ec213422c@10.31.138.108@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.658726:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.658729:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.658731:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.658736:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.658742:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.658743:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513083.658751:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688849545791296, offset 192 00000400:00000200:24.0:1616513083.658757:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.108@tcp1 00000400:00000200:24.0:1616513083.658764:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.108@tcp1 routed destination 00000400:00000200:24.0:1616513083.658779:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.108@tcp1(10.31.138.108@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.658786:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513083.658791:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.658794:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513083.658796:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513083.658796:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513083.658801:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513083.658803:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513083.658806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513083.658806:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.2:1616513083.658808:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:2.0:1616513083.658813:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:1.0:1616513083.658816:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:2.0:1616513083.658818:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdebb0 00000800:00000200:5.0:1616513083.658822:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:2.0:1616513083.658822:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:1.0:1616513083.658822:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:2.0:1616513083.658825:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:1.0:1616513083.658830:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513083.658838:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c626f90 00000400:00000200:1.0:1616513083.658844:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffcd5 [1] + 192 00000400:00000200:1.0:1616513083.658848:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513083.658852:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600900 x1693067725926288/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513083.658865:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.658869:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:7.0:1616513083.658871:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600900 x1693067725926288/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513083.658871:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:12.0:1616513083.658876:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:7.0:1616513083.658881:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600900 x1693067725926288/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513083.658883:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.658887:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513083.658888:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:7.0:1616513083.658889:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f7e8 00000400:00000200:12.0:1616513083.658895:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513083.658900:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a08 00000800:00000200:12.0:1616513083.658904:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.662174:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.662228:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.662237:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.662246:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.93@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.662256:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.93@tcp1 of length 224 into portal 26 MB=0x5ffdd35239e80 00000400:00000200:20.0:1616513083.662262:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.93@tcp1 of length 224/224 into md 0xf97ca5d9 [1] + 0 00000400:00000200:20.0:1616513083.662267:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42770 00000400:00000200:20.0:1616513083.662270:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.662273:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513083.662277:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:28.0:1616513083.662280:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.662280:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.662283:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.662285:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.662341:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700427935360 00010000:00000200:16.0:1616513083.662352:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688700427935360/t0(0) o400->da678dfe-b97a-8b62-6c4c-c7fda9f02ebb@10.31.128.93@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.662366:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700427935360, offset 192 00000400:00000200:16.0:1616513083.662373:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.93@tcp1 00000400:00000200:16.0:1616513083.662381:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.93@tcp1 routed destination 00000400:00000200:16.0:1616513083.662408:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.93@tcp1(10.31.128.93@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513083.662409:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.662414:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.662419:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513083.662420:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.662422:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.662424:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.662428:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513083.662429:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513083.662435:0:49833:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.662445:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513083.662448:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513083.662453:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:0.0:1616513083.662453:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.93@tcp1 : PUT - for me 00000400:00000200:30.0:1616513083.662455:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513083.662460:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.662461:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.93@tcp1 of length 224 into portal 12 MB=0x5ffdd3523a200 00000400:00000200:30.0:1616513083.662464:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdee58 00000800:00000200:30.0:1616513083.662466:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513083.662468:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513083.662468:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.93@tcp1 of length 224/224 into md 0x865bbec1 [1] + 63056 00000800:00000200:12.0:1616513083.662471:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513083.662473:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.662475:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.662482:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.662485:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.662487:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513083.662547:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700427936256 00010000:00000200:24.0:1616513083.662559:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86ec00 x1688700427936256/t0(0) o400->189c1d11-d1e5-b7c2-cb23-404596dd7899@10.31.128.93@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513083.662572:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700427936256, offset 192 00000400:00000200:24.0:1616513083.662578:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.93@tcp1 00000400:00000200:24.0:1616513083.662585:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.93@tcp1 routed destination 00000400:00000200:24.0:1616513083.662599:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.93@tcp1(10.31.128.93@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.662605:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513083.662610:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.662613:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513083.662615:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513083.662619:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.662624:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.662634:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.662640:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.662642:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513083.662643:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.662649:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.662653:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:28.0:1616513083.662657:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.667646:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.667703:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.667711:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513083.667720:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.240@tcp1 : PUT - for me 00000400:00000200:12.0:1616513083.667728:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.240@tcp1 of length 224 into portal 26 MB=0x601840f61fa40 00000400:00000200:12.0:1616513083.667734:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.240@tcp1 of length 224/224 into md 0xf97ca5e9 [1] + 0 00000400:00000200:12.0:1616513083.667739:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43d38 00000400:00000200:12.0:1616513083.667741:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513083.667744:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513083.667748:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43d38 00000800:00000200:12.0:1616513083.667751:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.667753:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.667755:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513083.667761:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513083.667764:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.667765:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.667773:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.240@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.667781:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.240@tcp1 of length 224 into portal 12 MB=0x601840f61fb40 00000400:00000200:0.0:1616513083.667787:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.240@tcp1 of length 224/224 into md 0x865bbec1 [1] + 63280 00000400:00000200:0.0:1616513083.667792:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.667794:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.667800:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.667805:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.667807:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.667811:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516565654080 00010000:00000200:16.0:1616513083.667822:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1690516565654080/t0(0) o400->9c069609-0ddb-ecd4-952f-520ee722092a@10.31.131.240@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.667835:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690516565654080, offset 192 00000400:00000200:16.0:1616513083.667841:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.240@tcp1 00000400:00000200:16.0:1616513083.667848:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.240@tcp1 routed destination 00000400:00000200:16.0:1616513083.667862:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.240@tcp1(10.31.131.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513083.667864:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516565654336 00000800:00000200:16.0:1616513083.667872:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:24.0:1616513083.667876:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86bf00 x1690516565654336/t0(0) o400->7d0f5736-82b8-73f1-e255-6fe5322e0af3@10.31.131.240@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.667877:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.667880:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.667882:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.667886:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513083.667889:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690516565654336, offset 192 00000100:00000200:16.0:1616513083.667893:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.667893:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513083.667895:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.240@tcp1 00000400:00000200:24.0:1616513083.667902:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.240@tcp1 routed destination 00000400:00000200:24.0:1616513083.667916:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.240@tcp1(10.31.131.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.667924:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513083.667929:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.667931:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513083.667933:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513083.667938:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.667943:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513083.667947:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.667954:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.667957:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513083.667958:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513083.667963:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513083.667965:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.667968:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513083.667969:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513083.667969:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdee58 00000800:00000200:2.0:1616513083.667973:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513083.667975:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513083.667976:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513083.667979:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2e58 00000800:00000200:28.0:1616513083.667983:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513083.667983:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513083.667985:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.667987:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513083.667989:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.667991:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.675754:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.675810:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.675819:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.675827:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.176@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.675836:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.176@tcp1 of length 224 into portal 26 MB=0x60184b354d240 00000400:00000200:0.0:1616513083.675842:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.176@tcp1 of length 224/224 into md 0xf97ca619 [1] + 0 00000400:00000200:0.0:1616513083.675847:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42198 00000400:00000200:0.0:1616513083.675849:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.675852:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.675856:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42198 00000800:00000200:0.0:1616513083.675859:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.675861:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.675863:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.675865:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.675918:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519316255296 00010000:00000200:16.0:1616513083.675929:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1690519316255296/t0(0) o400->c3aa218a-87c4-1fe0-7aa3-83ab6777a030@10.31.131.176@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.675942:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690519316255296, offset 192 00000400:00000200:16.0:1616513083.675948:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.176@tcp1 00000400:00000200:16.0:1616513083.675955:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.176@tcp1 routed destination 00000400:00000200:16.0:1616513083.675969:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.176@tcp1(10.31.131.176@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.675975:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.675980:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.675983:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.675985:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.675989:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513083.675996:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.676035:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.676092:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.676100:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513083.676103:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513083.676110:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.676115:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3d38 00000800:00000200:14.0:1616513083.676119:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.676122:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.676257:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.676318:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.676327:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:6.0:1616513083.676328:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513083.676335:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.176@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.676343:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.176@tcp1 of length 224 into portal 12 MB=0x60184b354d440 00000400:00000200:26.0:1616513083.676349:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.176@tcp1 of length 224/224 into md 0xf964d519 [1] + 30120 00000400:00000200:26.0:1616513083.676354:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.676357:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513083.676364:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.676367:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.676370:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.676449:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519316255808 00010000:00000200:18.0:1616513083.676462:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57c380 x1690519316255808/t0(0) o400->819cefaa-81d8-bbc5-48a1-59931c007a6b@10.31.131.176@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.676476:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690519316255808, offset 192 00000400:00000200:18.0:1616513083.676482:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.176@tcp1 00000400:00000200:18.0:1616513083.676489:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.176@tcp1 routed destination 00000400:00000200:18.0:1616513083.676504:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.176@tcp1(10.31.131.176@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.676513:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.676518:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.676521:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.676523:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.676528:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.676533:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.676585:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.676591:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513083.676594:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513083.676600:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513083.676604:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cded0 00000800:00000200:12.0:1616513083.676608:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.676610:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.690566:0:6482:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926304, portal 4 00000100:00000200:16.0:1616513083.690571:0:6482:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926304, offset 0 00000400:00000200:16.0:1616513083.690578:0:6482:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:16.0:1616513083.690583:0:6482:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000100:00000200:28.0:1616513083.690590:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926320, portal 4 00000100:00000200:28.0:1616513083.690595:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926320, offset 0 00000400:00000200:16.0:1616513083.690595:0:6482:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.690601:0:6482:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:28.0:1616513083.690604:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:16.0:1616513083.690607:0:6482:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513083.690610:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:16.0:1616513083.690610:0:6482:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:16.0:1616513083.690612:0:6482:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:16.0:1616513083.690617:0:6482:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:28.0:1616513083.690623:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513083.690630:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:28.0:1616513083.690636:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513083.690639:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:28.0:1616513083.690641:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:28.0:1616513083.690646:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.2:1616513083.690664:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513083.690685:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:27.0:1616513083.690691:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:27.0:1616513083.690694:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513083.690700:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513083.690707:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb23d100 x1693067725926304/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513083.690719:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3100 00000800:00000200:27.0:1616513083.690724:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513083.690726:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513083.690733:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513083.690735:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513083.690738:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513083.690740:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513083.690745:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4557ced580 x1693067725926320/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513083.690755:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcce58 00000800:00000200:27.0:1616513083.690759:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513083.690845:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513083.690855:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513083.690861:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513083.690864:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513083.690868:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:31.0:1616513083.690874:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513083.690876:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626fa0 00000800:00000200:3.0:1616513083.690877:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:31.0:1616513083.690883:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513083.690883:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166c7d [1] + 192 00000400:00000200:27.0:1616513083.690888:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:31.0:1616513083.690891:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626fb0 00000100:00000200:27.0:1616513083.690892:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb23d100 x1693067725926304/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513083.690906:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:31.0:1616513083.690908:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166c8d [1] + 192 00000800:00000200:27.0:1616513083.690909:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513083.690910:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:31.0:1616513083.690913:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513083.690916:0:6482:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb23d100 x1693067725926304/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513083.690932:0:6482:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb23d100 x1693067725926304/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513083.690943:0:6482:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3dc0 00000100:00000200:31.0:1616513083.690948:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4557ced580 x1693067725926320/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513083.690964:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513083.690967:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:31.0:1616513083.690969:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:28.0:1616513083.690973:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4557ced580 x1693067725926320/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513083.690986:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4557ced580 x1693067725926320/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513083.690997:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd078 00000800:00000200:11.2:1616513083.697737:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.697793:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.697802:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.697811:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.239@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.697819:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.239@tcp1 of length 224 into portal 12 MB=0x5fff603a4a6c0 00000400:00000200:0.0:1616513083.697825:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.239@tcp1 of length 224/224 into md 0x865bbec1 [1] + 63504 00000400:00000200:0.0:1616513083.697830:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.697832:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.697838:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.697841:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.697843:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.697845:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513083.697901:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688806971713216 00010000:00000200:24.0:1616513083.697913:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86ba80 x1688806971713216/t0(0) o400->983f5442-7bd3-bb93-9bd3-687dbcac7a76@10.31.141.239@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513083.697926:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688806971713216, offset 192 00000400:00000200:24.0:1616513083.697932:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.239@tcp1 00000400:00000200:24.0:1616513083.697939:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.239@tcp1 routed destination 00000400:00000200:24.0:1616513083.697953:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.239@tcp1(10.31.141.239@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.697959:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513083.697964:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.697967:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513083.697969:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513083.697973:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.697981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.698036:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.698043:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513083.698046:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513083.698052:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513083.698056:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000800:00000200:14.0:1616513083.698060:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513083.698064:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513083.706516:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926336, portal 4 00000100:00000200:26.0:1616513083.706521:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926336, offset 0 00000400:00000200:26.0:1616513083.706528:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:26.0:1616513083.706534:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:26.0:1616513083.706547:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513083.706553:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:26.0:1616513083.706559:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513083.706562:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513083.706564:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513083.706569:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513083.706629:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:6.0:1616513083.706687:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:6.0:1616513083.706706:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000800:00000200:26.0:1616513083.706708:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:6.0:1616513083.706708:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513083.706713:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:6.0:1616513083.706718:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf87980 x1693067725926336/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513083.706726:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae550 00000800:00000200:6.0:1616513083.706729:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513083.706790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513083.706844:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513083.706854:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:2.0:1616513083.706857:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513083.706863:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513083.706870:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626fc0 00000400:00000200:14.0:1616513083.706877:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cabe9 [1] + 192 00000400:00000200:14.0:1616513083.706884:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.706889:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf87980 x1693067725926336/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513083.706904:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513083.706908:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513083.706909:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:26.0:1616513083.706911:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf87980 x1693067725926336/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513083.706924:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf87980 x1693067725926336/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513083.706933:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa08 00000800:00000200:7.2:1616513083.707299:0:49837:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.707313:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.707320:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:6.0:1616513083.707322:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513083.707327:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.78@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.707335:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.78@tcp1 of length 224 into portal 26 MB=0x60214fa16fc00 00000400:00000200:26.0:1616513083.707341:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.78@tcp1 of length 224/224 into md 0xf97ca629 [1] + 0 00000400:00000200:26.0:1616513083.707346:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43540 00000400:00000200:26.0:1616513083.707348:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.707351:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.707356:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43540 00000800:00000200:26.0:1616513083.707358:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.707361:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.707362:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.707422:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691138978675712 00010000:00000200:16.0:1616513083.707447:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1691138978675712/t0(0) o400->00514737-946b-e2b2-39ff-3a8ebd6815ad@10.31.141.78@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.707461:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691138978675712, offset 192 00000400:00000200:16.0:1616513083.707468:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.78@tcp1 00000400:00000200:16.0:1616513083.707475:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.78@tcp1 routed destination 00000400:00000200:16.0:1616513083.707490:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.78@tcp1(10.31.141.78@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.707496:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.707502:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.707505:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.707507:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.707512:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513083.707519:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513083.707556:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.707612:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.707619:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513083.707621:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513083.707628:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513083.707634:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3dc0 00000800:00000200:12.0:1616513083.707637:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.707640:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.707786:0:49837:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513083.707799:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.707807:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513083.707808:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513083.707815:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.78@tcp1 : PUT - for me 00000400:00000200:2.0:1616513083.707826:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.78@tcp1 of length 224 into portal 12 MB=0x60214fa16ffc0 00000400:00000200:2.0:1616513083.707833:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.78@tcp1 of length 224/224 into md 0xf964d519 [1] + 30344 00000400:00000200:2.0:1616513083.707840:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513083.707842:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513083.707849:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.707852:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.707854:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.707915:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691138978676672 00010000:00000200:18.0:1616513083.707927:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d3a80 x1691138978676672/t0(0) o400->7e63e327-a184-64d9-5b9c-585b9ad07bda@10.31.141.78@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.707941:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691138978676672, offset 192 00000400:00000200:18.0:1616513083.707947:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.78@tcp1 00000400:00000200:18.0:1616513083.707955:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.78@tcp1 routed destination 00000400:00000200:18.0:1616513083.707969:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.78@tcp1(10.31.141.78@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.707976:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513083.707982:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.707984:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513083.707987:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513083.707991:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.707995:0:49837:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513083.708003:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513083.708006:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513083.708007:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513083.708011:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:6.0:1616513083.708011:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513083.708014:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdf58 00000800:00000200:6.0:1616513083.708016:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.709449:0:49837:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513083.709504:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.709513:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513083.709522:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.239@tcp1 : PUT - for me 00000400:00000200:14.0:1616513083.709530:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.239@tcp1 of length 224 into portal 26 MB=0x5fff603a49e80 00000400:00000200:14.0:1616513083.709536:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.239@tcp1 of length 224/224 into md 0xf97ca649 [1] + 0 00000400:00000200:14.0:1616513083.709541:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43078 00000400:00000200:14.0:1616513083.709544:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.709546:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513083.709551:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43078 00000800:00000200:14.0:1616513083.709554:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.709556:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513083.709557:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513083.709561:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.709622:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688806971711104 00010000:00000200:16.0:1616513083.709633:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688806971711104/t0(0) o400->cddffccc-bc97-514b-0633-bbcf1dd99a02@10.31.141.239@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.709647:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688806971711104, offset 192 00000400:00000200:16.0:1616513083.709653:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.239@tcp1 00000400:00000200:16.0:1616513083.709660:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.239@tcp1 routed destination 00000400:00000200:16.0:1616513083.709675:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.239@tcp1(10.31.141.239@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.709682:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.709687:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.709690:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.709692:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.709697:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513083.709703:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513083.709704:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.709769:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.709776:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.709779:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513083.709786:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.709791:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3078 00000800:00000200:20.0:1616513083.709795:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.709798:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.721810:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.721867:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.721879:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513083.721886:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513083.721888:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.193@tcp1 : PUT - for me 00000400:00000200:12.0:1616513083.721896:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.193@tcp1 of length 224 into portal 26 MB=0x5ffdd65d6bf00 00000400:00000200:12.0:1616513083.721903:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.193@tcp1 of length 224/224 into md 0xf97ca659 [1] + 0 00000400:00000200:12.0:1616513083.721908:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42cc0 00000400:00000200:12.0:1616513083.721911:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513083.721914:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513083.721919:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42cc0 00000800:00000200:12.0:1616513083.721922:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.721924:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.721926:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.721982:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701244980992 00010000:00000200:16.0:1616513083.721993:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688701244980992/t0(0) o400->1b49c2ff-a78f-bc1e-6a7b-07b0564eeece@10.31.132.193@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.722007:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701244980992, offset 192 00000400:00000200:16.0:1616513083.722013:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.193@tcp1 00000400:00000200:16.0:1616513083.722020:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.193@tcp1 routed destination 00000400:00000200:16.0:1616513083.722036:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.193@tcp1(10.31.132.193@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.722042:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.722048:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.722051:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.722054:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.722059:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513083.722066:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.722066:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.722122:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.722129:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.722133:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:6.0:1616513083.722133:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.722140:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.722146:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b25d8 00000800:00000200:26.0:1616513083.722149:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.722580:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.722637:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.722646:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.722655:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.193@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.722663:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.193@tcp1 of length 224 into portal 12 MB=0x5ffdd65d6bfc0 00000400:00000200:20.0:1616513083.722670:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.193@tcp1 of length 224/224 into md 0x865bbec1 [1] + 63728 00000400:00000200:20.0:1616513083.722675:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.722677:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513083.722684:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.722687:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.722689:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.722692:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513083.722749:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701244981184 00010000:00000200:24.0:1616513083.722762:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ced100 x1688701244981184/t0(0) o400->654928f6-d531-26dd-41d2-4adccf07be0e@10.31.132.193@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513083.722776:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701244981184, offset 192 00000400:00000200:24.0:1616513083.722783:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.193@tcp1 00000400:00000200:24.0:1616513083.722798:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.193@tcp1 routed destination 00000400:00000200:24.0:1616513083.722813:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.193@tcp1(10.31.132.193@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.722820:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513083.722825:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.722828:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513083.722830:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513083.722835:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.722840:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.722852:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.722857:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.722859:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513083.722861:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.722866:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.722870:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2908 00000800:00000200:28.0:1616513083.722874:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.736643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.736699:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.736709:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.736719:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.98@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.736729:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.98@tcp1 of length 224 into portal 26 MB=0x601960c77e7c0 00000400:00000200:20.0:1616513083.736735:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.98@tcp1 of length 224/224 into md 0xf97ca669 [1] + 0 00000400:00000200:20.0:1616513083.736740:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf870 00000400:00000200:20.0:1616513083.736743:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.736746:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513083.736750:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf870 00000800:00000200:20.0:1616513083.736753:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.736754:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.736756:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.736761:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.736764:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.736765:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.736774:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.98@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.736784:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.98@tcp1 of length 224 into portal 12 MB=0x601960c77e840 00000400:00000200:0.0:1616513083.736790:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.98@tcp1 of length 224/224 into md 0x865bbec1 [1] + 63952 00000400:00000200:0.0:1616513083.736795:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.736800:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.736807:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.736810:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513083.736812:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690593826170816 00000800:00000200:0.0:1616513083.736812:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:16.0:1616513083.736823:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1690593826170816/t0(0) o400->e1d8fe91-6fde-db5e-8903-c1f7986edfb1@10.31.140.98@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.736836:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690593826170816, offset 192 00000400:00000200:16.0:1616513083.736842:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.98@tcp1 00000400:00000200:16.0:1616513083.736849:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.98@tcp1 routed destination 00000400:00000200:16.0:1616513083.736863:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.98@tcp1(10.31.140.98@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.736869:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:24.0:1616513083.736872:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690593826170944 00000800:00000200:16.0:1616513083.736875:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.736877:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.736879:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:24.0:1616513083.736884:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86e780 x1690593826170944/t0(0) o400->e9d68f46-0010-ea1d-33fb-c03c805aa2cc@10.31.140.98@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.736884:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513083.736891:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.736891:0:49840:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513083.736899:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690593826170944, offset 192 00000400:00000200:24.0:1616513083.736905:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.98@tcp1 00000400:00000200:24.0:1616513083.736912:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.98@tcp1 routed destination 00000400:00000200:24.0:1616513083.736928:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.98@tcp1(10.31.140.98@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.736934:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513083.736940:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.736943:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513083.736945:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513083.736945:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513083.736950:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513083.736951:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.736954:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513083.736955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513083.736960:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.736966:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3cb0 00000800:00000200:2.0:1616513083.736969:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.736972:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513083.737020:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.737027:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.737030:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513083.737031:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.737037:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.737041:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:28.0:1616513083.737045:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.739915:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513083.739975:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.739985:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513083.739994:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.30@tcp1 : PUT - for me 00000400:00000200:0.0:1616513083.740003:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.30@tcp1 of length 224 into portal 26 MB=0x5ffd3d25c2140 00000400:00000200:0.0:1616513083.740010:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.30@tcp1 of length 224/224 into md 0xf97ca689 [1] + 0 00000400:00000200:0.0:1616513083.740015:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdfc28 00000400:00000200:0.0:1616513083.740017:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.740020:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513083.740024:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfc28 00000800:00000200:0.0:1616513083.740028:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.740030:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.740030:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.740036:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513083.740039:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.740040:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513083.740048:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.30@tcp1 : PUT - for me 00000400:00000200:12.0:1616513083.740058:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.30@tcp1 of length 224 into portal 12 MB=0x5ffd3d25c2240 00000400:00000200:12.0:1616513083.740064:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.30@tcp1 of length 224/224 into md 0x865bbec1 [1] + 64176 00000400:00000200:12.0:1616513083.740069:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513083.740072:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513083.740078:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.740081:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.740083:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.740090:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660115988800 00010000:00000200:16.0:1616513083.740101:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688660115988800/t0(0) o400->71bb1550-62d3-7906-741c-901d82c3b37c@10.31.138.30@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.740115:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660115988800, offset 192 00000400:00000200:16.0:1616513083.740124:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.30@tcp1 00000400:00000200:16.0:1616513083.740131:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.30@tcp1 routed destination 00000100:00000200:24.0:1616513083.740138:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660115989056 00000400:00000200:16.0:1616513083.740146:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.30@tcp1(10.31.138.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:24.0:1616513083.740150:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dc800 x1688660115989056/t0(0) o400->1f40472f-282c-0ffa-54d6-15e23b69c8a8@10.31.138.30@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.740153:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.740159:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.740161:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513083.740163:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660115989056, offset 192 00000800:00000200:16.0:1616513083.740163:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.740168:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513083.740169:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.30@tcp1 00000100:00000200:16.0:1616513083.740175:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513083.740176:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.30@tcp1 routed destination 00000800:00000200:7.2:1616513083.740176:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513083.740187:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513083.740190:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.30@tcp1(10.31.138.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513083.740192:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513083.740194:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513083.740196:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:6.0:1616513083.740199:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513083.740201:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:6.0:1616513083.740203:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3980 00000800:00000200:24.0:1616513083.740204:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513083.740206:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513083.740206:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513083.740208:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513083.740210:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.740215:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.740227:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.740234:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.740237:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513083.740238:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513083.740243:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.740247:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2aa0 00000800:00000200:28.0:1616513083.740251:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.751641:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.751696:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.751705:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513083.751714:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.15@tcp1 : PUT - for me 00000400:00000200:12.0:1616513083.751723:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.15@tcp1 of length 224 into portal 26 MB=0x6018490129a80 00000400:00000200:12.0:1616513083.751729:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.15@tcp1 of length 224/224 into md 0xf97ca6a9 [1] + 0 00000400:00000200:12.0:1616513083.751734:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde660 00000400:00000200:12.0:1616513083.751737:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513083.751739:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513083.751744:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde660 00000800:00000200:12.0:1616513083.751747:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.751748:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.751751:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513083.751753:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.751787:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513083.751807:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690518724713088 00010000:00000200:16.0:1616513083.751818:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1690518724713088/t0(0) o400->2ad6b9e6-1cda-9a85-d65d-a13c85c95ec2@10.31.132.15@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.751833:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690518724713088, offset 192 00000400:00000200:16.0:1616513083.751839:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.15@tcp1 00000400:00000200:16.0:1616513083.751847:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.15@tcp1 routed destination 00000800:00000200:0.0:1616513083.751859:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513083.751865:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.15@tcp1(10.31.132.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513083.751869:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513083.751871:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513083.751872:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513083.751877:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.15@tcp1 : PUT - for me 00000800:00000200:16.0:1616513083.751878:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.751880:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.751883:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513083.751886:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.15@tcp1 of length 224 into portal 12 MB=0x6018490129bc0 00000800:00000200:16.0:1616513083.751887:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513083.751892:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.15@tcp1 of length 224/224 into md 0x865bbec1 [1] + 64400 00000100:00000200:16.0:1616513083.751894:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513083.751894:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513083.751898:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513083.751900:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513083.751907:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513083.751910:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.751912:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513083.751951:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.751958:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.751960:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.751967:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.751973:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3650 00000800:00000200:2.0:1616513083.751976:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.751979:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.751985:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000100:00000200:24.0:1616513083.751986:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690518724713408 00000800:00000200:6.0:1616513083.751986:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.751987:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.751989:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.751991:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513083.751998:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86c800 x1690518724713408/t0(0) o400->3736a6c2-58c6-83aa-ce74-f8388a30c156@10.31.132.15@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513083.752011:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690518724713408, offset 192 00000400:00000200:24.0:1616513083.752017:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.15@tcp1 00000400:00000200:24.0:1616513083.752023:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.15@tcp1 routed destination 00000400:00000200:24.0:1616513083.752039:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.15@tcp1(10.31.132.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.752045:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513083.752050:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.752053:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513083.752055:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513083.752059:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.752063:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.752074:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.752081:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.752083:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513083.752084:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.752090:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.752094:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:20.0:1616513083.752098:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.767658:0:49843:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.767721:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.767731:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:6.0:1616513083.767733:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513083.767740:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.99@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.767749:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.99@tcp1 of length 224 into portal 26 MB=0x60182ae5d2b40 00000400:00000200:26.0:1616513083.767756:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.99@tcp1 of length 224/224 into md 0xf97ca6b9 [1] + 0 00000400:00000200:26.0:1616513083.767761:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf540 00000400:00000200:26.0:1616513083.767764:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.767767:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513083.767771:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf540 00000800:00000200:26.0:1616513083.767774:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.767777:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.767779:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.767836:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510642981696 00010000:00000200:16.0:1616513083.767846:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1690510642981696/t0(0) o400->24ac9e3a-518e-8ed4-f00b-b2e1b292f520@10.31.140.99@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.767859:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510642981696, offset 192 00000400:00000200:16.0:1616513083.767865:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.99@tcp1 00000400:00000200:16.0:1616513083.767872:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.99@tcp1 routed destination 00000800:00000200:11.2:1616513083.767879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:16.0:1616513083.767886:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.99@tcp1(10.31.140.99@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513083.767890:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.767892:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.767899:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513083.767900:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.767901:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.767903:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.767908:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513083.767909:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.99@tcp1 : PUT - for me 00000800:00000200:7.2:1616513083.767913:0:49843:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.767914:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513083.767918:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.99@tcp1 of length 224 into portal 12 MB=0x60182ae5d3180 00000400:00000200:28.0:1616513083.767924:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.99@tcp1 of length 224/224 into md 0x865bbec1 [1] + 64624 00000400:00000200:28.0:1616513083.767930:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.767932:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513083.767939:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.767942:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.767944:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513083.767946:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513083.767970:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.767977:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.767980:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.767986:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.767992:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3f58 00000800:00000200:2.0:1616513083.767995:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.767998:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513083.768004:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510642983296 00010000:00000200:24.0:1616513083.768015:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed8900 x1690510642983296/t0(0) o400->80479aa9-a9f5-d9b6-497c-ab1dcddaeae1@10.31.140.99@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513083.768029:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510642983296, offset 192 00000400:00000200:24.0:1616513083.768035:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.99@tcp1 00000400:00000200:24.0:1616513083.768041:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.99@tcp1 routed destination 00000400:00000200:24.0:1616513083.768055:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.99@tcp1(10.31.140.99@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.768061:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513083.768066:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.768069:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513083.768071:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513083.768075:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.768080:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.768132:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.768139:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513083.768141:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513083.768144:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513083.768147:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513083.768151:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf24c8 00000800:00000200:12.0:1616513083.768155:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.772581:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513083.772596:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513083.772606:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:6.0:1616513083.772615:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.122@tcp1 : PUT - for me 00000400:00000200:6.0:1616513083.772624:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.122@tcp1 of length 224 into portal 26 MB=0x60020aff926c0 00000400:00000200:6.0:1616513083.772630:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.122@tcp1 of length 224/224 into md 0xf97ca6c9 [1] + 0 00000400:00000200:6.0:1616513083.772635:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde990 00000400:00000200:6.0:1616513083.772638:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513083.772641:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:6.0:1616513083.772645:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde990 00000800:00000200:6.0:1616513083.772648:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.772650:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513083.772651:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513083.772654:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.772709:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688990251558592 00010000:00000200:16.0:1616513083.772720:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688990251558592/t0(0) o400->fbb2ed68-3869-a3d2-c20e-c6915bdae1e2@10.31.142.122@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.772734:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688990251558592, offset 192 00000400:00000200:16.0:1616513083.772741:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.122@tcp1 00000400:00000200:16.0:1616513083.772749:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.122@tcp1 routed destination 00000400:00000200:16.0:1616513083.772764:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.122@tcp1(10.31.142.122@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.772770:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.772776:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.772779:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.772781:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.772785:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.772790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.772792:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513083.772842:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.772849:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.772852:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.772858:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.772866:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2b28 00000800:00000200:2.0:1616513083.772869:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.772872:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.773701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513083.773711:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.773720:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.773729:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.122@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.773738:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.122@tcp1 of length 224 into portal 12 MB=0x60020aff92c40 00000400:00000200:26.0:1616513083.773745:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.122@tcp1 of length 224/224 into md 0xf964d519 [1] + 30568 00000400:00000200:26.0:1616513083.773750:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.773753:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513083.773760:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.773763:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513083.773763:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513083.773767:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.773826:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688990251560000 00010000:00000200:18.0:1616513083.773839:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57da00 x1688990251560000/t0(0) o400->7c83726b-5762-e1e2-c9d8-bc73555f4210@10.31.142.122@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.773854:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688990251560000, offset 192 00000400:00000200:18.0:1616513083.773860:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.122@tcp1 00000400:00000200:18.0:1616513083.773867:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.122@tcp1 routed destination 00000400:00000200:18.0:1616513083.773882:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.122@tcp1(10.31.142.122@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.773889:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513083.773894:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.773897:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.773899:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513083.773904:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.773911:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.773967:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.773973:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.773976:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.773983:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.773987:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdf58 00000800:00000200:28.0:1616513083.773991:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.773994:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513083.786599:0:6484:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926352, portal 4 00000100:00000200:20.0:1616513083.786605:0:6484:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926352, offset 0 00000400:00000200:20.0:1616513083.786612:0:6484:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:20.0:1616513083.786618:0:6484:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:20.0:1616513083.786631:0:6484:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513083.786638:0:6484:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:20.0:1616513083.786644:0:6484:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513083.786647:0:6484:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:20.0:1616513083.786649:0:6484:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:20.0:1616513083.786654:0:6484:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513083.786700:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513083.786757:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513083.786764:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513083.786767:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.786774:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513083.786780:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cbeedc800 x1693067725926352/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513083.786792:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a908 00000800:00000200:2.0:1616513083.786796:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513083.786799:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513083.786880:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513083.786935:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513083.786945:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:10.0:1616513083.786954:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:10.0:1616513083.786962:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c626fd0 00000400:00000200:10.0:1616513083.786968:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97caca9 [1] + 192 00000400:00000200:10.0:1616513083.786973:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513083.786978:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cbeedc800 x1693067725926352/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513083.786993:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513083.786993:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:10.0:1616513083.786996:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:10.0:1616513083.786998:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:20.0:1616513083.787001:0:6484:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cbeedc800 x1693067725926352/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513083.787015:0:6484:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cbeedc800 x1693067725926352/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513083.787024:0:6484:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bba0 00000100:00000200:11.0:1616513083.803322:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926368, portal 4 00000100:00000200:11.0:1616513083.803326:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926368, offset 0 00000400:00000200:11.0:1616513083.803331:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:11.0:1616513083.803336:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:11.0:1616513083.803346:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513083.803351:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:11.0:1616513083.803357:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513083.803359:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:11.0:1616513083.803361:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:11.0:1616513083.803365:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513083.803412:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513083.803432:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513083.803457:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513083.803459:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513083.803466:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513083.803472:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be42d00 x1693067725926368/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:3.0:1616513083.803483:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513083.803485:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3b320 00000800:00000200:27.0:1616513083.803489:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513083.803586:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513083.803596:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513083.803602:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513083.803610:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513083.803617:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c626fe0 00000400:00000200:27.0:1616513083.803624:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166c9d [1] + 192 00000400:00000200:27.0:1616513083.803629:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513083.803633:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be42d00 x1693067725926368/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513083.803647:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513083.803648:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513083.803650:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000100:00000200:11.0:1616513083.803652:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be42d00 x1693067725926368/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513083.803653:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:11.0:1616513083.803661:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be42d00 x1693067725926368/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513083.803668:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bc28 00000800:00000200:11.2:1616513083.831326:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.831401:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.831412:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.831422:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.95@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.831447:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.95@tcp1 of length 224 into portal 26 MB=0x5ffe18b8396c0 00000400:00000200:20.0:1616513083.831455:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.95@tcp1 of length 224/224 into md 0xf97ca6d9 [1] + 0 00000800:00000200:12.0:1616513083.831457:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513083.831460:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdeff0 00000400:00000200:20.0:1616513083.831463:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.831466:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513083.831468:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.831471:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdeff0 00000800:00000200:28.0:1616513083.831473:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.831475:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:12.0:1616513083.831477:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.95@tcp1 : PUT - for me 00000800:00000200:20.0:1616513083.831478:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.831482:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513083.831487:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.95@tcp1 of length 224 into portal 12 MB=0x5ffe18b839740 00000400:00000200:12.0:1616513083.831493:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.95@tcp1 of length 224/224 into md 0x865bbec1 [1] + 64848 00000400:00000200:12.0:1616513083.831499:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513083.831502:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513083.831509:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.831512:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.831514:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.831538:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688719056934592 00010000:00000200:16.0:1616513083.831551:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688719056934592/t0(0) o400->219faea0-82f7-0c92-b5d9-e7763e5a18fd@10.31.141.95@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.831566:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688719056934592, offset 192 00000400:00000200:16.0:1616513083.831573:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.95@tcp1 00000100:00000200:24.0:1616513083.831575:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688719056934720 00000400:00000200:16.0:1616513083.831581:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.95@tcp1 routed destination 00010000:00000200:24.0:1616513083.831589:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d9200 x1688719056934720/t0(0) o400->3c729296-7a48-6265-3792-3b5c19ea0c39@10.31.141.95@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513083.831597:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.95@tcp1(10.31.141.95@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513083.831604:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688719056934720, offset 192 00000800:00000200:16.0:1616513083.831604:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.831610:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513083.831611:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.95@tcp1 00000800:00000200:16.0:1616513083.831613:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.831615:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513083.831619:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.95@tcp1 routed destination 00000800:00000200:16.0:1616513083.831621:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.831629:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.831630:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513083.831634:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.95@tcp1(10.31.141.95@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.831641:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513083.831647:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.831649:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513083.831653:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513083.831658:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.831663:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513083.831687:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.831694:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.831697:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513083.831700:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513083.831704:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.831709:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b37e8 00000800:00000200:2.0:1616513083.831713:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513083.831718:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513083.831726:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513083.831728:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513083.831729:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513083.831735:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.831741:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000800:00000200:0.0:1616513083.831745:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:3.0:1616513083.834615:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926384, portal 4 00000100:00000200:3.0:1616513083.834620:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926384, offset 0 00000400:00000200:3.0:1616513083.834626:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:3.0:1616513083.834632:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:3.0:1616513083.834644:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513083.834651:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513083.834657:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513083.834659:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:3.0:1616513083.834662:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:3.0:1616513083.834666:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:4.0:1616513083.834669:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926400, portal 4 00000100:00000200:4.0:1616513083.834675:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926400, offset 0 00000400:00000200:4.0:1616513083.834682:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:4.0:1616513083.834688:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:4.0:1616513083.834701:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513083.834708:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:5.2:1616513083.834713:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513083.834714:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513083.834717:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:4.0:1616513083.834721:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:4.0:1616513083.834727:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:20.0:1616513083.834728:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513083.834735:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513083.834738:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513083.834744:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513083.834751:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc567080 x1693067725926384/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513083.834763:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd36d8 00000800:00000200:20.0:1616513083.834767:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513083.834770:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:25.2:1616513083.834775:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513083.834831:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513083.834839:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513083.834841:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.834848:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513083.834854:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cde868480 x1693067725926400/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513083.834868:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba2a8 00000800:00000200:26.0:1616513083.834872:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513083.834875:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:5.2:1616513083.834886:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:25.2:1616513083.834939:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:28.0:1616513083.834940:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513083.834949:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:0.0:1616513083.834952:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513083.834958:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513083.834966:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c626ff0 00000400:00000200:28.0:1616513083.834973:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d78e1 [1] + 192 00000400:00000200:28.0:1616513083.834978:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.834984:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc567080 x1693067725926384/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513083.834993:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:28.0:1616513083.834999:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513083.835002:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:14.0:1616513083.835002:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:28.0:1616513083.835004:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:3.0:1616513083.835007:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc567080 x1693067725926384/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513083.835012:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000100:00000200:3.0:1616513083.835019:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc567080 x1693067725926384/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513083.835020:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627000 00000400:00000200:14.0:1616513083.835027:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cacc9 [1] + 192 00000400:00000200:3.0:1616513083.835030:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000400:00000200:14.0:1616513083.835032:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.835037:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cde868480 x1693067725926400/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513083.835047:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513083.835052:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513083.835055:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513083.835056:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:4.0:1616513083.835060:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cde868480 x1693067725926400/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513083.835073:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cde868480 x1693067725926400/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513083.835083:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00000800:00000200:7.2:1616513083.845667:0:49852:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513083.845669:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513083.845724:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.845733:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.845743:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.86@tcp1 : PUT - for me 00000800:00000200:20.0:1616513083.845745:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513083.845752:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.86@tcp1 of length 224 into portal 12 MB=0x5ffd4ecbbdb40 00000800:00000200:20.0:1616513083.845754:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513083.845759:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.86@tcp1 of length 224/224 into md 0xf964d519 [1] + 30792 00000400:00000200:26.0:1616513083.845764:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513083.845764:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.86@tcp1 : PUT - for me 00000100:00000200:26.0:1616513083.845767:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513083.845772:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.86@tcp1 of length 224 into portal 26 MB=0x5ffd4ecbbdac0 00000800:00000200:26.0:1616513083.845774:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513083.845777:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513083.845778:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513083.845779:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.86@tcp1 of length 224/224 into md 0xf97ca6f9 [1] + 0 00000800:00000200:26.0:1616513083.845781:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513083.845785:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde7f8 00000400:00000200:20.0:1616513083.845787:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.845790:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513083.845795:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde7f8 00000800:00000200:20.0:1616513083.845798:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.845800:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.845801:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.845804:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513083.845843:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664853437248 00010000:00000200:18.0:1616513083.845857:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57c800 x1688664853437248/t0(0) o400->78baa57b-d7d7-9b14-3dd3-ab495904e4ed@10.31.131.86@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513083.845862:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664853437120 00000100:00000200:18.0:1616513083.845871:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664853437248, offset 192 00010000:00000200:16.0:1616513083.845874:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688664853437120/t0(0) o400->062acd59-9faa-5557-5d4a-0681ecd437c2@10.31.131.86@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513083.845877:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.86@tcp1 00000400:00000200:18.0:1616513083.845885:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.86@tcp1 routed destination 00000100:00000200:16.0:1616513083.845889:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664853437120, offset 192 00000400:00000200:16.0:1616513083.845895:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.86@tcp1 00000400:00000200:18.0:1616513083.845900:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.86@tcp1(10.31.131.86@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513083.845902:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.86@tcp1 routed destination 00000800:00000200:18.0:1616513083.845906:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513083.845912:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.845915:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513083.845917:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513083.845917:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.86@tcp1(10.31.131.86@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.845922:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513083.845924:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513083.845926:0:49852:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.845930:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.845932:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.845935:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.845940:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.845944:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513083.845946:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513083.845979:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.845986:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.845995:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513083.846001:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513083.846001:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513083.846001:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513083.846006:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cded0 00000800:00000200:0.0:1616513083.846008:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513083.846010:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513083.846011:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513083.846014:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513083.846018:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513083.846023:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3870 00000800:00000200:0.0:1616513083.846027:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513083.866575:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926416, portal 4 00000100:00000200:19.0:1616513083.866580:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926416, offset 0 00000400:00000200:19.0:1616513083.866587:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:19.0:1616513083.866593:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:19.0:1616513083.866606:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513083.866613:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:19.0:1616513083.866619:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513083.866622:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:19.0:1616513083.866624:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:19.0:1616513083.866629:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513083.866677:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.866696:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513083.866701:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513083.866704:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513083.866711:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513083.866717:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be41b00 x1693067725926416/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513083.866729:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17074b8 00000800:00000200:1.0:1616513083.866734:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513083.866736:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513083.866878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513083.866886:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.866892:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513083.866896:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513083.866901:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513083.866909:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627010 00000400:00000200:1.0:1616513083.866915:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffce5 [1] + 192 00000400:00000200:1.0:1616513083.866920:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513083.866924:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be41b00 x1693067725926416/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513083.866938:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.866940:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513083.866942:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:19.0:1616513083.866945:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be41b00 x1693067725926416/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513083.866957:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be41b00 x1693067725926416/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513083.866967:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000400:00000200:27.0:1616513083.870668:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:27.0:1616513083.870677:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513083.874074:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513083.874129:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.874139:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513083.874149:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.7@tcp1 : PUT - for me 00000400:00000200:12.0:1616513083.874158:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.7@tcp1 of length 224 into portal 26 MB=0x604b9adf0f480 00000400:00000200:12.0:1616513083.874165:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.7@tcp1 of length 224/224 into md 0xf97ca709 [1] + 0 00000400:00000200:12.0:1616513083.874170:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde6e8 00000400:00000200:12.0:1616513083.874173:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513083.874176:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513083.874181:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde6e8 00000800:00000200:20.0:1616513083.874183:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.874184:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.874187:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.874189:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.874191:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513083.874249:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694045393974400 00000800:00000200:28.0:1616513083.874252:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:16.0:1616513083.874261:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1694045393974400/t0(0) o400->26b420df-f7b8-576e-8ca5-8c78cdd9cd15@10.31.141.7@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513083.874263:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513083.874274:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.7@tcp1 : PUT - for me 00000100:00000200:16.0:1616513083.874275:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694045393974400, offset 192 00000400:00000200:16.0:1616513083.874282:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.7@tcp1 00000400:00000200:28.0:1616513083.874283:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.7@tcp1 of length 224 into portal 12 MB=0x604b9adf0f740 00000400:00000200:28.0:1616513083.874289:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.7@tcp1 of length 224/224 into md 0x865bbec1 [1] + 65072 00000400:00000200:16.0:1616513083.874290:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.7@tcp1 routed destination 00000400:00000200:28.0:1616513083.874295:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.874297:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513083.874305:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513083.874306:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.7@tcp1(10.31.141.7@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513083.874308:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.874310:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513083.874312:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513083.874313:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513083.874320:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.874323:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.874325:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.874330:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.874337:0:49855:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.874338:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513083.874371:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694045393975104 00010000:00000200:24.0:1616513083.874385:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedb180 x1694045393975104/t0(0) o400->b2ebb576-0e9a-63f8-a752-3ceda8ae0761@10.31.141.7@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513083.874399:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694045393975104, offset 192 00000800:00000200:10.0:1616513083.874405:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513083.874406:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.7@tcp1 00000800:00000200:10.0:1616513083.874412:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:24.0:1616513083.874413:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.7@tcp1 routed destination 00000400:00000200:10.0:1616513083.874415:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513083.874421:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513083.874426:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b23b8 00000400:00000200:24.0:1616513083.874429:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.7@tcp1(10.31.141.7@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513083.874430:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513083.874432:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513083.874448:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513083.874454:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.874457:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513083.874459:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513083.874464:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.874469:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.874479:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.874484:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513083.874487:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513083.874489:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513083.874494:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513083.874497:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2b28 00000800:00000200:20.0:1616513083.874501:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.912446:0:49859:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513083.912503:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513083.912512:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513083.912522:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.245@tcp1 : PUT - for me 00000400:00000200:14.0:1616513083.912532:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.245@tcp1 of length 224 into portal 12 MB=0x6009cce629340 00000400:00000200:14.0:1616513083.912539:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.245@tcp1 of length 224/224 into md 0xf964d519 [1] + 31016 00000400:00000200:14.0:1616513083.912545:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.912548:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513083.912555:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.912558:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513083.912559:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513083.912566:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513083.912569:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.912570:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513083.912578:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.245@tcp1 : PUT - for me 00000400:00000200:2.0:1616513083.912587:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.245@tcp1 of length 224 into portal 26 MB=0x6009cce6292c0 00000400:00000200:2.0:1616513083.912594:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.245@tcp1 of length 224/224 into md 0xf97ca729 [1] + 0 00000400:00000200:2.0:1616513083.912599:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf980 00000400:00000200:2.0:1616513083.912602:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513083.912604:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513083.912611:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf980 00000800:00000200:2.0:1616513083.912614:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.912617:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.912619:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.912622:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689523337728832 00010000:00000200:18.0:1616513083.912635:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623a80 x1689523337728832/t0(0) o400->85b276ef-905e-6e4c-dab0-6576f44f16b6@10.31.141.245@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.912650:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689523337728832, offset 192 00000400:00000200:18.0:1616513083.912656:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.245@tcp1 00000400:00000200:18.0:1616513083.912664:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.245@tcp1 routed destination 00000100:00000200:16.0:1616513083.912677:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689523337728704 00000400:00000200:18.0:1616513083.912679:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.245@tcp1(10.31.141.245@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.912685:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:16.0:1616513083.912688:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1689523337728704/t0(0) o400->988933d0-6932-6dd7-6859-e3d64454a9ff@10.31.141.245@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513083.912692:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.912694:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513083.912696:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513083.912702:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513083.912702:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689523337728704, offset 192 00000800:00000200:7.2:1616513083.912707:0:49859:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513083.912711:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.245@tcp1 00000400:00000200:16.0:1616513083.912718:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.245@tcp1 routed destination 00000400:00000200:16.0:1616513083.912734:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.245@tcp1(10.31.141.245@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.912740:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.912746:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.912749:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.912751:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.912756:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513083.912763:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:16.0:1616513083.912763:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513083.912770:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.912773:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513083.912773:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513083.912780:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.912784:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc088 00000800:00000200:26.0:1616513083.912788:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.912801:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.912857:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.912864:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.912867:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.912874:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.912879:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3e48 00000800:00000200:28.0:1616513083.912883:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.912886:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513083.930606:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926432, portal 4 00000100:00000200:19.0:1616513083.930612:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926432, offset 0 00000400:00000200:19.0:1616513083.930619:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:19.0:1616513083.930625:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:19.0:1616513083.930639:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513083.930646:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:19.0:1616513083.930652:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513083.930655:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:19.0:1616513083.930658:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:19.0:1616513083.930663:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513083.930711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.930731:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513083.930737:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513083.930739:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513083.930742:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513083.930746:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513083.930753:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456eb600 x1693067725926432/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513083.930765:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707100 00000800:00000200:1.0:1616513083.930770:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513083.930858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513083.930867:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.930874:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513083.930875:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513083.930882:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513083.930892:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627020 00000400:00000200:1.0:1616513083.930899:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffcf5 [1] + 192 00000400:00000200:1.0:1616513083.930904:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513083.930908:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456eb600 x1693067725926432/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513083.930923:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513083.930925:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513083.930927:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:19.0:1616513083.930931:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456eb600 x1693067725926432/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513083.930944:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456eb600 x1693067725926432/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513083.930954:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000100:00000200:19.0:1616513083.931476:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926448, portal 4 00000100:00000200:19.0:1616513083.931482:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926448, offset 0 00000400:00000200:19.0:1616513083.931487:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:19.0:1616513083.931491:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:19.0:1616513083.931503:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513083.931509:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:19.0:1616513083.931514:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513083.931517:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:19.0:1616513083.931519:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:19.0:1616513083.931523:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513083.931569:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513083.931583:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:10.0:1616513083.931591:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:10.0:1616513083.931594:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513083.931601:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:10.0:1616513083.931607:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc566300 x1693067725926448/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513083.931619:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706198 00000800:00000200:10.0:1616513083.931623:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:10.0:1616513083.931626:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513083.931752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513083.931768:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513083.931778:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513083.931787:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513083.931796:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627030 00000400:00000200:14.0:1616513083.931803:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cad29 [1] + 192 00000400:00000200:14.0:1616513083.931808:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513083.931814:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc566300 x1693067725926448/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513083.931822:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513083.931829:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513083.931832:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513083.931834:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:19.0:1616513083.931836:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc566300 x1693067725926448/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513083.931847:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc566300 x1693067725926448/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513083.931856:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000800:00000200:7.2:1616513083.937591:0:49861:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513083.937648:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513083.937658:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513083.937667:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.140@tcp1 : PUT - for me 00000400:00000200:10.0:1616513083.937676:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.140@tcp1 of length 224 into portal 26 MB=0x5ffe21bb3a640 00000400:00000200:10.0:1616513083.937683:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.140@tcp1 of length 224/224 into md 0xf97ca749 [1] + 0 00000400:00000200:10.0:1616513083.937688:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdfba0 00000400:00000200:10.0:1616513083.937691:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513083.937694:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513083.937698:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfba0 00000800:00000200:10.0:1616513083.937702:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513083.937704:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513083.937704:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:10.0:1616513083.937711:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513083.937714:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.937715:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513083.937724:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.140@tcp1 : PUT - for me 00000400:00000200:2.0:1616513083.937733:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.140@tcp1 of length 224 into portal 12 MB=0x5ffe21bb3a6c0 00000400:00000200:2.0:1616513083.937739:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.140@tcp1 of length 224/224 into md 0xf964d519 [1] + 31240 00000400:00000200:2.0:1616513083.937745:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513083.937747:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513083.937755:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.937758:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.937760:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.937765:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688721476003392 00010000:00000200:16.0:1616513083.937777:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688721476003392/t0(0) o400->98cd0e49-1485-8f00-d262-eb5985d2e5b7@10.31.142.140@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.937791:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688721476003392, offset 192 00000400:00000200:16.0:1616513083.937798:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.140@tcp1 00000400:00000200:16.0:1616513083.937806:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.140@tcp1 routed destination 00000100:00000200:18.0:1616513083.937821:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688721476003520 00000400:00000200:16.0:1616513083.937821:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.140@tcp1(10.31.142.140@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.937828:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:18.0:1616513083.937834:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d7080 x1688721476003520/t0(0) o400->84838935-ae7c-7797-416a-25b6df084f03@10.31.142.140@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.937834:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.937837:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513083.937839:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513083.937844:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:18.0:1616513083.937849:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688721476003520, offset 192 00000800:00000200:7.2:1616513083.937850:0:49861:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513083.937852:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513083.937856:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.140@tcp1 00000400:00000200:18.0:1616513083.937863:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.140@tcp1 routed destination 00000800:00000200:14.0:1616513083.937873:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513083.937880:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513083.937881:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.140@tcp1(10.31.142.140@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513083.937883:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513083.937885:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513083.937887:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513083.937890:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513083.937893:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513083.937895:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b36d8 00000800:00000200:18.0:1616513083.937896:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513083.937898:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513083.937899:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:18.0:1616513083.937903:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.937912:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513083.937973:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.937980:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513083.937983:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513083.937990:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513083.937994:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd078 00000800:00000200:12.0:1616513083.937998:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513083.938001:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513083.940998:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513083.941061:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.941070:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513083.941079:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.110.165@tcp1 : PUT - for me 00000400:00000200:2.0:1616513083.941088:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.110.165@tcp1 of length 224 into portal 12 MB=0x5fa07dd43aee8 00000400:00000200:2.0:1616513083.941095:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.110.165@tcp1 of length 224/224 into md 0xf964d519 [1] + 31464 00000400:00000200:2.0:1616513083.941100:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513083.941103:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513083.941109:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.941111:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513083.941113:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.941116:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513083.941172:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682286567468776 00010000:00000200:18.0:1616513083.941183:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d5580 x1682286567468776/t0(0) o400->19c48508-fa3a-d574-22c8-b363e9f96f00@10.242.110.165@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513083.941197:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1682286567468776, offset 192 00000400:00000200:18.0:1616513083.941204:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.110.165@tcp1 00000400:00000200:18.0:1616513083.941211:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.110.165@tcp1 routed destination 00000400:00000200:18.0:1616513083.941226:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.110.165@tcp1(10.242.110.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.941232:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513083.941238:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.941241:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513083.941243:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513083.941247:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.941253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513083.941308:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513083.941315:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513083.941317:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513083.941320:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513083.941324:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513083.941328:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd188 00000800:00000200:10.0:1616513083.941332:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513083.943857:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513083.943912:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.943921:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513083.943930:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.221@tcp1 : PUT - for me 00000400:00000200:2.0:1616513083.943939:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.221@tcp1 of length 224 into portal 26 MB=0x5fff2f504bdc0 00000400:00000200:2.0:1616513083.943946:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.221@tcp1 of length 224/224 into md 0xf97ca759 [1] + 0 00000400:00000200:2.0:1616513083.943951:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde908 00000400:00000200:2.0:1616513083.943954:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513083.943956:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513083.943963:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde908 00000800:00000200:26.0:1616513083.943965:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.943966:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.943968:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513083.943971:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513083.943974:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.943975:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513083.943984:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.221@tcp1 : PUT - for me 00000400:00000200:26.0:1616513083.943993:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.221@tcp1 of length 224 into portal 12 MB=0x5fff2f504be40 00000400:00000200:26.0:1616513083.943999:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.221@tcp1 of length 224/224 into md 0xf964d519 [1] + 31688 00000400:00000200:26.0:1616513083.944004:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513083.944007:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513083.944014:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.944017:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.944019:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513083.944027:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688793841450432 00010000:00000200:16.0:1616513083.944038:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688793841450432/t0(0) o400->07f88a81-13fc-6d6d-b008-32d2624394bf@10.31.141.221@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.944052:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688793841450432, offset 192 00000400:00000200:16.0:1616513083.944058:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.221@tcp1 00000400:00000200:16.0:1616513083.944066:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.221@tcp1 routed destination 00000400:00000200:16.0:1616513083.944080:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.221@tcp1(10.31.141.221@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513083.944084:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688793841450560 00010000:00000200:18.0:1616513083.944096:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57f500 x1688793841450560/t0(0) o400->bdf45c52-4fd3-5c4a-288b-95ee83124167@10.31.141.221@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513083.944096:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.944101:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.944104:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.944106:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:18.0:1616513083.944110:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688793841450560, offset 192 00000800:00000200:16.0:1616513083.944111:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513083.944116:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.221@tcp1 00000100:00000200:16.0:1616513083.944118:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513083.944120:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513083.944124:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.221@tcp1 routed destination 00000400:00000200:18.0:1616513083.944139:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.221@tcp1(10.31.141.221@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513083.944145:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513083.944151:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513083.944154:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513083.944157:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513083.944162:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.944167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513083.944176:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.944183:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.944185:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513083.944192:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513083.944197:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3ba0 00000800:00000200:28.0:1616513083.944201:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.944204:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.944209:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513083.944212:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.944214:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.944215:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.944217:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513083.944217:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513083.944223:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513083.944225:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513083.944227:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513083.944231:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513083.944234:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccee0 00000800:00000200:10.0:1616513083.944239:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.945082:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.945137:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.945146:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513083.945148:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.945153:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.945155:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.238@tcp1 : PUT - for me 00000800:00000200:4.0:1616513083.945157:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513083.945160:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.238@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.945165:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.238@tcp1 of length 224 into portal 26 MB=0x604a2a08533c0 00000400:00000200:28.0:1616513083.945170:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.238@tcp1 of length 224 into portal 12 MB=0x604a2a0853580 00000400:00000200:20.0:1616513083.945172:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.238@tcp1 of length 224/224 into md 0xf97ca779 [1] + 0 00000400:00000200:28.0:1616513083.945176:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.238@tcp1 of length 224/224 into md 0x865bbec1 [1] + 65296 00000400:00000200:20.0:1616513083.945178:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf188 00000400:00000200:28.0:1616513083.945181:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513083.945181:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.945184:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:28.0:1616513083.945185:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513083.945189:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf188 00000800:00000200:28.0:1616513083.945192:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.945193:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513083.945194:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.945196:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.945196:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.945198:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.945202:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946384561088 00010000:00000200:16.0:1616513083.945208:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1693946384561088/t0(0) o400->0985935b-6615-7320-e9de-ba555c0bfe90@10.31.140.238@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.945219:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946384561088, offset 192 00000400:00000200:16.0:1616513083.945224:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.238@tcp1 00000400:00000200:16.0:1616513083.945230:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.238@tcp1 routed destination 00000400:00000200:16.0:1616513083.945243:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.238@tcp1(10.31.140.238@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.945249:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.945253:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.945256:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513083.945257:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946384561536 00000800:00000200:16.0:1616513083.945258:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.945261:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513083.945265:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513083.945266:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513083.945270:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedc800 x1693946384561536/t0(0) o400->f4c92f1d-1f1e-8bec-e891-3787fe7cf134@10.31.140.238@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513083.945277:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513083.945284:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946384561536, offset 192 00000800:00000200:12.0:1616513083.945284:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513083.945287:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513083.945288:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513083.945290:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.238@tcp1 00000400:00000200:12.0:1616513083.945294:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513083.945298:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.238@tcp1 routed destination 00000400:00000200:12.0:1616513083.945299:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b35c8 00000800:00000200:12.0:1616513083.945304:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513083.945313:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.238@tcp1(10.31.140.238@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.945320:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513083.945325:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.945328:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513083.945330:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513083.945335:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.945340:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.945392:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.945400:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.945402:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.945409:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.945412:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2198 00000800:00000200:26.0:1616513083.945416:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.945419:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.973914:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513083.973971:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513083.973981:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513083.973991:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.63@tcp1 : PUT - for me 00000400:00000200:28.0:1616513083.974000:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.63@tcp1 of length 224 into portal 26 MB=0x5fff248e6f0c0 00000400:00000200:28.0:1616513083.974007:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.63@tcp1 of length 224/224 into md 0xf97ca799 [1] + 0 00000400:00000200:28.0:1616513083.974011:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdecc0 00000400:00000200:28.0:1616513083.974015:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.974018:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513083.974022:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdecc0 00000800:00000200:28.0:1616513083.974025:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.974025:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513083.974028:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.974030:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513083.974088:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688790953816256 00010000:00000200:16.0:1616513083.974099:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688790953816256/t0(0) o400->49720526-db8f-b77e-79e5-1615a491916c@10.31.138.63@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513083.974113:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688790953816256, offset 192 00000400:00000200:16.0:1616513083.974121:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.63@tcp1 00000400:00000200:16.0:1616513083.974128:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.63@tcp1 routed destination 00000400:00000200:16.0:1616513083.974143:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.63@tcp1(10.31.138.63@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513083.974150:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513083.974156:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.974159:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513083.974162:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.974168:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.974173:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513083.974174:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513083.974225:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513083.974232:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513083.974235:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513083.974242:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513083.974247:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2880 00000800:00000200:4.0:1616513083.974251:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513083.974254:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.975497:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513083.975509:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513083.975517:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513083.975526:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.63@tcp1 : PUT - for me 00000400:00000200:20.0:1616513083.975534:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.63@tcp1 of length 224 into portal 12 MB=0x5fff248e6f140 00000400:00000200:20.0:1616513083.975541:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.63@tcp1 of length 224/224 into md 0x865bbec1 [1] + 65520 00000400:00000200:20.0:1616513083.975546:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513083.975549:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513083.975556:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513083.975559:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513083.975561:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.975563:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513083.975621:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688790953816384 00010000:00000200:24.0:1616513083.975634:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cea400 x1688790953816384/t0(0) o400->b128aaf5-4247-288d-5614-4edb44b77ff5@10.31.138.63@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513083.975649:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688790953816384, offset 192 00000400:00000200:24.0:1616513083.975656:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.63@tcp1 00000400:00000200:24.0:1616513083.975664:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.63@tcp1 routed destination 00000400:00000200:24.0:1616513083.975679:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.63@tcp1(10.31.138.63@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.975685:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513083.975692:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.975694:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513083.975697:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513083.975702:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.975709:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513083.975776:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513083.975783:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513083.975785:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513083.975791:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513083.975795:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3298 00000800:00000200:2.0:1616513083.975799:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513083.975802:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513083.989587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513083.989603:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513083.989612:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513083.989620:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.253@tcp1 : PUT - for me 00000400:00000200:4.0:1616513083.989630:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.253@tcp1 of length 224 into portal 26 MB=0x5ffe88012b640 00000400:00000200:4.0:1616513083.989636:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.253@tcp1 of length 224/224 into md 0xf97ca7a9 [1] + 0 00000400:00000200:4.0:1616513083.989640:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf100 00000400:00000200:4.0:1616513083.989643:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513083.989646:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513083.989650:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf100 00000800:00000200:4.0:1616513083.989653:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513083.989656:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513083.989657:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513083.989661:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513083.989668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513083.989715:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748929758784 00000800:00000200:28.0:1616513083.989725:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:16.0:1616513083.989726:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688748929758784/t0(0) o400->7cf1eed8-de2a-4d25-eed2-11dd3648b690@10.31.142.253@tcp1:40/0 lens 224/224 e 0 to 0 dl 1616513175 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513083.989734:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:16.0:1616513083.989741:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688748929758784, offset 192 00000400:00000200:28.0:1616513083.989743:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.253@tcp1 : PUT - for me 00000400:00000200:16.0:1616513083.989747:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.253@tcp1 00000400:00000200:28.0:1616513083.989752:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.253@tcp1 of length 224 into portal 12 MB=0x5ffe88012b940 00000400:00000200:16.0:1616513083.989755:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.253@tcp1 routed destination 00000400:00000200:28.0:1616513083.989759:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.253@tcp1 of length 224/224 into md 0x865bbec1 [1] + 65744 00000400:00000200:28.0:1616513083.989764:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513083.989767:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513083.989770:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.253@tcp1(10.31.142.253@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513083.989775:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513083.989777:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513083.989778:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513083.989780:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513083.989783:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513083.989783:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513083.989785:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.989788:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513083.989792:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513083.989797:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513083.989799:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513083.989808:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513083.989814:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513083.989817:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513083.989818:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513083.989823:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513083.989829:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2220 00000800:00000200:12.0:1616513083.989833:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513083.989837:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748929759552 00010000:00000200:24.0:1616513083.989850:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeede300 x1688748929759552/t0(0) o400->e3cc989b-5741-ff09-f0b9-bbb61f1353a2@10.31.142.253@tcp1:709/0 lens 224/224 e 0 to 0 dl 1616513089 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513083.989865:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688748929759552, offset 192 00000400:00000200:24.0:1616513083.989871:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.253@tcp1 00000400:00000200:24.0:1616513083.989878:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.253@tcp1 routed destination 00000400:00000200:24.0:1616513083.989893:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.253@tcp1(10.31.142.253@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513083.989900:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513083.989906:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513083.989909:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513083.989911:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513083.989916:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513083.989924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.989981:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.989989:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513083.989991:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513083.989998:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513083.990002:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:26.0:1616513083.990006:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.990009:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513083.990016:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513083.990018:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513083.990020:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513083.990022:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513083.990024:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.007705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513084.007763:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513084.007773:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513084.007782:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.225@tcp1 : PUT - for me 00000400:00000200:10.0:1616513084.007792:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.225@tcp1 of length 224 into portal 26 MB=0x5ffd4d85d9800 00000400:00000200:10.0:1616513084.007798:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.225@tcp1 of length 224/224 into md 0xf97ca7b9 [1] + 0 00000400:00000200:10.0:1616513084.007803:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdfdc0 00000400:00000200:10.0:1616513084.007806:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513084.007809:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513084.007813:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfdc0 00000800:00000200:10.0:1616513084.007817:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.007818:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513084.007820:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513084.007823:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513084.007894:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664511715328 00010000:00000200:16.0:1616513084.007905:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688664511715328/t0(0) o400->1f19ba16-0a22-7f44-be2d-885fad578593@10.31.132.225@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.007920:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664511715328, offset 192 00000400:00000200:16.0:1616513084.007926:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.225@tcp1 00000400:00000200:16.0:1616513084.007934:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.225@tcp1 routed destination 00000400:00000200:16.0:1616513084.007949:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.225@tcp1(10.31.132.225@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.007956:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.007962:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.007964:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.007966:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.007971:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513084.007978:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.007978:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.008034:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.008040:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.008043:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.008050:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.008055:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2cc0 00000800:00000200:28.0:1616513084.008059:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.008062:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.008519:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.008530:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.008538:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.008547:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.225@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.008554:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.225@tcp1 of length 224 into portal 12 MB=0x5ffd4d85d9980 00000400:00000200:8.0:1616513084.008560:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.225@tcp1 of length 224/224 into md 0x865bbec1 [1] + 65968 00000400:00000200:8.0:1616513084.008564:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.008567:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513084.008573:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.008575:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.008577:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.008579:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.008639:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664511715712 00010000:00000200:24.0:1616513084.008652:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8bf00 x1688664511715712/t0(0) o400->06acc30c-4ce6-c8a0-c0fc-341d1c5810cc@10.31.132.225@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513084.008666:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664511715712, offset 192 00000400:00000200:24.0:1616513084.008673:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.225@tcp1 00000400:00000200:24.0:1616513084.008680:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.225@tcp1 routed destination 00000400:00000200:24.0:1616513084.008695:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.225@tcp1(10.31.132.225@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.008701:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.008708:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.008712:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.008714:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.008719:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.008724:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.008779:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.008786:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.008789:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.008798:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.008802:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000800:00000200:14.0:1616513084.008806:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.008809:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513084.042555:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926464, portal 4 00000100:00000200:2.0:1616513084.042561:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926464, offset 0 00000400:00000200:2.0:1616513084.042568:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:2.0:1616513084.042575:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:2.0:1616513084.042589:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513084.042596:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:2.0:1616513084.042602:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513084.042605:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:2.0:1616513084.042608:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:2.0:1616513084.042614:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513084.042661:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.042681:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513084.042687:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513084.042690:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513084.042697:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513084.042704:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf86300 x1693067725926464/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513084.042716:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec5d8 00000800:00000200:27.0:1616513084.042721:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513084.042723:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513084.042832:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513084.042841:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.042847:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513084.042851:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513084.042855:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513084.042862:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627040 00000400:00000200:27.0:1616513084.042873:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166cfd [1] + 192 00000400:00000200:27.0:1616513084.042879:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513084.042883:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf86300 x1693067725926464/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513084.042898:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.042900:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513084.042902:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:2.0:1616513084.042908:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf86300 x1693067725926464/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513084.042921:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf86300 x1693067725926464/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513084.042932:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec440 00000800:00000200:7.2:1616513084.045911:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.045969:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.045979:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513084.045981:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513084.045989:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.44@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.045999:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.44@tcp1 of length 224 into portal 12 MB=0x60156c1582c80 00000400:00000200:26.0:1616513084.046005:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.44@tcp1 of length 224/224 into md 0xf964d519 [1] + 31912 00000400:00000200:26.0:1616513084.046011:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.046014:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513084.046022:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513084.046024:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.046025:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.046028:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513084.046048:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.046057:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513084.046066:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.44@tcp1 : PUT - for me 00000400:00000200:28.0:1616513084.046075:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.44@tcp1 of length 224 into portal 26 MB=0x60156c1582980 00000400:00000200:28.0:1616513084.046082:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.44@tcp1 of length 224/224 into md 0xf97ca7c9 [1] + 0 00000400:00000200:28.0:1616513084.046087:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde770 00000400:00000200:28.0:1616513084.046090:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513084.046093:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513084.046097:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde770 00000800:00000200:28.0:1616513084.046100:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.046103:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.046105:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513084.046109:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513084.046119:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690321982860416 00010000:00000200:18.0:1616513084.046132:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57d100 x1690321982860416/t0(0) o400->d34c9905-50b3-0d7d-c46c-6965c4d43b88@10.31.140.44@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513084.046147:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690321982860416, offset 192 00000400:00000200:18.0:1616513084.046153:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.44@tcp1 00000400:00000200:18.0:1616513084.046160:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.44@tcp1 routed destination 00000100:00000200:16.0:1616513084.046163:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690321982859648 00010000:00000200:16.0:1616513084.046175:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1690321982859648/t0(0) o400->9aaa6d40-f3d4-b2c2-9ef9-deafd912b8a2@10.31.140.44@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513084.046176:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.44@tcp1(10.31.140.44@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513084.046182:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513084.046188:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513084.046191:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513084.046191:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690321982859648, offset 192 00000800:00000200:18.0:1616513084.046193:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513084.046197:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513084.046197:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.44@tcp1 00000800:00000200:11.2:1616513084.046202:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513084.046205:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.44@tcp1 routed destination 00000800:00000200:8.0:1616513084.046213:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.046219:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513084.046220:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.44@tcp1(10.31.140.44@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513084.046222:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513084.046226:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513084.046228:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:16.0:1616513084.046232:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:8.0:1616513084.046233:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc660 00000800:00000200:16.0:1616513084.046235:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513084.046237:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513084.046237:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.046240:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513084.046243:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.046248:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513084.046250:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513084.046266:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.046273:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.046276:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.046282:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.046288:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3430 00000800:00000200:14.0:1616513084.046291:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.046294:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.067432:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513084.067487:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513084.067496:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513084.067505:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.202@tcp1 : PUT - for me 00000400:00000200:10.0:1616513084.067514:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.202@tcp1 of length 224 into portal 26 MB=0x60186be305440 00000400:00000200:10.0:1616513084.067521:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.202@tcp1 of length 224/224 into md 0xf97ca7d9 [1] + 0 00000400:00000200:10.0:1616513084.067525:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdeaa0 00000400:00000200:10.0:1616513084.067528:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513084.067531:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513084.067535:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdeaa0 00000800:00000200:10.0:1616513084.067538:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.067540:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513084.067541:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513084.067544:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513084.067602:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690528088347712 00000800:00000200:7.2:1616513084.067609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:16.0:1616513084.067615:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1690528088347712/t0(0) o400->8ebb9409-adbc-599f-0f6c-e1566e65bf6e@10.31.142.202@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.067630:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690528088347712, offset 192 00000400:00000200:16.0:1616513084.067637:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.202@tcp1 00000400:00000200:16.0:1616513084.067644:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.202@tcp1 routed destination 00000400:00000200:16.0:1616513084.067660:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.202@tcp1(10.31.142.202@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.067666:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513084.067669:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513084.067672:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.067675:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.067677:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513084.067679:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:16.0:1616513084.067682:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513084.067687:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.202@tcp1 : PUT - for me 00000100:00000200:16.0:1616513084.067689:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.067691:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513084.067696:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.202@tcp1 of length 224 into portal 12 MB=0x60186be306140 00000400:00000200:14.0:1616513084.067703:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.202@tcp1 of length 224/224 into md 0xf964d519 [1] + 32136 00000400:00000200:14.0:1616513084.067708:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.067711:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.067718:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.067721:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.067724:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.067728:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513084.067749:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.067757:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.067760:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.067767:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.067772:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2550 00000800:00000200:20.0:1616513084.067776:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.067779:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513084.067787:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690528088351040 00010000:00000200:18.0:1616513084.067799:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464622d00 x1690528088351040/t0(0) o400->2908e87d-70b5-ee46-25fb-9cb855a8629d@10.31.142.202@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513084.067814:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690528088351040, offset 192 00000400:00000200:18.0:1616513084.067820:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.202@tcp1 00000400:00000200:18.0:1616513084.067828:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.202@tcp1 routed destination 00000400:00000200:18.0:1616513084.067843:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.202@tcp1(10.31.142.202@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513084.067849:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513084.067855:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513084.067858:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.067860:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.067865:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.067870:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.067880:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.067886:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.067888:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513084.067890:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513084.067895:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.067901:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd540 00000800:00000200:26.0:1616513084.067905:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.069398:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.069408:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.069416:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.069424:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.87@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.069433:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.87@tcp1 of length 224 into portal 26 MB=0x5fc50a78ca880 00000400:00000200:8.0:1616513084.069445:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.87@tcp1 of length 224/224 into md 0xf97ca7e9 [1] + 0 00000400:00000200:8.0:1616513084.069450:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf210 00000400:00000200:8.0:1616513084.069452:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.069455:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.069459:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf210 00000800:00000200:12.0:1616513084.069461:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.069461:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.069464:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.069465:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.069484:0:49874:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513084.069526:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1684798222149760 00010000:00000200:16.0:1616513084.069537:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1684798222149760/t0(0) o400->7d4cdbf0-2fd9-5072-309a-74213c449c70@10.31.133.87@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513084.069540:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.069550:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:16.0:1616513084.069552:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1684798222149760, offset 192 00000400:00000200:16.0:1616513084.069558:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.87@tcp1 00000400:00000200:2.0:1616513084.069559:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.87@tcp1 : PUT - for me 00000400:00000200:16.0:1616513084.069566:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.87@tcp1 routed destination 00000400:00000200:2.0:1616513084.069568:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.87@tcp1 of length 224 into portal 12 MB=0x5fc50a78cab00 00000400:00000200:2.0:1616513084.069576:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.87@tcp1 of length 224/224 into md 0xf964d519 [1] + 32360 00000400:00000200:2.0:1616513084.069582:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513084.069583:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.87@tcp1(10.31.133.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:2.0:1616513084.069584:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513084.069589:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513084.069591:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.069594:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513084.069595:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513084.069596:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513084.069598:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.069600:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513084.069600:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513084.069605:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.069609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.069611:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513084.069653:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1684798222150400 00000800:00000200:20.0:1616513084.069664:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:18.0:1616513084.069665:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d2880 x1684798222150400/t0(0) o400->4c34e501-95f3-4e5c-aea6-bcab101df6eb@10.31.133.87@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513084.069671:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.069674:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.069680:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:18.0:1616513084.069680:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1684798222150400, offset 192 00000400:00000200:20.0:1616513084.069686:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2330 00000800:00000200:20.0:1616513084.069689:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513084.069689:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.87@tcp1 00000800:00000200:20.0:1616513084.069694:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513084.069696:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.87@tcp1 routed destination 00000400:00000200:18.0:1616513084.069711:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.87@tcp1(10.31.133.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513084.069717:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513084.069723:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513084.069726:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.069728:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.069733:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.069737:0:49874:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513084.069748:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513084.069754:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513084.069756:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513084.069763:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513084.069766:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd210 00000800:00000200:10.0:1616513084.069770:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513084.069775:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.076259:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.076314:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.076324:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.076333:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.161@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.076342:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.161@tcp1 of length 224 into portal 26 MB=0x5ffd4df82ca00 00000400:00000200:14.0:1616513084.076348:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.161@tcp1 of length 224/224 into md 0xf97ca809 [1] + 0 00000400:00000200:14.0:1616513084.076353:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdfb18 00000400:00000200:14.0:1616513084.076355:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.076358:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513084.076362:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfb18 00000800:00000200:14.0:1616513084.076365:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.076368:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.076368:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513084.076375:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.076378:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.076378:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.076387:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.161@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.076396:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.161@tcp1 of length 224 into portal 12 MB=0x5ffd4df82cac0 00000400:00000200:2.0:1616513084.076402:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.161@tcp1 of length 224/224 into md 0xf964d519 [1] + 32584 00000400:00000200:2.0:1616513084.076407:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.076410:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.076416:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.076419:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.076421:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513084.076426:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664631593472 00010000:00000200:16.0:1616513084.076454:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688664631593472/t0(0) o400->a60ecb50-224b-985f-022a-e630083921d8@10.31.138.161@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.076468:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664631593472, offset 192 00000400:00000200:16.0:1616513084.076474:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.161@tcp1 00000100:00000200:18.0:1616513084.076479:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664631593664 00000400:00000200:16.0:1616513084.076482:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.161@tcp1 routed destination 00010000:00000200:18.0:1616513084.076491:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d0480 x1688664631593664/t0(0) o400->25f18df7-9f22-fffa-776b-997b41d9a6d1@10.31.138.161@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513084.076497:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.161@tcp1(10.31.138.161@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.076504:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:18.0:1616513084.076505:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664631593664, offset 192 00000800:00000200:16.0:1616513084.076509:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513084.076511:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.161@tcp1 00000800:00000200:16.0:1616513084.076512:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.076514:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513084.076519:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.161@tcp1 routed destination 00000800:00000200:16.0:1616513084.076519:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513084.076526:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.076526:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513084.076534:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.161@tcp1(10.31.138.161@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513084.076541:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513084.076546:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513084.076549:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.076551:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.076555:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.076560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513084.076581:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.076588:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.076591:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.076591:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.076598:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513084.076603:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2908 00000800:00000200:12.0:1616513084.076606:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513084.076613:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.076620:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.076623:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513084.076623:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513084.076629:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.076633:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cde48 00000800:00000200:26.0:1616513084.076637:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.084440:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513084.084494:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.084503:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.084512:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.238@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.084521:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.238@tcp1 of length 224 into portal 26 MB=0x5ffd45b4c0500 00000400:00000200:20.0:1616513084.084527:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.238@tcp1 of length 224/224 into md 0xf97ca839 [1] + 0 00000400:00000200:20.0:1616513084.084532:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde198 00000400:00000200:20.0:1616513084.084535:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.084538:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513084.084542:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde198 00000800:00000200:20.0:1616513084.084545:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.084548:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.084550:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.084552:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513084.084606:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662413411584 00010000:00000200:16.0:1616513084.084617:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688662413411584/t0(0) o400->9634ce85-7a92-66a0-1bac-7b58e4e91d73@10.31.128.238@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.084631:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662413411584, offset 192 00000400:00000200:16.0:1616513084.084637:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.238@tcp1 00000400:00000200:16.0:1616513084.084644:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.238@tcp1 routed destination 00000400:00000200:16.0:1616513084.084659:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.238@tcp1(10.31.128.238@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.084666:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.084672:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.084676:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.084678:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.084683:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.084687:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.084690:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513084.084696:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.084701:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.084703:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513084.084708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513084.084709:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.084714:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b27f8 00000800:00000200:8.0:1616513084.084717:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.084720:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513084.084771:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.084781:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.084790:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.238@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.084798:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.238@tcp1 of length 224 into portal 12 MB=0x5ffd45b4c0880 00000400:00000200:2.0:1616513084.084804:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.238@tcp1 of length 224/224 into md 0xf964d519 [1] + 32808 00000400:00000200:2.0:1616513084.084810:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.084812:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.084818:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.084821:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.084823:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.084826:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513084.084883:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662413412480 00010000:00000200:18.0:1616513084.084895:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d6300 x1688662413412480/t0(0) o400->4c5ef40e-ad93-0220-89ab-6886af9375e3@10.31.128.238@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513084.084909:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662413412480, offset 192 00000400:00000200:18.0:1616513084.084916:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.238@tcp1 00000400:00000200:18.0:1616513084.084923:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.238@tcp1 routed destination 00000400:00000200:18.0:1616513084.084937:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.238@tcp1(10.31.128.238@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513084.084944:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513084.084949:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513084.084954:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.084956:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.084961:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.084966:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.084977:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.084983:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.084986:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.084992:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.084996:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd430 00000800:00000200:14.0:1616513084.084999:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.085002:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513084.090580:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926480, portal 4 00000100:00000200:4.0:1616513084.090586:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926480, offset 0 00000400:00000200:4.0:1616513084.090592:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:4.0:1616513084.090598:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:4.0:1616513084.090613:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513084.090619:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:4.0:1616513084.090626:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513084.090629:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:4.0:1616513084.090631:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:4.0:1616513084.090636:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513084.090683:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.090702:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513084.090707:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513084.090710:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513084.090716:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513084.090723:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affd3f00 x1693067725926480/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513084.090735:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb7e8 00000800:00000200:27.0:1616513084.090739:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513084.090742:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513084.090848:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513084.090857:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.090863:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513084.090867:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513084.090873:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513084.090880:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627050 00000400:00000200:27.0:1616513084.090887:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166d0d [1] + 192 00000400:00000200:27.0:1616513084.090892:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513084.090896:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affd3f00 x1693067725926480/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513084.090910:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.090913:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513084.090915:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:4.0:1616513084.090920:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affd3f00 x1693067725926480/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513084.090933:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affd3f00 x1693067725926480/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513084.090944:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00000800:00000200:7.2:1616513084.106418:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.106474:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.106484:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513084.106488:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513084.106493:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.43@tcp1 : PUT - for me 00000800:00000200:10.0:1616513084.106497:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.106502:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.43@tcp1 of length 224 into portal 26 MB=0x5fff8f5752100 00000400:00000200:10.0:1616513084.106505:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.43@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.106508:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.43@tcp1 of length 224/224 into md 0xf97ca859 [1] + 0 00000400:00000200:10.0:1616513084.106514:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.43@tcp1 of length 224 into portal 12 MB=0x5fff8f57521c0 00000400:00000200:2.0:1616513084.106515:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf760 00000400:00000200:2.0:1616513084.106518:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:10.0:1616513084.106520:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.43@tcp1 of length 224/224 into md 0xf964d519 [1] + 33032 00000400:00000200:10.0:1616513084.106525:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.106525:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.106529:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf760 00000100:00000200:10.0:1616513084.106532:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.106533:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513084.106535:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513084.106537:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513084.106539:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.106540:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513084.106542:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513084.106544:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513084.106603:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688819618619648 00010000:00000200:16.0:1616513084.106614:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688819618619648/t0(0) o400->a735b642-528d-47f8-b725-b13191ccd441@10.31.140.43@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513084.106623:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688819618619840 00000100:00000200:16.0:1616513084.106628:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688819618619648, offset 192 00000400:00000200:16.0:1616513084.106634:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.43@tcp1 00010000:00000200:18.0:1616513084.106635:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bd8d80 x1688819618619840/t0(0) o400->04400787-96f9-bfe9-921f-d6eb852a68bd@10.31.140.43@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513084.106641:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.43@tcp1 routed destination 00000100:00000200:18.0:1616513084.106648:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688819618619840, offset 192 00000400:00000200:18.0:1616513084.106654:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.43@tcp1 00000400:00000200:16.0:1616513084.106655:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.43@tcp1(10.31.140.43@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513084.106661:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.43@tcp1 routed destination 00000800:00000200:16.0:1616513084.106661:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.106667:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.106669:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.106671:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513084.106675:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.43@tcp1(10.31.140.43@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.106675:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513084.106681:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:16.0:1616513084.106682:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.106683:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513084.106686:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513084.106689:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.106691:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.106695:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.106701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513084.106740:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.106747:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.106750:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.106757:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513084.106759:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513084.106762:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b26e8 00000800:00000200:20.0:1616513084.106765:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513084.106767:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513084.106768:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513084.106770:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513084.106771:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513084.106776:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.106780:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccb28 00000800:00000200:26.0:1616513084.106784:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.107613:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.107624:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.107631:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.107640:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.191@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.107649:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.191@tcp1 of length 224 into portal 12 MB=0x5ffd48eafb340 00000400:00000200:8.0:1616513084.107655:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.191@tcp1 of length 224/224 into md 0x865bbec1 [1] + 66192 00000400:00000200:8.0:1616513084.107660:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.107663:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513084.107669:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.107672:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.107673:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.107676:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.107735:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663275582272 00010000:00000200:24.0:1616513084.107747:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8c380 x1688663275582272/t0(0) o400->2771e735-ef62-b250-79bb-45e1c95f6115@10.31.136.191@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513084.107761:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663275582272, offset 192 00000400:00000200:24.0:1616513084.107768:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.191@tcp1 00000400:00000200:24.0:1616513084.107775:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.191@tcp1 routed destination 00000400:00000200:24.0:1616513084.107790:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.191@tcp1(10.31.136.191@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.107797:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.107803:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.107805:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.107808:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.107812:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513084.107816:0:49878:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513084.107817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.107826:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.107829:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.107831:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513084.107836:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.107836:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.107839:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2000 00000800:00000200:20.0:1616513084.107842:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513084.107890:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.107900:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.107909:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.191@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.107917:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.191@tcp1 of length 224 into portal 26 MB=0x5ffd48eafaf80 00000400:00000200:14.0:1616513084.107924:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.191@tcp1 of length 224/224 into md 0xf97ca869 [1] + 0 00000400:00000200:14.0:1616513084.107928:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdeb28 00000400:00000200:14.0:1616513084.107931:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.107934:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513084.107938:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdeb28 00000800:00000200:14.0:1616513084.107941:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.107943:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513084.107944:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.107948:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513084.108020:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663275581312 00010000:00000200:16.0:1616513084.108031:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688663275581312/t0(0) o400->3af0bef0-edc7-5da7-dab0-437f390f72ac@10.31.136.191@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.108045:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663275581312, offset 192 00000400:00000200:16.0:1616513084.108051:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.191@tcp1 00000400:00000200:16.0:1616513084.108059:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.191@tcp1 routed destination 00000400:00000200:16.0:1616513084.108074:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.191@tcp1(10.31.136.191@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.108081:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513084.108086:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.108089:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513084.108091:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513084.108096:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.108100:0:49878:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513084.108102:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513084.108154:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.108160:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.108163:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.108170:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.108175:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2990 00000800:00000200:2.0:1616513084.108179:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.108182:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.110476:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513084.110485:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513084.110499:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513084.110507:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513084.110515:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.133@tcp1 : PUT - for me 00000400:00000200:10.0:1616513084.110523:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.133@tcp1 of length 224 into portal 26 MB=0x5f98eafd62fc0 00000800:00000200:12.0:1616513084.110529:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513084.110529:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.133@tcp1 of length 224/224 into md 0xf97ca879 [1] + 0 00000400:00000200:10.0:1616513084.110534:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdea18 00000400:00000200:10.0:1616513084.110536:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513084.110538:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513084.110538:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.110539:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:10.0:1616513084.110542:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdea18 00000800:00000200:10.0:1616513084.110545:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513084.110547:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:12.0:1616513084.110548:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.133@tcp1 : PUT - for me 00000800:00000200:10.0:1616513084.110549:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513084.110551:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513084.110556:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.133@tcp1 of length 224 into portal 12 MB=0x5f98eafd63040 00000400:00000200:12.0:1616513084.110563:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.133@tcp1 of length 224/224 into md 0x865bbec1 [1] + 66416 00000400:00000200:12.0:1616513084.110568:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.110570:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.110578:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.110581:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.110583:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513084.110605:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681766114275264 00010000:00000200:16.0:1616513084.110616:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1681766114275264/t0(0) o400->dcd2ec8d-a588-e979-4416-d7bced461da8@10.31.143.133@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.110631:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681766114275264, offset 192 00000400:00000200:16.0:1616513084.110637:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.133@tcp1 00000100:00000200:24.0:1616513084.110641:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681766114275392 00000400:00000200:16.0:1616513084.110644:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.133@tcp1 routed destination 00010000:00000200:24.0:1616513084.110653:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dda00 x1681766114275392/t0(0) o400->b712cdc1-92bc-20fb-6849-e975502d14eb@10.31.143.133@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513084.110659:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.133@tcp1(10.31.143.133@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.110666:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513084.110668:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681766114275392, offset 192 00000800:00000200:16.0:1616513084.110672:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513084.110674:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.133@tcp1 00000800:00000200:16.0:1616513084.110674:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.110676:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513084.110681:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.133@tcp1 routed destination 00000800:00000200:16.0:1616513084.110681:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.110686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.110688:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513084.110696:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.133@tcp1(10.31.143.133@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.110703:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.110708:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.110711:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.110713:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.110718:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.110724:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513084.110758:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.110765:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.110768:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513084.110770:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.110774:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.110780:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b24c8 00000800:00000200:2.0:1616513084.110781:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513084.110784:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513084.110788:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.110790:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.110797:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.110801:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf27f8 00000800:00000200:2.0:1616513084.110805:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.110808:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.118958:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.118969:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.118976:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.118984:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.203@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.118993:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.203@tcp1 of length 224 into portal 26 MB=0x5ffebc28c6f40 00000400:00000200:8.0:1616513084.118999:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.203@tcp1 of length 224/224 into md 0xf97ca899 [1] + 0 00000400:00000200:8.0:1616513084.119003:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf6d8 00000400:00000200:8.0:1616513084.119006:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.119008:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.119012:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf6d8 00000800:00000200:8.0:1616513084.119015:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.119017:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.119019:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.119021:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.119031:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.119040:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.203@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.119049:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.203@tcp1 of length 224 into portal 12 MB=0x5ffebc28c6fc0 00000400:00000200:12.0:1616513084.119055:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.203@tcp1 of length 224/224 into md 0x865bbec1 [1] + 66640 00000400:00000200:12.0:1616513084.119061:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.119063:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.119069:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.119072:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.119074:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513084.119076:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688762929934144 00000800:00000200:12.0:1616513084.119077:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:16.0:1616513084.119087:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688762929934144/t0(0) o400->72e4f4d0-2632-d534-0276-69e7fce4b048@10.31.140.203@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.119101:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688762929934144, offset 192 00000400:00000200:16.0:1616513084.119108:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.203@tcp1 00000400:00000200:16.0:1616513084.119115:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.203@tcp1 routed destination 00000400:00000200:16.0:1616513084.119130:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.203@tcp1(10.31.140.203@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513084.119134:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688762929934272 00000800:00000200:16.0:1616513084.119137:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.119143:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:24.0:1616513084.119146:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dbf00 x1688762929934272/t0(0) o400->dc6f97b8-40e7-c40e-d094-f5ab1ef9f37e@10.31.140.203@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513084.119146:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.119148:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.119152:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.119157:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.119159:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513084.119171:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688762929934272, offset 192 00000400:00000200:24.0:1616513084.119177:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.203@tcp1 00000400:00000200:24.0:1616513084.119184:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.203@tcp1 routed destination 00000400:00000200:24.0:1616513084.119200:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.203@tcp1(10.31.140.203@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.119206:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513084.119210:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513084.119212:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.119215:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513084.119217:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513084.119217:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513084.119220:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.119220:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513084.119222:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513084.119227:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513084.119229:0:49879:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513084.119232:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b26e8 00000800:00000200:28.0:1616513084.119236:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513084.119283:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513084.119290:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513084.119293:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513084.119299:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513084.119303:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:10.0:1616513084.119307:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513084.119310:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.130732:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.130794:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513084.130809:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513084.130810:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.130819:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.239@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.130829:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.239@tcp1 of length 224 into portal 26 MB=0x603e0f1d28cc0 00000400:00000200:2.0:1616513084.130835:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.239@tcp1 of length 224/224 into md 0xf97ca8a9 [1] + 0 00000400:00000200:2.0:1616513084.130840:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde110 00000400:00000200:2.0:1616513084.130843:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.130845:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.130850:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde110 00000800:00000200:2.0:1616513084.130853:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.130856:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.130857:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.130861:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513084.130879:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.130888:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.130898:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.239@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.130907:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.239@tcp1 of length 224 into portal 12 MB=0x603e0f1d28d40 00000400:00000200:20.0:1616513084.130913:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.239@tcp1 of length 224/224 into md 0x865bbec1 [1] + 66864 00000100:00000200:16.0:1616513084.130915:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114524929216 00000400:00000200:20.0:1616513084.130918:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.130923:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:16.0:1616513084.130925:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1693114524929216/t0(0) o400->e5a73f61-256c-da59-2637-a868523d8402@10.31.136.239@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513084.130929:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.130932:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.130932:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.130936:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513084.130939:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114524929216, offset 192 00000400:00000200:16.0:1616513084.130946:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.239@tcp1 00000400:00000200:16.0:1616513084.130953:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.239@tcp1 routed destination 00000400:00000200:16.0:1616513084.130967:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.239@tcp1(10.31.136.239@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.130974:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.130979:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.130981:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.130983:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.130988:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513084.130992:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114524929344 00000800:00000200:11.2:1616513084.130992:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.130994:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513084.131001:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:24.0:1616513084.131004:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cef980 x1693114524929344/t0(0) o400->a362cd1c-91a4-67cc-bd1e-05be4ef0ae0b@10.31.136.239@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:8.0:1616513084.131005:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.131007:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.131013:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.131017:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2908 00000100:00000200:24.0:1616513084.131020:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114524929344, offset 192 00000800:00000200:8.0:1616513084.131020:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.131022:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513084.131026:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.239@tcp1 00000400:00000200:24.0:1616513084.131033:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.239@tcp1 routed destination 00000400:00000200:24.0:1616513084.131047:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.239@tcp1(10.31.136.239@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.131053:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.131059:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.131061:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.131063:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.131067:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.131072:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513084.131125:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513084.131133:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513084.131136:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513084.131142:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513084.131145:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2d48 00000800:00000200:10.0:1616513084.131149:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513084.131152:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.133979:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.134033:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.134043:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.134052:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.72@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.134060:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.72@tcp1 of length 224 into portal 26 MB=0x601a9cdc9eb00 00000400:00000200:26.0:1616513084.134067:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.72@tcp1 of length 224/224 into md 0xf97ca8c9 [1] + 0 00000400:00000200:26.0:1616513084.134072:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf298 00000400:00000200:26.0:1616513084.134074:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.134077:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513084.134081:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf298 00000800:00000200:26.0:1616513084.134085:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.134086:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.134087:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.134091:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513084.134146:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690678673926912 00010000:00000200:16.0:1616513084.134157:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1690678673926912/t0(0) o400->d045aba3-816e-c6e3-7aaf-4b329d643870@10.31.133.72@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.134171:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690678673926912, offset 192 00000400:00000200:16.0:1616513084.134177:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.72@tcp1 00000400:00000200:16.0:1616513084.134185:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.72@tcp1 routed destination 00000400:00000200:16.0:1616513084.134200:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.72@tcp1(10.31.133.72@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.134206:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.134212:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.134215:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.134217:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.134222:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513084.134228:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.134228:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.134291:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.134298:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.134301:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513084.134307:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513084.134313:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2550 00000800:00000200:12.0:1616513084.134316:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.134319:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.134357:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513084.134410:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513084.134420:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513084.134428:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.72@tcp1 : PUT - for me 00000400:00000200:10.0:1616513084.134449:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.72@tcp1 of length 224 into portal 12 MB=0x601a9cdc9eb80 00000400:00000200:10.0:1616513084.134455:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.72@tcp1 of length 224/224 into md 0xf964d519 [1] + 33256 00000400:00000200:10.0:1616513084.134461:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513084.134463:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.134465:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513084.134471:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513084.134474:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513084.134475:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513084.134536:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690678673927040 00010000:00000200:18.0:1616513084.134548:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bde300 x1690678673927040/t0(0) o400->d6657a07-795b-a6d4-1455-ff6663563b11@10.31.133.72@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:18.0:1616513084.134563:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690678673927040, offset 192 00000400:00000200:18.0:1616513084.134570:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.72@tcp1 00000400:00000200:18.0:1616513084.134577:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.72@tcp1 routed destination 00000400:00000200:18.0:1616513084.134592:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.72@tcp1(10.31.133.72@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513084.134598:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513084.134604:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513084.134607:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.134609:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.134613:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.134618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.134629:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.134634:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.134636:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513084.134638:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.134643:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.134647:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cce58 00000800:00000200:2.0:1616513084.134651:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.142936:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.142990:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.143000:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.143009:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.154@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.143018:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.154@tcp1 of length 224 into portal 26 MB=0x6018982f7eb80 00000400:00000200:14.0:1616513084.143025:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.154@tcp1 of length 224/224 into md 0xf97ca8e9 [1] + 0 00000400:00000200:14.0:1616513084.143029:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde3b8 00000400:00000200:14.0:1616513084.143032:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.143035:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513084.143039:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde3b8 00000800:00000200:18.0:1616513084.143044:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.143044:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.143047:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.143051:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.143054:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.143055:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513084.143062:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513084.143063:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.154@tcp1 : PUT - for me 00000800:00000200:8.0:1616513084.143073:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513084.143074:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.154@tcp1 of length 224 into portal 12 MB=0x6018982f7ed00 00000800:00000200:8.0:1616513084.143078:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:18.0:1616513084.143080:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.154@tcp1 of length 224/224 into md 0xf964d519 [1] + 33480 00000400:00000200:18.0:1616513084.143085:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.143085:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.147@tcp1 : PUT - for me 00000100:00000200:18.0:1616513084.143088:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513084.143092:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.147@tcp1 of length 224 into portal 26 MB=0x601843e05d900 00000800:00000200:18.0:1616513084.143095:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.143098:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513084.143098:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.147@tcp1 of length 224/224 into md 0xf97ca8f9 [1] + 0 00000800:00000200:18.0:1616513084.143100:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513084.143101:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde440 00000100:00000200:16.0:1616513084.143102:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690539979697024 00000400:00000200:8.0:1616513084.143103:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.143105:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.143110:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde440 00010000:00000200:16.0:1616513084.143112:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1690539979697024/t0(0) o400->46ba1e9b-3125-9f1b-3b7d-cf2bd9f56fea@10.31.129.154@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:8.0:1616513084.143112:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.143114:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.143116:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.143118:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513084.143125:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690539979697024, offset 192 00000400:00000200:16.0:1616513084.143131:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.154@tcp1 00000400:00000200:16.0:1616513084.143138:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.154@tcp1 routed destination 00000400:00000200:16.0:1616513084.143152:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.154@tcp1(10.31.129.154@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.143162:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.143167:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.143170:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.143172:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.143176:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513084.143179:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517348145408 00000100:00000200:22.0:1616513084.143181:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690539979697408 00000800:00000200:11.2:1616513084.143181:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.143183:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:24.0:1616513084.143189:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1690517348145408/t0(0) o400->bbda1470-6a27-7721-4f9f-3425399c0829@10.31.131.147@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:22.0:1616513084.143196:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfa880 x1690539979697408/t0(0) o400->0474958a-720d-c4fa-bfef-0fa689a643c5@10.31.129.154@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513084.143203:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690517348145408, offset 192 00000400:00000200:24.0:1616513084.143208:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.147@tcp1 00000100:00000200:22.0:1616513084.143210:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690539979697408, offset 192 00000400:00000200:24.0:1616513084.143216:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.147@tcp1 routed destination 00000400:00000200:22.0:1616513084.143216:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.154@tcp1 00000400:00000200:22.0:1616513084.143223:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.154@tcp1 routed destination 00000800:00000200:12.0:1616513084.143234:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513084.143238:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.154@tcp1(10.31.129.154@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513084.143240:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.143243:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513084.143244:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513084.143250:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513084.143252:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.147@tcp1(10.31.131.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513084.143256:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2cc0 00000800:00000200:24.0:1616513084.143259:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513084.143259:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513084.143260:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513084.143262:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513084.143263:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.143265:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.143265:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.143268:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513084.143270:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513084.143270:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513084.143275:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:7.2:1616513084.143275:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (23)++ 00000100:00000200:24.0:1616513084.143282:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513084.143329:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513084.143337:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513084.143339:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513084.143340:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513084.143344:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513084.143346:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.143346:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513084.143349:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.143350:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b430 00000400:00000200:14.0:1616513084.143352:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:2.0:1616513084.143354:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513084.143357:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:14.0:1616513084.143359:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.143468:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.143480:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.143490:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513084.143491:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513084.143499:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.147@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.143508:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.147@tcp1 of length 224 into portal 12 MB=0x601843e05dcc0 00000400:00000200:26.0:1616513084.143514:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.147@tcp1 of length 224/224 into md 0xf964d519 [1] + 33704 00000400:00000200:26.0:1616513084.143522:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.143525:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513084.143532:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.143535:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.143537:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.143544:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517348146368 00010000:00000200:22.0:1616513084.143551:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57a880 x1690517348146368/t0(0) o400->9b1f8806-5a2d-5d9f-6c0c-966c9abb16d4@10.31.131.147@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.143571:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690517348146368, offset 192 00000400:00000200:22.0:1616513084.143576:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.147@tcp1 00000400:00000200:22.0:1616513084.143581:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.147@tcp1 routed destination 00000400:00000200:22.0:1616513084.143594:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.147@tcp1(10.31.131.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.143599:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.143604:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.143606:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.143608:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.143611:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.143616:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.143624:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.143627:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.143629:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513084.143634:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.143634:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.143636:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a770 00000800:00000200:8.0:1616513084.143639:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.160775:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.160832:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.160842:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.160850:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.133@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.160861:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.133@tcp1 of length 224 into portal 26 MB=0x601831a324d00 00000400:00000200:2.0:1616513084.160867:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.133@tcp1 of length 224/224 into md 0xf97ca919 [1] + 0 00000400:00000200:2.0:1616513084.160872:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdfa08 00000400:00000200:2.0:1616513084.160875:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.160878:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.160882:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfa08 00000800:00000200:2.0:1616513084.160885:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.160886:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.160888:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.160893:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.160896:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.160897:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.160905:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.133@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.160915:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.133@tcp1 of length 224 into portal 12 MB=0x601831a324e40 00000400:00000200:14.0:1616513084.160921:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.133@tcp1 of length 224/224 into md 0xf964d519 [1] + 33928 00000400:00000200:14.0:1616513084.160926:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.160929:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.160936:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.160939:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.160941:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.160946:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512452111616 00010000:00000200:24.0:1616513084.160957:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1690512452111616/t0(0) o400->e9b905ae-9249-bf55-dfa2-fc8e9113de2d@10.31.136.133@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.160971:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512452111616, offset 192 00000400:00000200:24.0:1616513084.160977:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.133@tcp1 00000400:00000200:24.0:1616513084.160985:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.133@tcp1 routed destination 00000400:00000200:24.0:1616513084.161000:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.133@tcp1(10.31.136.133@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513084.161001:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512452111936 00000800:00000200:24.0:1616513084.161007:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.161013:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513084.161013:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464626780 x1690512452111936/t0(0) o400->49d7fb2c-0c19-5511-6d0d-7f057fcdd48b@10.31.136.133@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513084.161015:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.161018:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.161023:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513084.161027:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512452111936, offset 192 00000800:00000200:7.2:1616513084.161028:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.161029:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513084.161034:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.133@tcp1 00000400:00000200:22.0:1616513084.161041:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.133@tcp1 routed destination 00000800:00000200:26.0:1616513084.161044:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.161051:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.161054:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513084.161055:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513084.161056:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.133@tcp1(10.31.136.133@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513084.161060:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513084.161077:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513084.161078:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2550 00000800:00000200:26.0:1616513084.161081:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:22.0:1616513084.161083:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.161085:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.161087:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.161092:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.161137:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.161193:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.161200:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.161203:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513084.161210:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513084.161214:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b8f8 00000800:00000200:12.0:1616513084.161220:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.161223:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.179965:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513084.179984:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.179994:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513084.180004:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.58@tcp1 : PUT - for me 00000400:00000200:28.0:1616513084.180015:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.58@tcp1 of length 224 into portal 26 MB=0x60016cbc09100 00000400:00000200:28.0:1616513084.180021:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.58@tcp1 of length 224/224 into md 0xf97ca929 [1] + 0 00000400:00000200:28.0:1616513084.180026:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf3a8 00000400:00000200:28.0:1616513084.180031:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513084.180034:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513084.180037:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513084.180039:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf3a8 00000800:00000200:28.0:1616513084.180042:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513084.180045:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513084.180046:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.180047:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513084.180049:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.180054:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.58@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.180062:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.58@tcp1 of length 224 into portal 12 MB=0x60016cbc09240 00000400:00000200:8.0:1616513084.180068:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.58@tcp1 of length 224/224 into md 0x865bbec1 [1] + 67088 00000400:00000200:8.0:1616513084.180072:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.180075:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513084.180082:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.180085:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.180087:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513084.180103:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688947767939648 00000100:00000200:24.0:1616513084.180104:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688947767939328 00010000:00000200:28.0:1616513084.180113:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8d100 x1688947767939648/t0(0) o400->a5e5f6c5-cca4-d5a4-f3c1-0754208ebc1e@10.31.133.58@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513084.180115:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688947767939328/t0(0) o400->38301f74-ffc0-5908-d1f9-7aa05a9bef78@10.31.133.58@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513084.180127:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688947767939648, offset 192 00000100:00000200:24.0:1616513084.180130:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688947767939328, offset 192 00000400:00000200:28.0:1616513084.180133:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.58@tcp1 00000400:00000200:24.0:1616513084.180138:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.58@tcp1 00000400:00000200:28.0:1616513084.180139:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.58@tcp1 routed destination 00000400:00000200:24.0:1616513084.180149:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.58@tcp1 routed destination 00000400:00000200:28.0:1616513084.180154:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.58@tcp1(10.31.133.58@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513084.180160:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513084.180164:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.58@tcp1(10.31.133.58@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513084.180171:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.180171:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513084.180174:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513084.180176:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.180176:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.180179:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513084.180180:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513084.180183:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.180188:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513084.180188:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513084.180193:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.180194:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513084.180212:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.180219:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.180222:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513084.180223:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.180229:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.180234:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:20.0:1616513084.180237:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513084.180241:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.180248:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.180250:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.180257:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.180260:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd078 00000800:00000200:14.0:1616513084.180264:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.180267:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:9.0:1616513084.186571:0:6488:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926496, portal 4 00000100:00000200:9.0:1616513084.186577:0:6488:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926496, offset 0 00000400:00000200:9.0:1616513084.186584:0:6488:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:9.0:1616513084.186590:0:6488:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:9.0:1616513084.186604:0:6488:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:9.0:1616513084.186611:0:6488:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:9.0:1616513084.186618:0:6488:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:9.0:1616513084.186621:0:6488:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:9.0:1616513084.186623:0:6488:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:9.0:1616513084.186629:0:6488:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513084.186676:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.186696:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513084.186701:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513084.186704:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513084.186711:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513084.186717:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8302325580 x1693067725926496/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513084.186729:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35dc0 00000800:00000200:27.0:1616513084.186734:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513084.186736:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513084.186847:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513084.186857:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.186863:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513084.186867:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513084.186871:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513084.186879:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627060 00000400:00000200:27.0:1616513084.186885:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166d1d [1] + 192 00000400:00000200:27.0:1616513084.186891:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513084.186895:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8302325580 x1693067725926496/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513084.186909:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.186912:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513084.186913:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:9.0:1616513084.186917:0:6488:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8302325580 x1693067725926496/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:9.0:1616513084.186930:0:6488:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8302325580 x1693067725926496/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:9.0:1616513084.186942:0:6488:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35430 00000800:00000200:11.2:1616513084.188105:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.188117:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.188123:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.188132:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.55@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.188139:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.55@tcp1 of length 224 into portal 12 MB=0x5ffdbd5feacc0 00000400:00000200:8.0:1616513084.188144:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.55@tcp1 of length 224/224 into md 0x865bbec1 [1] + 67312 00000400:00000200:8.0:1616513084.188147:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.188150:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513084.188155:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.188157:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.188159:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.188161:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.188214:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:28.0:1616513084.188220:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694536711360 00010000:00000200:28.0:1616513084.188232:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89200 x1688694536711360/t0(0) o400->c829dd43-6d77-4551-8172-86c80dacd57e@10.31.140.55@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513084.188247:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694536711360, offset 192 00000400:00000200:28.0:1616513084.188253:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.55@tcp1 00000400:00000200:28.0:1616513084.188261:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.55@tcp1 routed destination 00000400:00000200:28.0:1616513084.188276:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.55@tcp1(10.31.140.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513084.188283:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513084.188285:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513084.188289:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513084.188292:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513084.188294:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513084.188295:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:28.0:1616513084.188301:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513084.188308:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.55@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.188317:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.55@tcp1 of length 224 into portal 26 MB=0x5ffdbd5feac40 00000400:00000200:2.0:1616513084.188323:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.55@tcp1 of length 224/224 into md 0xf97ca939 [1] + 0 00000400:00000200:2.0:1616513084.188328:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdfed0 00000400:00000200:2.0:1616513084.188333:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.188336:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513084.188338:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:2.0:1616513084.188340:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfed0 00000800:00000200:2.0:1616513084.188344:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513084.188346:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:2.0:1616513084.188346:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:2.0:1616513084.188348:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513084.188349:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513084.188351:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513084.188356:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.188360:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccd48 00000800:00000200:26.0:1616513084.188364:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.188404:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694536711232 00010000:00000200:24.0:1616513084.188415:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688694536711232/t0(0) o400->41edfaf5-5c70-8cdf-d59d-95fa40807527@10.31.140.55@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.188429:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694536711232, offset 192 00000400:00000200:24.0:1616513084.188449:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.55@tcp1 00000400:00000200:24.0:1616513084.188456:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.55@tcp1 routed destination 00000400:00000200:24.0:1616513084.188471:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.55@tcp1(10.31.140.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.188478:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.188483:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.188486:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.188488:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.188493:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.188498:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.188500:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513084.188509:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.188513:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.188516:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513084.188519:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.188521:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.188525:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2a18 00000800:00000200:28.0:1616513084.188528:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.202669:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.202682:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.202688:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.202695:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.77@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.202702:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.77@tcp1 of length 224 into portal 26 MB=0x5ffde6c398d00 00000400:00000200:8.0:1616513084.202707:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.77@tcp1 of length 224/224 into md 0xf97ca949 [1] + 0 00000400:00000200:8.0:1616513084.202711:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf650 00000400:00000200:8.0:1616513084.202714:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.202716:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.202720:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf650 00000800:00000200:8.0:1616513084.202722:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.202724:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.202726:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.202728:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.202732:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513084.202734:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.202738:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.77@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.202742:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.77@tcp1 of length 224 into portal 12 MB=0x5ffde6c398fc0 00000400:00000200:8.0:1616513084.202747:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.77@tcp1 of length 224/224 into md 0x865bbec1 [1] + 67536 00000400:00000200:8.0:1616513084.202750:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.202752:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513084.202756:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.202758:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.202759:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.202783:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705647086848 00010000:00000200:24.0:1616513084.202795:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688705647086848/t0(0) o400->296130c7-d24b-1f0a-40d7-f658d43261f1@10.31.142.77@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.202809:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705647086848, offset 192 00000400:00000200:24.0:1616513084.202815:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.77@tcp1 00000400:00000200:24.0:1616513084.202823:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.77@tcp1 routed destination 00000100:00000200:28.0:1616513084.202825:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705647087552 00010000:00000200:28.0:1616513084.202837:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8cc80 x1688705647087552/t0(0) o400->2f292f43-79dc-aaa0-bcd0-31aaac56ca5c@10.31.142.77@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513084.202838:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.77@tcp1(10.31.142.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.202845:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.202851:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:28.0:1616513084.202852:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705647087552, offset 192 00000800:00000200:24.0:1616513084.202854:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.202856:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513084.202858:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.77@tcp1 00000800:00000200:24.0:1616513084.202861:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513084.202866:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.77@tcp1 routed destination 00000800:00000200:7.2:1616513084.202867:0:49891:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.202868:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513084.202880:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.77@tcp1(10.31.142.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513084.202887:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513084.202892:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513084.202895:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513084.202897:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.202902:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.202908:0:49886:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.202921:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.202925:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.202927:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513084.202928:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513084.202929:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.202933:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.202935:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccc38 00000800:00000200:18.0:1616513084.202935:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513084.202938:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513084.202938:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.202944:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.202950:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:18.0:1616513084.202953:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.202956:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.202962:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513084.202970:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.134@tcp1 : PUT - for me 00000400:00000200:18.0:1616513084.202979:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.134@tcp1 of length 224 into portal 26 MB=0x6001f41cf8e00 00000800:00000200:2.0:1616513084.202982:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513084.202985:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.134@tcp1 of length 224/224 into md 0xf97ca969 [1] + 0 00000400:00000200:18.0:1616513084.202989:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdfa90 00000400:00000200:18.0:1616513084.202992:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513084.202994:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513084.202994:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513084.202995:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.202998:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfa90 00000800:00000200:18.0:1616513084.203000:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513084.203002:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.134@tcp1 : PUT - for me 00000800:00000200:18.0:1616513084.203003:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.203005:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.203011:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688984108371456 00000400:00000200:2.0:1616513084.203011:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.134@tcp1 of length 224 into portal 12 MB=0x6001f41cf8e80 00010000:00000200:24.0:1616513084.203017:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688984108371456/t0(0) o400->0f51e332-6708-231e-cdab-f6fbc2d64f6d@10.31.141.134@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513084.203018:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.134@tcp1 of length 224/224 into md 0xf964d519 [1] + 34152 00000400:00000200:2.0:1616513084.203023:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.203026:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:24.0:1616513084.203031:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688984108371456, offset 192 00000800:00000200:2.0:1616513084.203033:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513084.203036:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.134@tcp1 00000800:00000200:2.0:1616513084.203036:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.203038:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513084.203042:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.134@tcp1 routed destination 00000400:00000200:24.0:1616513084.203055:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.134@tcp1(10.31.141.134@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.203061:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.203065:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.203068:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.203069:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.203072:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.203077:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.203078:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.203086:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.203090:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.203092:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513084.203095:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513084.203098:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:22.0:1616513084.203100:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688984108371584 00000400:00000200:14.0:1616513084.203102:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:14.0:1616513084.203106:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:22.0:1616513084.203112:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d0900 x1688984108371584/t0(0) o400->6c772e14-0fc1-6afa-1adf-5b9eebc04935@10.31.141.134@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.203127:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688984108371584, offset 192 00000400:00000200:22.0:1616513084.203133:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.134@tcp1 00000400:00000200:22.0:1616513084.203140:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.134@tcp1 routed destination 00000400:00000200:22.0:1616513084.203156:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.134@tcp1(10.31.141.134@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.203163:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.203169:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.203171:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.203174:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.203178:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.203185:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.203194:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.203198:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.203200:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513084.203204:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.203205:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.203207:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a7f8 00000800:00000200:8.0:1616513084.203210:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.203878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.203889:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.203897:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513084.203898:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.203905:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.110@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.203915:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.110@tcp1 of length 224 into portal 26 MB=0x5ffdb591d8fc0 00000400:00000200:12.0:1616513084.203921:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.110@tcp1 of length 224/224 into md 0xf97ca979 [1] + 0 00000400:00000200:12.0:1616513084.203927:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf5c8 00000400:00000200:12.0:1616513084.203929:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.203932:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513084.203937:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf5c8 00000800:00000200:12.0:1616513084.203940:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.203943:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.203944:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.203947:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692441583552 00010000:00000200:24.0:1616513084.203952:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688692441583552/t0(0) o400->93a9fbf1-44cf-9870-0b92-e32da492b81a@10.31.141.110@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.203961:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692441583552, offset 192 00000400:00000200:24.0:1616513084.203965:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.110@tcp1 00000400:00000200:24.0:1616513084.203970:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.110@tcp1 routed destination 00000400:00000200:24.0:1616513084.203982:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.110@tcp1(10.31.141.110@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.203987:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.203991:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.203993:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.203994:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.203997:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513084.203998:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513084.204001:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.204002:0:49896:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513084.204009:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513084.204015:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513084.204017:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513084.204017:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513084.204022:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.204024:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.204025:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.110@tcp1 : PUT - for me 00000800:00000200:2.0:1616513084.204026:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513084.204031:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.204034:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.110@tcp1 of length 224 into portal 12 MB=0x5ffdb591d9040 00000400:00000200:26.0:1616513084.204036:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:26.0:1616513084.204040:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513084.204041:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.110@tcp1 of length 224/224 into md 0x865bbec1 [1] + 67760 00000400:00000200:20.0:1616513084.204047:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.204049:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513084.204056:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.204059:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.204061:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513084.204068:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692441583680 00010000:00000200:28.0:1616513084.204074:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceda00 x1688692441583680/t0(0) o400->baa53a76-ba5c-0325-b2a2-ef476105975f@10.31.141.110@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513084.204085:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692441583680, offset 192 00000400:00000200:28.0:1616513084.204090:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.110@tcp1 00000400:00000200:28.0:1616513084.204095:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.110@tcp1 routed destination 00000400:00000200:28.0:1616513084.204109:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.110@tcp1(10.31.141.110@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513084.204114:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513084.204119:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513084.204121:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513084.204122:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.204125:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.204130:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.204138:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.204141:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.204143:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513084.204146:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.204148:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.204150:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccc38 00000800:00000200:28.0:1616513084.204152:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.208332:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.208388:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.208397:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513084.208397:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.208402:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513084.208406:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.149@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.208409:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.149@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.208418:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.149@tcp1 of length 224 into portal 12 MB=0x603e145db69c0 00000400:00000200:18.0:1616513084.208419:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.149@tcp1 of length 224 into portal 26 MB=0x603e145db6940 00000400:00000200:14.0:1616513084.208424:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.149@tcp1 of length 224/224 into md 0xf964d519 [1] + 34376 00000400:00000200:14.0:1616513084.208428:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.208430:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.149@tcp1 of length 224/224 into md 0xf97ca999 [1] + 0 00000800:00000200:2.0:1616513084.208448:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.208451:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdedd0 00000400:00000200:18.0:1616513084.208454:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.208454:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:18.0:1616513084.208459:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513084.208459:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.208462:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.208463:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdedd0 00000800:00000200:14.0:1616513084.208463:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.208466:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.208469:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.208471:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.208539:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115934796096 00000100:00000200:22.0:1616513084.208543:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115934796224 00010000:00000200:24.0:1616513084.208550:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1693115934796096/t0(0) o400->d0013888-7186-2977-61fe-ff9d4f5689c4@10.31.136.149@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:22.0:1616513084.208556:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464625e80 x1693115934796224/t0(0) o400->a71b9889-0d70-31c0-ce0c-0593d6c87eb7@10.31.136.149@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513084.208563:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693115934796096, offset 192 00000400:00000200:24.0:1616513084.208569:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.149@tcp1 00000100:00000200:22.0:1616513084.208570:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693115934796224, offset 192 00000400:00000200:24.0:1616513084.208576:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.149@tcp1 routed destination 00000400:00000200:22.0:1616513084.208576:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.149@tcp1 00000400:00000200:22.0:1616513084.208584:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.149@tcp1 routed destination 00000400:00000200:24.0:1616513084.208599:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.149@tcp1(10.31.136.149@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513084.208599:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.149@tcp1(10.31.136.149@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.208605:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513084.208606:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.208611:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.208611:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.208613:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.208614:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.208615:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.208616:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.208620:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513084.208621:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513084.208625:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.208626:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.208627:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.208634:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513084.208637:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513084.208637:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.208639:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.208643:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513084.208644:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:8.0:1616513084.208645:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b210 00000400:00000200:26.0:1616513084.208647:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.208647:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513084.208647:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513084.208649:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513084.208653:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.208659:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:26.0:1616513084.208662:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.216599:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.216654:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.216663:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.216672:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.41@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.216680:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.41@tcp1 of length 224 into portal 26 MB=0x6000af3dd0200 00000400:00000200:12.0:1616513084.216687:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.41@tcp1 of length 224/224 into md 0xf97ca9a9 [1] + 0 00000400:00000200:12.0:1616513084.216692:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf078 00000400:00000200:12.0:1616513084.216694:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.216697:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513084.216701:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf078 00000800:00000200:12.0:1616513084.216705:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.216706:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.216707:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.216710:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.216762:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688896901284352 00010000:00000200:24.0:1616513084.216772:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688896901284352/t0(0) o400->c2c02d9b-c57e-2c57-2f85-b1193e45f64c@10.31.132.41@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.216785:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688896901284352, offset 192 00000800:00000200:11.2:1616513084.216786:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:24.0:1616513084.216791:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.41@tcp1 00000800:00000200:8.0:1616513084.216793:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513084.216798:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.41@tcp1 routed destination 00000800:00000200:8.0:1616513084.216804:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.216809:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.41@tcp1 : PUT - for me 00000400:00000200:24.0:1616513084.216813:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.41@tcp1(10.31.132.41@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513084.216814:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.41@tcp1 of length 224 into portal 12 MB=0x6000af3dd0300 00000400:00000200:8.0:1616513084.216818:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.41@tcp1 of length 224/224 into md 0x865bbec1 [1] + 67984 00000800:00000200:24.0:1616513084.216819:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513084.216821:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.216822:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513084.216824:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513084.216826:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513084.216827:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513084.216827:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513084.216829:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513084.216829:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.216830:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513084.216833:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.216839:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.216840:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513084.216848:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688896901284608 00000800:00000200:14.0:1616513084.216848:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.216852:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.216854:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.216858:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:28.0:1616513084.216859:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8b600 x1688896901284608/t0(0) o400->ab2610af-8773-d6eb-437b-c95834e2e93d@10.31.132.41@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513084.216861:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:14.0:1616513084.216863:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.216865:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513084.216872:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688896901284608, offset 192 00000400:00000200:28.0:1616513084.216878:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.41@tcp1 00000400:00000200:28.0:1616513084.216885:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.41@tcp1 routed destination 00000400:00000200:28.0:1616513084.216899:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.41@tcp1(10.31.132.41@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513084.216905:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513084.216910:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513084.216912:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513084.216914:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.216918:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.216924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.216934:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.216938:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.216940:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513084.216942:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.216945:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.216948:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccc38 00000800:00000200:28.0:1616513084.216950:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.238029:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.238043:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.238052:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.238060:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.222@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.238069:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.222@tcp1 of length 224 into portal 26 MB=0x5ffd3d419e540 00000400:00000200:8.0:1616513084.238075:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.222@tcp1 of length 224/224 into md 0xf97ca9c9 [1] + 0 00000400:00000200:8.0:1616513084.238080:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdef68 00000400:00000200:8.0:1616513084.238083:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.238086:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.238090:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdef68 00000800:00000200:8.0:1616513084.238093:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.238096:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.238097:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.238101:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.238154:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660145202496 00010000:00000200:24.0:1616513084.238165:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688660145202496/t0(0) o400->42b7285f-d838-c8a9-98c0-149a78e71f9b@10.31.128.222@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.238179:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660145202496, offset 192 00000400:00000200:24.0:1616513084.238185:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.222@tcp1 00000400:00000200:24.0:1616513084.238192:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.222@tcp1 routed destination 00000400:00000200:24.0:1616513084.238207:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.222@tcp1(10.31.128.222@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.238213:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.238219:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.238223:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.238226:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.238231:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.238237:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.238238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.238294:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513084.238299:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513084.238301:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.238304:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.238311:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.238316:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2000 00000800:00000200:2.0:1616513084.238320:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.238323:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513084.238368:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.238377:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.238385:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.222@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.238394:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.222@tcp1 of length 224 into portal 12 MB=0x5ffd3d419e940 00000400:00000200:12.0:1616513084.238402:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.222@tcp1 of length 224/224 into md 0x865bbec1 [1] + 68208 00000400:00000200:12.0:1616513084.238407:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.238410:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.238430:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.238450:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.238451:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.238454:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.238498:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660145203520 00010000:00000200:0.0:1616513084.238511:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0de300 x1688660145203520/t0(0) o400->b597536c-d905-cc3f-c271-0779b22a8abf@10.31.128.222@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.238525:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660145203520, offset 192 00000400:00000200:0.0:1616513084.238531:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.222@tcp1 00000400:00000200:0.0:1616513084.238538:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.222@tcp1 routed destination 00000400:00000200:0.0:1616513084.238553:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.222@tcp1(10.31.128.222@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.238560:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.238565:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.238568:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.238570:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.238575:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.238580:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.238591:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.238596:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.238599:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.238599:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.238606:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.238610:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb055c8 00000800:00000200:20.0:1616513084.238614:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.260798:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.260851:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.260859:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.260867:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.179@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.260875:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.179@tcp1 of length 224 into portal 26 MB=0x604196b2a86c0 00000400:00000200:14.0:1616513084.260881:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.179@tcp1 of length 224/224 into md 0xf97ca9e9 [1] + 0 00000400:00000200:14.0:1616513084.260885:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdfe48 00000400:00000200:14.0:1616513084.260888:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.260890:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513084.260894:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfe48 00000800:00000200:14.0:1616513084.260897:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.260899:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.260901:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.260903:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.260958:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693357078906560 00010000:00000200:24.0:1616513084.260968:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1693357078906560/t0(0) o400->aab9c9b9-4f5c-726b-8fce-4d84b622e285@10.31.128.179@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.260982:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693357078906560, offset 192 00000400:00000200:24.0:1616513084.260989:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.179@tcp1 00000400:00000200:24.0:1616513084.260997:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.179@tcp1 routed destination 00000400:00000200:24.0:1616513084.261013:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.179@tcp1(10.31.128.179@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.261019:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.261025:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.261028:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.261030:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.261035:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.261041:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.261041:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.261100:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.261107:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.261110:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.261116:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.261122:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:2.0:1616513084.261125:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.261129:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.261266:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513084.261324:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.261333:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513084.261342:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.179@tcp1 : PUT - for me 00000400:00000200:28.0:1616513084.261352:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.179@tcp1 of length 224 into portal 12 MB=0x604196b2a8b00 00000400:00000200:28.0:1616513084.261358:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.179@tcp1 of length 224/224 into md 0x865bbec1 [1] + 68432 00000400:00000200:28.0:1616513084.261363:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513084.261365:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513084.261373:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.261376:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.261377:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513084.261381:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.261456:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693357078907648 00010000:00000200:0.0:1616513084.261469:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ec800 x1693357078907648/t0(0) o400->a0728c71-3ad0-6c49-2966-af8ab297a2ff@10.31.128.179@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.261484:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693357078907648, offset 192 00000400:00000200:0.0:1616513084.261490:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.179@tcp1 00000400:00000200:0.0:1616513084.261498:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.179@tcp1 routed destination 00000400:00000200:0.0:1616513084.261513:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.179@tcp1(10.31.128.179@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.261520:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.261525:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.261530:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.261532:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.261537:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.261543:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.261552:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.261556:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.261558:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.261563:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.261566:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb046e8 00000800:00000200:8.0:1616513084.261569:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.261571:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.266335:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.266390:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.266400:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.266408:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.165@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.266418:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.165@tcp1 of length 224 into portal 26 MB=0x60215d9549d80 00000400:00000200:12.0:1616513084.266424:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.165@tcp1 of length 224/224 into md 0xf97ca9f9 [1] + 0 00000400:00000200:12.0:1616513084.266429:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdeee0 00000400:00000200:12.0:1616513084.266431:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.266445:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513084.266446:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.266449:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdeee0 00000800:00000200:12.0:1616513084.266452:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.266455:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.266457:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.266512:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691142724033920 00010000:00000200:24.0:1616513084.266522:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1691142724033920/t0(0) o400->49875f23-a975-f38b-2a84-f97bdd6aae8d@10.31.140.165@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.266537:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691142724033920, offset 192 00000400:00000200:24.0:1616513084.266544:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.165@tcp1 00000400:00000200:24.0:1616513084.266551:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.165@tcp1 routed destination 00000400:00000200:24.0:1616513084.266566:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.165@tcp1(10.31.140.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.266573:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.266578:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.266581:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.266583:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.266587:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.266593:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.266594:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513084.266646:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.266653:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.266656:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.266662:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.266668:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2198 00000800:00000200:18.0:1616513084.266671:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.266674:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.266928:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.266937:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.266942:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.266948:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.165@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.266955:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.165@tcp1 of length 224 into portal 12 MB=0x60215d954a240 00000400:00000200:8.0:1616513084.266959:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.165@tcp1 of length 224/224 into md 0x865bbec1 [1] + 68656 00000400:00000200:8.0:1616513084.266963:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.266965:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513084.266970:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.266972:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.266974:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.266976:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.267035:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691142724035136 00010000:00000200:0.0:1616513084.267047:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb88900 x1691142724035136/t0(0) o400->3d546716-2340-eda4-88bf-0fc9b3bc2403@10.31.140.165@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.267061:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691142724035136, offset 192 00000400:00000200:0.0:1616513084.267068:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.165@tcp1 00000400:00000200:0.0:1616513084.267075:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.165@tcp1 routed destination 00000400:00000200:0.0:1616513084.267090:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.165@tcp1(10.31.140.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.267097:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.267102:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.267105:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.267107:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.267112:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.267117:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.267128:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.267134:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.267136:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513084.267138:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.267143:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.267147:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05d38 00000800:00000200:28.0:1616513084.267151:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.276317:0:49904:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.276373:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.276382:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513084.276388:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.276391:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.199@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.276400:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.199@tcp1 of length 224 into portal 26 MB=0x60186bab23040 00000400:00000200:2.0:1616513084.276407:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.199@tcp1 of length 224/224 into md 0xf97caa09 [1] + 0 00000400:00000200:2.0:1616513084.276411:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdff58 00000400:00000200:2.0:1616513084.276414:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.276417:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.276421:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdff58 00000800:00000200:2.0:1616513084.276424:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.276427:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.276428:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.276483:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690528029749312 00010000:00000200:24.0:1616513084.276494:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1690528029749312/t0(0) o400->3041e8ab-dff3-5845-e559-7e5b668c7203@10.31.142.199@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.276508:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690528029749312, offset 192 00000400:00000200:24.0:1616513084.276515:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.199@tcp1 00000400:00000200:24.0:1616513084.276522:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.199@tcp1 routed destination 00000400:00000200:24.0:1616513084.276537:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.199@tcp1(10.31.142.199@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.276544:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.276549:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.276552:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.276555:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.276559:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.276565:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.276565:0:49904:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.276619:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.276625:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.276628:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.276635:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.276640:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000800:00000200:18.0:1616513084.276644:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.276647:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.276699:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.276711:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.276716:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.276722:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.199@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.276728:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.199@tcp1 of length 224 into portal 12 MB=0x60186bab239c0 00000400:00000200:8.0:1616513084.276733:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.199@tcp1 of length 224/224 into md 0x865bbec1 [1] + 68880 00000400:00000200:8.0:1616513084.276736:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.276738:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513084.276742:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.276744:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.276746:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.276748:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.276816:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690528029751744 00010000:00000200:0.0:1616513084.276829:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8c800 x1690528029751744/t0(0) o400->bd005e0a-3cd0-d9e9-e2c5-9eaa0472f4c9@10.31.142.199@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.276843:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690528029751744, offset 192 00000400:00000200:0.0:1616513084.276850:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.199@tcp1 00000400:00000200:0.0:1616513084.276857:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.199@tcp1 routed destination 00000400:00000200:0.0:1616513084.276872:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.199@tcp1(10.31.142.199@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.276878:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.276884:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.276887:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.276889:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.276893:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.276898:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.276952:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.276958:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.276961:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513084.276968:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513084.276972:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:12.0:1616513084.276976:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.276979:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:2.0:1616513084.282512:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926512, portal 4 00000100:00000200:2.0:1616513084.282518:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926512, offset 0 00000100:00000200:27.0:1616513084.282522:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926528, portal 4 00000400:00000200:2.0:1616513084.282524:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000100:00000200:27.0:1616513084.282528:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926528, offset 0 00000400:00000200:2.0:1616513084.282530:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:27.0:1616513084.282537:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:27.0:1616513084.282543:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:2.0:1616513084.282544:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513084.282552:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000400:00000200:27.0:1616513084.282558:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513084.282558:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513084.282562:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:2.0:1616513084.282564:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513084.282565:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:2.0:1616513084.282569:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:27.0:1616513084.282572:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513084.282575:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513084.282578:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:5.2:1616513084.282578:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513084.282583:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:8.0:1616513084.282588:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:27.2:1616513084.282590:0:6510:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:8.0:1616513084.282591:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513084.282593:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.282597:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513084.282602:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bda880 x1693067725926512/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513084.282612:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed188 00000800:00000200:8.0:1616513084.282615:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513084.282617:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:31.0:1616513084.282645:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513084.282655:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:31.0:1616513084.282658:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:31.0:1616513084.282665:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:31.0:1616513084.282671:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8f30a8b180 x1693067725926528/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513084.282684:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248440 00000800:00000200:31.0:1616513084.282688:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:31.0:1616513084.282691:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:5.2:1616513084.282788:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:27.2:1616513084.282796:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513084.282807:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.282813:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513084.282815:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513084.282821:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513084.282828:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627080 00000400:00000200:27.0:1616513084.282834:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166d2d [1] + 192 00000400:00000200:27.0:1616513084.282839:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513084.282840:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:27.0:1616513084.282843:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8f30a8b180 x1693067725926528/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513084.282849:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:28.0:1616513084.282855:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:27.0:1616513084.282858:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:12.0:1616513084.282858:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:27.0:1616513084.282860:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513084.282862:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:12.0:1616513084.282865:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627070 00000100:00000200:31.0:1616513084.282866:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8f30a8b180 x1693067725926528/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513084.282872:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d7c51 [1] + 192 00000100:00000200:31.0:1616513084.282878:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8f30a8b180 x1693067725926528/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513084.282881:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.282886:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bda880 x1693067725926512/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513084.282888:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248088 00000800:00000200:12.0:1616513084.282899:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513084.282902:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513084.282903:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:2.0:1616513084.282908:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bda880 x1693067725926512/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513084.282921:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bda880 x1693067725926512/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513084.282931:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec440 00000800:00000200:31.2:1616513084.294872:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:26.0:1616513084.294930:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:26.0:1616513084.294940:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:26.0:1616513084.294948:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513084.294956:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 224 into portal 24 MB=0x603db0154c510 00000400:00000200:26.0:1616513084.294965:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 18 from 12345-10.242.46.234@o2ib1 of length 224/224 into md 0xf9738cb9 [1] + 0 00000400:00000200:26.0:1616513084.294970:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d747a6cd4b8 00000400:00000200:26.0:1616513084.294973:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.294976:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt_out 00000400:00000200:26.0:1616513084.294980:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd4b8 00000800:00000200:26.0:1616513084.294984:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:2.0:1616513084.294985:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:26.0:1616513084.294988:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:26.0:1616513084.294989:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:14.0:1616513084.295047:0:185125:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693089015317776 00010000:00000200:14.0:1616513084.295069:0:185125:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d741ca80050 x1693089015317776/t0(0) o41->boslfs02-MDT0000-mdtlov_UUID@10.242.46.234@o2ib1:710/0 lens 224/368 e 0 to 0 dl 1616513090 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:14.0:1616513084.295095:0:185125:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 4, xid 1693089015317776, offset 192 00000400:00000200:14.0:1616513084.295101:0:185125:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:14.0:1616513084.295108:0:185125:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:14.0:1616513084.295122:0:185125:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513084.295128:0:185125:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:14.0:1616513084.295134:0:185125:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513084.295137:0:185125:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:14.0:1616513084.295139:0:185125:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:14.0:1616513084.295144:0:185125:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513084.295149:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:14.0:1616513084.295151:0:185125:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000800:00000200:18.0:1616513084.295208:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:18.0:1616513084.295215:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:18.0:1616513084.295218:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513084.295220:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:18.0:1616513084.295224:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.295230:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756770 00000800:00000200:18.0:1616513084.295234:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:7.2:1616513084.299004:0:49906:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.299060:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.299069:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.299078:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.209@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.299087:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.209@tcp1 of length 224 into portal 12 MB=0x5ffd3f1005780 00000400:00000200:2.0:1616513084.299096:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.209@tcp1 of length 224/224 into md 0xf964d519 [1] + 34600 00000400:00000200:2.0:1616513084.299102:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.299105:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.299112:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.299114:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.299115:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.299119:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.299179:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660630067072 00010000:00000200:22.0:1616513084.299191:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d6780 x1688660630067072/t0(0) o400->f6faef0a-4e03-96ee-92f2-ab3b65bfaceb@10.31.136.209@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.299205:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660630067072, offset 192 00000400:00000200:22.0:1616513084.299212:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.209@tcp1 00000400:00000200:22.0:1616513084.299219:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.209@tcp1 routed destination 00000400:00000200:22.0:1616513084.299234:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.209@tcp1(10.31.136.209@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.299240:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513084.299246:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.299248:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.299250:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.299255:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.299259:0:49906:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.299315:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.299321:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.299324:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.299331:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.299335:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a220 00000800:00000200:14.0:1616513084.299339:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.299341:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.299495:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.299554:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.299564:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.299573:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.209@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.299581:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.209@tcp1 of length 224 into portal 26 MB=0x5ffd3f1005700 00000400:00000200:8.0:1616513084.299588:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.209@tcp1 of length 224/224 into md 0xf97caa19 [1] + 0 00000400:00000200:8.0:1616513084.299593:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde550 00000400:00000200:8.0:1616513084.299595:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.299598:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.299603:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde550 00000800:00000200:8.0:1616513084.299606:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.299608:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.299609:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.299612:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.299668:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660630066944 00010000:00000200:24.0:1616513084.299679:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688660630066944/t0(0) o400->be92c48a-01a6-e59b-b05a-eee727e0c4d1@10.31.136.209@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.299693:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660630066944, offset 192 00000400:00000200:24.0:1616513084.299699:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.209@tcp1 00000400:00000200:24.0:1616513084.299706:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.209@tcp1 routed destination 00000400:00000200:24.0:1616513084.299721:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.209@tcp1(10.31.136.209@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.299728:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.299733:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.299736:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.299738:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.299743:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513084.299749:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.299750:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.299805:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.299812:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.299815:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.299821:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.299827:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:28.0:1616513084.299830:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.299833:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.315480:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.315535:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.315545:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.315554:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.160@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.315563:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.160@tcp1 of length 224 into portal 26 MB=0x603e135ee0f80 00000400:00000200:26.0:1616513084.315570:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.160@tcp1 of length 224/224 into md 0xf97caa29 [1] + 0 00000400:00000200:26.0:1616513084.315574:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde088 00000400:00000200:26.0:1616513084.315577:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.315580:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513084.315584:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde088 00000800:00000200:26.0:1616513084.315588:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.315589:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.315591:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.315596:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.315599:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.315599:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.315607:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.160@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.315615:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.160@tcp1 of length 224 into portal 12 MB=0x603e135ee1100 00000400:00000200:2.0:1616513084.315621:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.160@tcp1 of length 224/224 into md 0xf964d519 [1] + 34824 00000400:00000200:2.0:1616513084.315626:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.315629:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.315636:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.315639:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.315640:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.315649:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115667582848 00010000:00000200:24.0:1616513084.315660:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1693115667582848/t0(0) o400->2070c0b6-d92f-0727-92c3-912b382acacd@10.31.136.160@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.315674:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693115667582848, offset 192 00000400:00000200:24.0:1616513084.315682:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.160@tcp1 00000400:00000200:24.0:1616513084.315689:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.160@tcp1 routed destination 00000400:00000200:24.0:1616513084.315705:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.160@tcp1(10.31.136.160@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513084.315707:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115667583232 00000800:00000200:24.0:1616513084.315712:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.315718:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513084.315720:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d4380 x1693115667583232/t0(0) o400->8653706c-8a25-05b3-367a-b63dcfb7b73b@10.31.136.160@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513084.315721:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.315723:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.315728:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.315733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.315735:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513084.315735:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693115667583232, offset 192 00000400:00000200:22.0:1616513084.315742:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.160@tcp1 00000400:00000200:22.0:1616513084.315749:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.160@tcp1 routed destination 00000400:00000200:22.0:1616513084.315764:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.160@tcp1(10.31.136.160@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.315770:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.315776:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.315778:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.315781:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.315785:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513084.315787:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513084.315792:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513084.315794:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.315797:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513084.315798:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513084.315804:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.315809:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:18.0:1616513084.315813:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513084.315847:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.315854:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.315857:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.315858:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.315864:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.315870:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a440 00000800:00000200:20.0:1616513084.315874:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.322400:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.322458:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.322467:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513084.322472:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.322476:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.223@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.322485:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.223@tcp1 of length 224 into portal 26 MB=0x60185c6aa57c0 00000400:00000200:2.0:1616513084.322491:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.223@tcp1 of length 224/224 into md 0xf97caa39 [1] + 0 00000400:00000200:2.0:1616513084.322496:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf7e8 00000400:00000200:2.0:1616513084.322499:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.322501:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.322505:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf7e8 00000800:00000200:2.0:1616513084.322508:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.322511:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.322513:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.322569:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690523935594432 00010000:00000200:24.0:1616513084.322580:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1690523935594432/t0(0) o400->1667ec72-c2d2-89ed-ab90-29757247af39@10.31.132.223@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.322594:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690523935594432, offset 192 00000400:00000200:24.0:1616513084.322600:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.223@tcp1 00000400:00000200:24.0:1616513084.322607:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.223@tcp1 routed destination 00000800:00000200:7.2:1616513084.322618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513084.322622:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.223@tcp1(10.31.132.223@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.322628:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.322634:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.322637:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.322639:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513084.322644:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:24.0:1616513084.322650:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513084.322672:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.322682:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.322691:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.223@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.322699:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.223@tcp1 of length 224 into portal 12 MB=0x60185c6aa5940 00000400:00000200:26.0:1616513084.322705:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.223@tcp1 of length 224/224 into md 0xf964d519 [1] + 35048 00000400:00000200:26.0:1616513084.322711:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.322713:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513084.322720:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513084.322723:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513084.322727:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513084.322734:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513084.322735:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513084.322737:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.322740:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.322746:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.322751:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000800:00000200:18.0:1616513084.322755:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.322784:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690523935594816 00010000:00000200:22.0:1616513084.322796:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdc57f980 x1690523935594816/t0(0) o400->3de65e36-d4cc-609c-c3ef-02d560baea45@10.31.132.223@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.322810:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690523935594816, offset 192 00000400:00000200:22.0:1616513084.322817:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.223@tcp1 00000400:00000200:22.0:1616513084.322824:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.223@tcp1 routed destination 00000400:00000200:22.0:1616513084.322840:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.223@tcp1(10.31.132.223@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.322847:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.322853:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.322855:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.322858:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.322862:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.322869:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.322893:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.322900:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.322903:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.322909:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.322913:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a6e8 00000800:00000200:28.0:1616513084.322917:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.322920:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.324411:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.324424:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.324432:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.324446:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.244@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.324456:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.244@tcp1 of length 224 into portal 26 MB=0x604f6991605c0 00000400:00000200:8.0:1616513084.324463:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.244@tcp1 of length 224/224 into md 0xf97caa49 [1] + 0 00000400:00000200:8.0:1616513084.324467:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde4c8 00000400:00000200:8.0:1616513084.324470:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.324473:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513084.324477:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.324477:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde4c8 00000800:00000200:8.0:1616513084.324480:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.324483:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.324485:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.324490:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694307037087168 00010000:00000200:24.0:1616513084.324495:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1694307037087168/t0(0) o400->30f68e00-da63-612c-2791-408a9971450b@10.31.140.244@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.324507:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694307037087168, offset 192 00000400:00000200:24.0:1616513084.324513:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.244@tcp1 00000400:00000200:24.0:1616513084.324519:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.244@tcp1 routed destination 00000400:00000200:24.0:1616513084.324532:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.244@tcp1(10.31.140.244@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.324542:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.324547:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.324549:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.324551:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.324554:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.324558:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.324559:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.324613:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.324620:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.324622:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.324629:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.324637:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:2.0:1616513084.324640:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.324643:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.324725:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.324778:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.324787:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.324796:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.244@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.324805:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.244@tcp1 of length 224 into portal 12 MB=0x604f6991608c0 00000400:00000200:26.0:1616513084.324811:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.244@tcp1 of length 224/224 into md 0xf964d519 [1] + 35272 00000400:00000200:26.0:1616513084.324816:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.324819:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513084.324826:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.324829:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.324831:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.324835:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.324889:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694307037087936 00010000:00000200:22.0:1616513084.324901:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cd948e300 x1694307037087936/t0(0) o400->7db61b45-6658-0e7a-02b9-dfec69397c87@10.31.140.244@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.324915:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694307037087936, offset 192 00000400:00000200:22.0:1616513084.324921:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.244@tcp1 00000400:00000200:22.0:1616513084.324929:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.244@tcp1 routed destination 00000400:00000200:22.0:1616513084.324944:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.244@tcp1(10.31.140.244@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.324951:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.324956:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.324959:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.324961:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.324966:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.324971:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.325025:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.325034:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.325036:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.325043:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.325047:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a088 00000800:00000200:28.0:1616513084.325051:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.325054:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.325060:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513084.325062:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.325067:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.157@tcp1 : PUT - for me 00000400:00000200:28.0:1616513084.325076:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.157@tcp1 of length 224 into portal 26 MB=0x5fff584295a00 00000400:00000200:28.0:1616513084.325082:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.157@tcp1 of length 224/224 into md 0xf97caa69 [1] + 0 00000400:00000200:28.0:1616513084.325087:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde5d8 00000400:00000200:28.0:1616513084.325090:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513084.325092:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513084.325096:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde5d8 00000800:00000200:28.0:1616513084.325099:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.325102:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.325103:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.325107:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688804832926208 00010000:00000200:24.0:1616513084.325111:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688804832926208/t0(0) o400->8bc38b6e-dac9-cc19-23a0-69475d1fea3a@10.31.141.157@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.325122:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688804832926208, offset 192 00000400:00000200:24.0:1616513084.325126:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.157@tcp1 00000400:00000200:24.0:1616513084.325132:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.157@tcp1 routed destination 00000400:00000200:24.0:1616513084.325145:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.157@tcp1(10.31.141.157@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.325150:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.325155:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.325157:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.325159:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.325161:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.325165:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.325167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.325176:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.325180:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.325182:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513084.325186:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.325187:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.325190:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:2.0:1616513084.325193:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.325202:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.325213:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.325221:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513084.325222:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513084.325229:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.157@tcp1 : PUT - for me 00000400:00000200:18.0:1616513084.325237:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.157@tcp1 of length 224 into portal 12 MB=0x5fff584295a80 00000400:00000200:18.0:1616513084.325244:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.157@tcp1 of length 224/224 into md 0xf964d519 [1] + 35496 00000400:00000200:18.0:1616513084.325249:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513084.325252:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513084.325259:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.325262:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.325264:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.325270:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688804832926336 00010000:00000200:22.0:1616513084.325276:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfd580 x1688804832926336/t0(0) o400->2ef60498-c585-904a-1615-e8515f0d5cb7@10.31.141.157@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.325287:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688804832926336, offset 192 00000400:00000200:22.0:1616513084.325293:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.157@tcp1 00000400:00000200:22.0:1616513084.325298:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.157@tcp1 routed destination 00000400:00000200:22.0:1616513084.325311:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.157@tcp1(10.31.141.157@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.325317:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.325322:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.325324:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.325327:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.325331:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.325335:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.325343:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.325346:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.325349:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513084.325350:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.325354:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.325357:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ab28 00000800:00000200:8.0:1616513084.325359:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:23.0:1616513084.330824:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926544, portal 4 00000100:00000200:23.0:1616513084.330830:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926544, offset 0 00000400:00000200:23.0:1616513084.330837:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:23.0:1616513084.330844:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:23.0:1616513084.330858:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513084.330865:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:23.0:1616513084.330872:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513084.330875:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:23.0:1616513084.330877:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:23.0:1616513084.330883:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513084.330930:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513084.330987:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513084.330994:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513084.330998:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.331005:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513084.331011:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be41b00 x1693067725926544/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513084.331023:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fba0 00000800:00000200:14.0:1616513084.331028:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513084.331031:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513084.331097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513084.331151:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513084.331160:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:26.0:1616513084.331169:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513084.331177:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627090 00000400:00000200:26.0:1616513084.331184:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cb0f9 [1] + 192 00000400:00000200:26.0:1616513084.331189:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.331194:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be41b00 x1693067725926544/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513084.331207:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513084.331209:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513084.331212:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513084.331214:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:23.0:1616513084.331220:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be41b00 x1693067725926544/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513084.331233:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be41b00 x1693067725926544/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513084.331247:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f4b8 00000800:00000200:7.2:1616513084.359564:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.359619:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.359629:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.359638:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.141@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.359650:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.141@tcp1 of length 224 into portal 26 MB=0x5ffde30777280 00000400:00000200:14.0:1616513084.359656:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.141@tcp1 of length 224/224 into md 0xf97caa79 [1] + 0 00000400:00000200:14.0:1616513084.359661:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf4b8 00000400:00000200:14.0:1616513084.359664:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.359666:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513084.359671:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf4b8 00000800:00000200:14.0:1616513084.359674:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.359676:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.359677:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.359683:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.359686:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.359687:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.359695:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.141@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.359705:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.141@tcp1 of length 224 into portal 12 MB=0x5ffde30777300 00000400:00000200:2.0:1616513084.359711:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.141@tcp1 of length 224/224 into md 0xf964d519 [1] + 35720 00000400:00000200:2.0:1616513084.359716:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.359719:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.359727:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.359730:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.359732:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.359739:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704644510336 00010000:00000200:24.0:1616513084.359751:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688704644510336/t0(0) o400->90c6ae5a-28b0-6964-d239-7e1609adf4b6@10.31.142.141@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.359766:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704644510336, offset 192 00000400:00000200:24.0:1616513084.359772:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.141@tcp1 00000400:00000200:24.0:1616513084.359780:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.141@tcp1 routed destination 00000100:00000200:22.0:1616513084.359791:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704644510464 00000400:00000200:24.0:1616513084.359796:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.141@tcp1(10.31.142.141@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.359802:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:22.0:1616513084.359803:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d6c00 x1688704644510464/t0(0) o400->b7a71afa-0a60-5945-7d81-e3c943febdc9@10.31.142.141@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513084.359809:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.359811:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.359815:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513084.359817:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704644510464, offset 192 00000800:00000200:24.0:1616513084.359821:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513084.359822:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.141@tcp1 00000800:00000200:7.2:1616513084.359826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.359828:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513084.359830:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.141@tcp1 routed destination 00000800:00000200:26.0:1616513084.359843:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513084.359844:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.141@tcp1(10.31.142.141@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.359850:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513084.359853:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.359854:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.359855:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513084.359855:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.359857:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.359859:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513084.359861:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513084.359864:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513084.359867:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:26.0:1616513084.359870:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.359871:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513084.359873:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.359884:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.359893:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513084.359895:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513084.359901:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.170@tcp1 : PUT - for me 00000400:00000200:18.0:1616513084.359911:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.170@tcp1 of length 224 into portal 26 MB=0x604a295c419c0 00000400:00000200:18.0:1616513084.359917:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.170@tcp1 of length 224/224 into md 0xf97caa89 [1] + 0 00000400:00000200:18.0:1616513084.359922:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf430 00000400:00000200:18.0:1616513084.359925:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513084.359926:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:18.0:1616513084.359928:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513084.359932:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf430 00000800:00000200:12.0:1616513084.359933:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513084.359935:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513084.359935:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513084.359938:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.359940:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513084.359942:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.359942:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:24.0:1616513084.359945:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946204133824 00000400:00000200:12.0:1616513084.359946:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b3a8 00000800:00000200:12.0:1616513084.359950:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:24.0:1616513084.359951:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1693946204133824/t0(0) o400->ed1ae968-6ab9-f7ff-4e67-f40415e8bca1@10.31.140.170@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.359962:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946204133824, offset 192 00000400:00000200:24.0:1616513084.359967:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.170@tcp1 00000400:00000200:24.0:1616513084.359973:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.170@tcp1 routed destination 00000400:00000200:24.0:1616513084.359986:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.170@tcp1(10.31.140.170@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.359992:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.359996:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.359999:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:7.2:1616513084.360000:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.360001:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513084.360004:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.360008:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:24.0:1616513084.360009:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513084.360013:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513084.360015:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513084.360019:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513084.360019:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.170@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.360020:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513084.360023:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.360026:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.170@tcp1 of length 224 into portal 12 MB=0x604a295c41ac0 00000400:00000200:26.0:1616513084.360028:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.360031:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3c28 00000400:00000200:14.0:1616513084.360031:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.170@tcp1 of length 224/224 into md 0xf964d519 [1] + 35944 00000800:00000200:26.0:1616513084.360033:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513084.360034:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.360037:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.360042:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.360044:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.360045:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.360053:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946204134080 00010000:00000200:22.0:1616513084.360059:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464622880 x1693946204134080/t0(0) o400->f5178e28-3459-c4c4-e290-81511edf3530@10.31.140.170@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.360070:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946204134080, offset 192 00000400:00000200:22.0:1616513084.360075:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.170@tcp1 00000400:00000200:22.0:1616513084.360080:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.170@tcp1 routed destination 00000400:00000200:22.0:1616513084.360092:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.170@tcp1(10.31.140.170@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.360098:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.360102:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.360104:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.360106:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.360108:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.360113:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.360175:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.360182:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.360185:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.360187:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513084.360191:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.360195:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b7e8 00000800:00000200:8.0:1616513084.360195:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513084.360196:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.360197:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513084.360199:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.360199:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.360202:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.375406:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.375461:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.375474:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.375482:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.51@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.375492:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.51@tcp1 of length 224 into portal 26 MB=0x6000e36c50e80 00000400:00000200:12.0:1616513084.375499:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.51@tcp1 of length 224/224 into md 0xf97caa99 [1] + 0 00000400:00000200:12.0:1616513084.375504:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf320 00000400:00000200:12.0:1616513084.375507:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.375510:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513084.375514:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf320 00000800:00000200:12.0:1616513084.375517:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.375520:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513084.375521:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.375525:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.375579:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688910908690048 00010000:00000200:24.0:1616513084.375591:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688910908690048/t0(0) o400->96e43ca1-1f33-0131-d94a-8a0361270428@10.31.133.51@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.375606:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688910908690048, offset 192 00000400:00000200:24.0:1616513084.375612:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.51@tcp1 00000400:00000200:24.0:1616513084.375620:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.51@tcp1 routed destination 00000400:00000200:24.0:1616513084.375635:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.51@tcp1(10.31.133.51@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.375642:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.375648:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.375651:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.375653:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.375658:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513084.375662:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513084.375665:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.375665:0:49914:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513084.375715:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.375725:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513084.375727:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.375733:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.51@tcp1 : PUT - for me 00000800:00000200:18.0:1616513084.375733:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.375740:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513084.375742:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.51@tcp1 of length 224 into portal 12 MB=0x6000e36c50f00 00000400:00000200:18.0:1616513084.375743:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.375749:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.51@tcp1 of length 224/224 into md 0x865bbec1 [1] + 69104 00000400:00000200:18.0:1616513084.375750:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.375754:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.375756:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000100:00000200:20.0:1616513084.375765:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513084.375765:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.375768:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513084.375773:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.375776:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.375778:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.375839:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688910908690176 00010000:00000200:0.0:1616513084.375851:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cef080 x1688910908690176/t0(0) o400->c68a4370-3a6f-ca49-4635-26e25ef7b534@10.31.133.51@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.375875:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688910908690176, offset 192 00000400:00000200:0.0:1616513084.375881:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.51@tcp1 00000400:00000200:0.0:1616513084.375888:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.51@tcp1 routed destination 00000400:00000200:0.0:1616513084.375902:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.51@tcp1(10.31.133.51@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.375908:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.375913:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.375916:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.375918:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.375922:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.375927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.375938:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.375943:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.375946:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513084.375947:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.375952:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.375956:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04440 00000800:00000200:28.0:1616513084.375960:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:11.0:1616513084.378573:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926560, portal 4 00000100:00000200:11.0:1616513084.378578:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926560, offset 0 00000400:00000200:11.0:1616513084.378585:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:11.0:1616513084.378591:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:11.0:1616513084.378604:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513084.378611:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:11.0:1616513084.378618:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513084.378621:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:11.0:1616513084.378623:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:11.0:1616513084.378628:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513084.378675:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513084.378688:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513084.378693:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513084.378696:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.378703:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513084.378709:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600900 x1693067725926560/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513084.378722:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3b650 00000800:00000200:8.0:1616513084.378725:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513084.378728:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513084.378869:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:12.0:1616513084.378923:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513084.378932:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:12.0:1616513084.378941:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513084.378948:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6270a0 00000400:00000200:12.0:1616513084.378956:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d7cf1 [1] + 192 00000400:00000200:12.0:1616513084.378961:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.378966:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600900 x1693067725926560/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513084.378976:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513084.378983:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513084.378986:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513084.378988:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:11.0:1616513084.378989:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600900 x1693067725926560/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513084.379008:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600900 x1693067725926560/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513084.379016:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bc28 00000800:00000200:11.2:1616513084.386338:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.386349:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.386355:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.386363:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.198@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.386371:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.198@tcp1 of length 224 into portal 26 MB=0x5ffd474fb4240 00000400:00000200:8.0:1616513084.386378:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.198@tcp1 of length 224/224 into md 0xf97caab9 [1] + 0 00000400:00000200:8.0:1616513084.386382:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdfd38 00000400:00000200:8.0:1616513084.386385:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.386387:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.386391:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfd38 00000800:00000200:8.0:1616513084.386394:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.386396:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.386398:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.386400:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.386405:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513084.386406:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.386412:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.198@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.386417:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.198@tcp1 of length 224 into portal 12 MB=0x5ffd474fb42c0 00000400:00000200:8.0:1616513084.386422:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.198@tcp1 of length 224/224 into md 0x865bbec1 [1] + 69328 00000400:00000200:8.0:1616513084.386426:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.386428:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513084.386433:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.386441:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.386443:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.386462:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662844326592 00000100:00000200:24.0:1616513084.386467:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662844326464 00010000:00000200:0.0:1616513084.386475:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8da00 x1688662844326592/t0(0) o400->e33ec66c-0faf-39b4-085d-9cc999e1c02a@10.31.132.198@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513084.386478:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688662844326464/t0(0) o400->f3a356a5-ed90-4f8c-112c-c66ed4841a23@10.31.132.198@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513084.386490:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662844326592, offset 192 00000400:00000200:0.0:1616513084.386496:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.198@tcp1 00000100:00000200:24.0:1616513084.386499:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662844326464, offset 192 00000400:00000200:0.0:1616513084.386504:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.198@tcp1 routed destination 00000400:00000200:24.0:1616513084.386505:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.198@tcp1 00000400:00000200:24.0:1616513084.386513:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.198@tcp1 routed destination 00000400:00000200:0.0:1616513084.386520:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.198@tcp1(10.31.132.198@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.386527:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513084.386529:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.198@tcp1(10.31.132.198@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.386536:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.386537:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.386540:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.386542:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.386542:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.386544:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.386546:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.386547:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513084.386551:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513084.386553:0:49915:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513084.386557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.386558:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513084.386609:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.386616:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.386619:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.386625:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513084.386626:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513084.386629:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04908 00000800:00000200:26.0:1616513084.386633:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513084.386633:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513084.386636:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513084.386636:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513084.386637:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.386643:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513084.386648:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2cc0 00000800:00000200:12.0:1616513084.386652:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.388474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.388532:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.388542:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513084.388551:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.196@tcp1 : PUT - for me 00000400:00000200:18.0:1616513084.388561:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.196@tcp1 of length 224 into portal 26 MB=0x5ffe3bf05d700 00000400:00000200:18.0:1616513084.388567:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.196@tcp1 of length 224/224 into md 0xf97caac9 [1] + 0 00000400:00000200:18.0:1616513084.388572:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fded48 00000400:00000200:18.0:1616513084.388575:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513084.388578:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513084.388582:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fded48 00000800:00000200:18.0:1616513084.388585:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.388587:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.388588:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.388591:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.388598:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.388656:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.388665:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:24.0:1616513084.388669:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728511043328 00000400:00000200:26.0:1616513084.388674:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.196@tcp1 : PUT - for me 00010000:00000200:24.0:1616513084.388679:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688728511043328/t0(0) o400->4695f5e3-9348-3ebf-6616-c24a0e6f58cb@10.31.128.196@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513084.388683:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.196@tcp1 of length 224 into portal 12 MB=0x5ffe3bf05d9c0 00000400:00000200:26.0:1616513084.388690:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.196@tcp1 of length 224/224 into md 0xf964d519 [1] + 36168 00000100:00000200:24.0:1616513084.388694:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688728511043328, offset 192 00000400:00000200:26.0:1616513084.388695:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.388698:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513084.388700:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.196@tcp1 00000800:00000200:26.0:1616513084.388706:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513084.388708:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.196@tcp1 routed destination 00000800:00000200:26.0:1616513084.388709:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.388709:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.388714:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513084.388723:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.196@tcp1(10.31.128.196@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.388729:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.388735:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.388738:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.388740:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.388745:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.388750:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.388751:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513084.388761:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.388766:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.388769:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513084.388770:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.388775:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:22.0:1616513084.388776:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728511044032 00010000:00000200:22.0:1616513084.388788:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cd9489b00 x1688728511044032/t0(0) o400->f386f65d-f229-1a43-151d-bac423867e2b@10.31.128.196@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513084.388788:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:2.0:1616513084.388791:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.388802:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688728511044032, offset 192 00000400:00000200:22.0:1616513084.388811:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.196@tcp1 00000400:00000200:22.0:1616513084.388818:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.196@tcp1 routed destination 00000400:00000200:22.0:1616513084.388833:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.196@tcp1(10.31.128.196@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.388840:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.388845:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.388848:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.388850:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.388855:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.388860:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.388869:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.388873:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.388875:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.388880:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.388884:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a198 00000800:00000200:8.0:1616513084.388887:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.388889:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.408595:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.408652:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.408661:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.408671:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.92@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.408681:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.92@tcp1 of length 224 into portal 26 MB=0x60184b247b580 00000400:00000200:14.0:1616513084.408688:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.92@tcp1 of length 224/224 into md 0xf97caad9 [1] + 0 00000400:00000200:14.0:1616513084.408693:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde330 00000400:00000200:14.0:1616513084.408696:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.408699:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513084.408703:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde330 00000800:00000200:14.0:1616513084.408706:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.408707:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.408711:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.408713:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.408768:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519298618752 00010000:00000200:24.0:1616513084.408779:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1690519298618752/t0(0) o400->e9fe44f5-7e1d-3b1d-5d3c-b2ff51476bac@10.31.142.92@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.408793:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690519298618752, offset 192 00000400:00000200:24.0:1616513084.408799:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.92@tcp1 00000400:00000200:24.0:1616513084.408819:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.92@tcp1 routed destination 00000400:00000200:24.0:1616513084.408835:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.92@tcp1(10.31.142.92@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.408841:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.408846:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.408849:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.408851:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.408856:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.408861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.408862:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513084.408914:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.408921:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.408924:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.408930:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.408936:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:18.0:1616513084.408939:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.408942:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.409065:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.409076:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.409084:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513084.409086:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513084.409094:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.92@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.409102:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.92@tcp1 of length 224 into portal 12 MB=0x60184b247c240 00000400:00000200:26.0:1616513084.409109:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.92@tcp1 of length 224/224 into md 0xf964d519 [1] + 36392 00000400:00000200:26.0:1616513084.409114:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.409117:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513084.409124:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.409127:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.409129:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.409188:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519298622016 00010000:00000200:22.0:1616513084.409200:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cd948f980 x1690519298622016/t0(0) o400->29fb94b0-e0d0-d37e-8be7-b36ca6d87383@10.31.142.92@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.409213:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690519298622016, offset 192 00000400:00000200:22.0:1616513084.409219:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.92@tcp1 00000400:00000200:22.0:1616513084.409226:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.92@tcp1 routed destination 00000400:00000200:22.0:1616513084.409240:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.92@tcp1(10.31.142.92@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.409246:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.409251:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.409254:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.409256:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.409260:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.409267:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.409321:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.409328:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.409331:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.409338:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.409342:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b540 00000800:00000200:20.0:1616513084.409346:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.409349:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.414347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513084.414384:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.414394:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.414401:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:18.0:1616513084.414404:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513084.414409:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.168@tcp1 : PUT - for me 00000800:00000200:18.0:1616513084.414414:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:8.0:1616513084.414418:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.168@tcp1 of length 224 into portal 12 MB=0x6041217abc180 00000400:00000200:18.0:1616513084.414422:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.168@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.414423:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.168@tcp1 of length 224/224 into md 0x865bbec1 [1] + 69552 00000400:00000200:8.0:1616513084.414428:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.414430:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513084.414431:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.168@tcp1 of length 224 into portal 26 MB=0x6041217abc100 00000800:00000200:8.0:1616513084.414441:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.414444:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.414445:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.414448:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513084.414451:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.168@tcp1 of length 224/224 into md 0xf97caae9 [1] + 0 00000400:00000200:18.0:1616513084.414455:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde880 00000400:00000200:18.0:1616513084.414458:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513084.414461:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513084.414464:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde880 00000800:00000200:2.0:1616513084.414465:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.414467:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.414470:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.414474:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513084.414503:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693325613318528 00010000:00000200:0.0:1616513084.414515:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8f500 x1693325613318528/t0(0) o400->2438a571-10ca-f63c-9a96-81c6fa890c72@10.242.104.168@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.414528:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693325613318528, offset 192 00000400:00000200:0.0:1616513084.414534:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.168@tcp1 00000100:00000200:24.0:1616513084.414535:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693325613318400 00000400:00000200:0.0:1616513084.414541:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.168@tcp1 routed destination 00010000:00000200:24.0:1616513084.414546:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1693325613318400/t0(0) o400->40310153-ea95-379d-c52c-bd650f0b6d00@10.242.104.168@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513084.414555:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.168@tcp1(10.242.104.168@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513084.414559:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693325613318400, offset 192 00000800:00000200:0.0:1616513084.414563:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513084.414565:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.168@tcp1 00000800:00000200:0.0:1616513084.414568:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.414571:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513084.414572:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.168@tcp1 routed destination 00000800:00000200:0.0:1616513084.414573:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.414577:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.414583:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513084.414586:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.168@tcp1(10.242.104.168@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.414592:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.414597:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.414600:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.414602:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.414606:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.414611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.414612:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513084.414638:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.414645:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.414648:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.414654:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.414658:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04cc0 00000800:00000200:14.0:1616513084.414662:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.414665:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513084.414668:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.414675:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.414678:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.414684:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.414693:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf34b8 00000800:00000200:20.0:1616513084.414697:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.414700:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.414985:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.414997:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.415005:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513084.415007:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.415013:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.415014:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.127@tcp1 : PUT - for me 00000800:00000200:14.0:1616513084.415016:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.415020:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.127@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.415024:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.127@tcp1 of length 224 into portal 26 MB=0x601951d213240 00000400:00000200:18.0:1616513084.415027:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.127@tcp1 of length 224 into portal 12 MB=0x601951d213500 00000400:00000200:2.0:1616513084.415030:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.127@tcp1 of length 224/224 into md 0xf97cab09 [1] + 0 00000400:00000200:2.0:1616513084.415036:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde000 00000400:00000200:2.0:1616513084.415039:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.415041:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.127@tcp1 of length 224/224 into md 0xf964d519 [1] + 36616 00000400:00000200:18.0:1616513084.415045:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.415045:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.415049:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde000 00000100:00000200:18.0:1616513084.415052:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.415053:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.415055:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.415057:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.415057:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.415060:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.415061:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.415062:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589810733632 00010000:00000200:24.0:1616513084.415068:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1690589810733632/t0(0) o400->adc6987f-9c55-3f8c-ca56-8aeb25b64c2c@10.31.140.127@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.415078:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690589810733632, offset 192 00000400:00000200:24.0:1616513084.415083:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.127@tcp1 00000400:00000200:24.0:1616513084.415088:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.127@tcp1 routed destination 00000400:00000200:24.0:1616513084.415101:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.127@tcp1(10.31.140.127@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.415106:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.415111:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.415115:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.415116:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.415119:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513084.415120:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589810734336 00000100:00000200:24.0:1616513084.415124:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.415125:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:22.0:1616513084.415131:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfb600 x1690589810734336/t0(0) o400->b9b1b831-adae-71dd-47d5-48767de0da07@10.31.140.127@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:26.0:1616513084.415136:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.415144:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:22.0:1616513084.415145:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690589810734336, offset 192 00000800:00000200:14.0:1616513084.415146:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.415149:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513084.415150:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.127@tcp1 00000400:00000200:26.0:1616513084.415156:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513084.415158:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.127@tcp1 routed destination 00000400:00000200:26.0:1616513084.415161:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3f58 00000800:00000200:26.0:1616513084.415165:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:22.0:1616513084.415172:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.127@tcp1(10.31.140.127@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.415178:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.415183:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.415188:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.415190:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.415194:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.415199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.415211:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.415218:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.415220:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.415221:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.415227:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.415231:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a330 00000800:00000200:28.0:1616513084.415235:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.437625:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.437681:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.437690:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.437699:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.166@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.437708:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.166@tcp1 of length 224 into portal 26 MB=0x60182eaa1bb40 00000400:00000200:2.0:1616513084.437715:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.166@tcp1 of length 224/224 into md 0xf97cab39 [1] + 0 00000400:00000200:2.0:1616513084.437719:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde220 00000400:00000200:2.0:1616513084.437722:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.437725:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.437730:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde220 00000800:00000200:2.0:1616513084.437733:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.437735:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.437735:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.437740:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.437793:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511654107968 00010000:00000200:24.0:1616513084.437803:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1690511654107968/t0(0) o400->9255b2ec-399a-88b4-5430-5dfd3a40873b@10.31.140.166@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.437816:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511654107968, offset 192 00000400:00000200:24.0:1616513084.437822:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.166@tcp1 00000400:00000200:24.0:1616513084.437829:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.166@tcp1 routed destination 00000400:00000200:24.0:1616513084.437842:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.166@tcp1(10.31.140.166@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.437849:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513084.437852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.437854:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.437857:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.437859:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.437864:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.437869:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.437870:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513084.437908:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.437918:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513084.437924:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513084.437927:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.166@tcp1 : PUT - for me 00000800:00000200:14.0:1616513084.437931:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.437934:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.437935:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.166@tcp1 of length 224 into portal 12 MB=0x60182eaa1bbc0 00000400:00000200:14.0:1616513084.437940:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.437941:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.166@tcp1 of length 224/224 into md 0x865bbec1 [1] + 69776 00000400:00000200:20.0:1616513084.437947:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.437948:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3210 00000100:00000200:20.0:1616513084.437951:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.437952:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.437954:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513084.437958:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.437961:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.437962:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.437966:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.438026:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511654108096 00010000:00000200:0.0:1616513084.438038:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8900 x1690511654108096/t0(0) o400->e949ce10-d923-57e9-74e4-7628f3c072c9@10.31.140.166@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.438053:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511654108096, offset 192 00000400:00000200:0.0:1616513084.438059:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.166@tcp1 00000400:00000200:0.0:1616513084.438066:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.166@tcp1 routed destination 00000400:00000200:0.0:1616513084.438081:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.166@tcp1(10.31.140.166@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.438087:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.438093:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.438096:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.438098:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.438103:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.438108:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.438118:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.438123:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.438125:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.438130:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.438134:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:8.0:1616513084.438138:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.438140:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513084.442529:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926576, portal 4 00000100:00000200:14.0:1616513084.442534:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926576, offset 0 00000400:00000200:14.0:1616513084.442540:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:14.0:1616513084.442546:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:14.0:1616513084.442558:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513084.442564:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513084.442570:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513084.442572:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513084.442574:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513084.442579:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513084.442626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513084.442684:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513084.442691:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513084.442694:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513084.442696:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513084.442701:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513084.442707:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bdf980 x1693067725926576/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513084.442719:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756f68 00000800:00000200:18.0:1616513084.442723:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513084.442799:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513084.442811:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513084.442817:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513084.442824:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513084.442830:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6270b0 00000400:00000200:14.0:1616513084.442837:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cb1e9 [1] + 192 00000400:00000200:14.0:1616513084.442842:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.442846:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bdf980 x1693067725926576/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513084.442860:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513084.442862:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:26.0:1616513084.442863:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (23)-- 00000800:00000200:14.0:1616513084.442867:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:18.0:1616513084.442869:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bdf980 x1693067725926576/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:18.0:1616513084.442880:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bdf980 x1693067725926576/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513084.442889:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757540 00000800:00000200:7.2:1616513084.459314:0:49923:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.459370:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.459380:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513084.459382:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513084.459389:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.155@tcp1 : PUT - for me 00000800:00000200:18.0:1616513084.459391:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.459398:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.155@tcp1 of length 224 into portal 26 MB=0x5ffe50ec60700 00000400:00000200:18.0:1616513084.459400:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.155@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.459405:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.155@tcp1 of length 224/224 into md 0xf97cab59 [1] + 0 00000400:00000200:18.0:1616513084.459408:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.155@tcp1 of length 224 into portal 12 MB=0x5ffe50ec607c0 00000400:00000200:2.0:1616513084.459410:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdf8f8 00000400:00000200:2.0:1616513084.459412:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.459414:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.155@tcp1 of length 224/224 into md 0xf964d519 [1] + 36840 00000400:00000200:18.0:1616513084.459419:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.459419:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.459423:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdf8f8 00000100:00000200:18.0:1616513084.459426:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.459426:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513084.459429:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513084.459431:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.459435:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.459445:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.459447:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.459449:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.459487:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688734144005888 00010000:00000200:24.0:1616513084.459498:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688734144005888/t0(0) o400->a01587ef-df1b-9f30-c834-1a350582ac77@10.31.141.155@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.459513:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688734144005888, offset 192 00000400:00000200:24.0:1616513084.459519:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.155@tcp1 00000100:00000200:22.0:1616513084.459523:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688734144006080 00000400:00000200:24.0:1616513084.459526:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.155@tcp1 routed destination 00010000:00000200:22.0:1616513084.459535:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfd100 x1688734144006080/t0(0) o400->fef8709e-d628-4657-7fa1-5196d79f21d2@10.31.141.155@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513084.459542:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.155@tcp1(10.31.141.155@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.459548:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:22.0:1616513084.459549:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688734144006080, offset 192 00000800:00000200:24.0:1616513084.459554:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513084.459556:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.155@tcp1 00000800:00000200:24.0:1616513084.459557:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.459559:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:22.0:1616513084.459563:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.155@tcp1 routed destination 00000800:00000200:24.0:1616513084.459564:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.459569:0:49923:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.459570:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513084.459578:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.155@tcp1(10.31.141.155@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.459584:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.459590:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.459593:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.459595:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.459600:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513084.459620:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.459627:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.459630:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513084.459630:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.459636:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.459641:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:14.0:1616513084.459644:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.459645:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.459702:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.459709:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.459712:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513084.459718:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513084.459722:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b188 00000800:00000200:12.0:1616513084.459726:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.459729:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.463416:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513084.463419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.463431:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.463443:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.463450:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.180@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.463459:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.180@tcp1 of length 224 into portal 12 MB=0x5ffd70f70b8c0 00000400:00000200:8.0:1616513084.463464:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.180@tcp1 of length 224/224 into md 0x865bbec1 [1] + 70000 00000400:00000200:8.0:1616513084.463469:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.463471:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.463472:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513084.463477:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.463479:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.463481:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513084.463482:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:8.0:1616513084.463483:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513084.463491:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.180@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.463499:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.180@tcp1 of length 224 into portal 26 MB=0x5ffd70f70b500 00000400:00000200:2.0:1616513084.463506:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.180@tcp1 of length 224/224 into md 0xf97cab69 [1] + 0 00000400:00000200:2.0:1616513084.463510:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdec38 00000400:00000200:2.0:1616513084.463513:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.463516:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.463520:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdec38 00000800:00000200:2.0:1616513084.463523:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.463525:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.463527:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.463530:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513084.463541:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674025650368 00010000:00000200:0.0:1616513084.463553:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8f980 x1688674025650368/t0(0) o400->467be00c-d6cf-be51-2bb2-4a5f8fc9a6d4@10.31.129.180@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.463568:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688674025650368, offset 192 00000400:00000200:0.0:1616513084.463574:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.180@tcp1 00000400:00000200:0.0:1616513084.463581:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.180@tcp1 routed destination 00000400:00000200:0.0:1616513084.463596:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.180@tcp1(10.31.129.180@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513084.463597:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674025649408 00000800:00000200:0.0:1616513084.463603:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:24.0:1616513084.463608:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688674025649408/t0(0) o400->066e14e8-e865-ba32-368b-6774437b805e@10.31.129.180@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513084.463609:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.463612:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.463614:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.463619:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.463622:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688674025649408, offset 192 00000800:00000200:7.2:1616513084.463624:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513084.463629:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.180@tcp1 00000800:00000200:18.0:1616513084.463633:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513084.463636:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.180@tcp1 routed destination 00000800:00000200:18.0:1616513084.463637:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.463639:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.463644:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.463648:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000400:00000200:24.0:1616513084.463651:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.180@tcp1(10.31.129.180@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513084.463651:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.463653:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513084.463658:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.463663:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.463666:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.463668:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.463672:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.463678:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.463679:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513084.463730:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.463737:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.463740:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513084.463747:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513084.463752:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2f68 00000800:00000200:12.0:1616513084.463756:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.463759:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.470037:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.470049:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.470055:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.470063:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.183@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.470070:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.183@tcp1 of length 224 into portal 26 MB=0x604bbc3b0f140 00000400:00000200:8.0:1616513084.470076:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.183@tcp1 of length 224/224 into md 0xf97cab79 [1] + 0 00000400:00000200:8.0:1616513084.470080:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fde2a8 00000400:00000200:8.0:1616513084.470083:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.470085:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.470089:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fde2a8 00000800:00000200:8.0:1616513084.470091:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.470093:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.470095:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.470097:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.470110:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513084.470153:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694054348812608 00010000:00000200:24.0:1616513084.470164:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1694054348812608/t0(0) o400->5e6c72eb-a2ad-a899-01c9-9b3330f39eff@10.31.140.183@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513084.470167:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.470176:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:24.0:1616513084.470178:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694054348812608, offset 192 00000400:00000200:24.0:1616513084.470184:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.183@tcp1 00000400:00000200:12.0:1616513084.470185:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.183@tcp1 : PUT - for me 00000400:00000200:24.0:1616513084.470192:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.183@tcp1 routed destination 00000400:00000200:12.0:1616513084.470202:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.183@tcp1 of length 224 into portal 12 MB=0x604bbc3b0f240 00000400:00000200:24.0:1616513084.470207:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.183@tcp1(10.31.140.183@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513084.470208:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.183@tcp1 of length 224/224 into md 0x865bbec1 [1] + 70224 00000400:00000200:12.0:1616513084.470213:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513084.470214:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:12.0:1616513084.470215:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513084.470219:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.470222:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513084.470222:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.470223:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513084.470224:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513084.470225:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513084.470229:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513084.470229:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.470235:0:49924:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.470236:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513084.470288:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694054348812864 00000800:00000200:2.0:1616513084.470295:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:0.0:1616513084.470300:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d9b00 x1694054348812864/t0(0) o400->b513e8ef-5c12-ea9d-1daa-e824e0ace0d5@10.31.140.183@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513084.470301:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.470317:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.470323:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:0.0:1616513084.470326:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694054348812864, offset 192 00000400:00000200:2.0:1616513084.470329:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ee0 00000800:00000200:2.0:1616513084.470332:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513084.470334:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.183@tcp1 00000800:00000200:2.0:1616513084.470335:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513084.470341:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.183@tcp1 routed destination 00000400:00000200:0.0:1616513084.470354:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.183@tcp1(10.31.140.183@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.470361:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.470366:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.470368:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.470370:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.470375:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.470380:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.470388:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.470391:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.470393:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513084.470397:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.470398:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.470400:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04aa0 00000800:00000200:8.0:1616513084.470403:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.482642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513084.482699:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.482709:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.482718:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.152@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.482727:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.152@tcp1 of length 224 into portal 26 MB=0x5ffdb69fb8e00 00000400:00000200:20.0:1616513084.482734:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.152@tcp1 of length 224/224 into md 0xf97cab99 [1] + 0 00000400:00000200:20.0:1616513084.482739:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdfcb0 00000400:00000200:20.0:1616513084.482741:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.482744:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513084.482748:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdfcb0 00000800:00000200:20.0:1616513084.482752:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.482752:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.482755:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.482758:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.482761:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.482762:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.482769:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.152@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.482778:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.152@tcp1 of length 224 into portal 12 MB=0x5ffdb69fb9080 00000400:00000200:12.0:1616513084.482784:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.152@tcp1 of length 224/224 into md 0x865bbec1 [1] + 70448 00000400:00000200:12.0:1616513084.482788:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.482791:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.482797:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.482800:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.482801:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.482810:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692724567552 00010000:00000200:24.0:1616513084.482820:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688692724567552/t0(0) o400->796ba767-f7ad-2b38-91fa-b3640934d860@10.31.140.152@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513084.482821:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692724568192 00010000:00000200:0.0:1616513084.482832:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0df980 x1688692724568192/t0(0) o400->adff02a4-9e9a-6e3b-9d72-39ad475425dd@10.31.140.152@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513084.482833:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692724567552, offset 192 00000400:00000200:24.0:1616513084.482841:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.152@tcp1 00000100:00000200:0.0:1616513084.482846:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692724568192, offset 192 00000400:00000200:24.0:1616513084.482848:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.152@tcp1 routed destination 00000400:00000200:0.0:1616513084.482852:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.152@tcp1 00000400:00000200:0.0:1616513084.482859:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.152@tcp1 routed destination 00000400:00000200:24.0:1616513084.482862:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.152@tcp1(10.31.140.152@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.482869:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513084.482881:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.152@tcp1(10.31.140.152@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.482887:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.482888:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.482890:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.482892:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.482893:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.482895:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.482896:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513084.482897:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.482901:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513084.482903:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.482904:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513084.482907:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.482917:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.482922:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.482924:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513084.482926:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.482929:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.482933:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05540 00000800:00000200:8.0:1616513084.482936:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513084.482957:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.482964:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.482967:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.482973:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.482978:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000800:00000200:18.0:1616513084.482981:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.482984:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513084.490539:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926592, portal 4 00000100:00000200:10.0:1616513084.490543:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926592, offset 0 00000400:00000200:10.0:1616513084.490549:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:10.0:1616513084.490554:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:10.0:1616513084.490565:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513084.490570:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:10.0:1616513084.490575:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513084.490578:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:10.0:1616513084.490580:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:10.0:1616513084.490585:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:11.0:1616513084.490639:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926608, portal 4 00000100:00000200:11.0:1616513084.490645:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926608, offset 0 00000400:00000200:11.0:1616513084.490653:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:11.0:1616513084.490660:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:27.2:1616513084.490662:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:11.0:1616513084.490673:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513084.490677:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000100:00000200:3.0:1616513084.490678:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926624, portal 4 00000800:00000200:11.0:1616513084.490680:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000100:00000200:3.0:1616513084.490683:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926624, offset 0 00000800:00000200:31.0:1616513084.490684:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:31.0:1616513084.490687:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.0:1616513084.490687:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513084.490689:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:11.0:1616513084.490690:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000400:00000200:3.0:1616513084.490690:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:11.0:1616513084.490693:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000400:00000200:31.0:1616513084.490694:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:3.0:1616513084.490697:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:11.0:1616513084.490699:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:31.0:1616513084.490701:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c8a2f980 x1693067725926592/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513084.490710:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:31.0:1616513084.490713:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82990 00000800:00000200:31.0:1616513084.490717:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:3.0:1616513084.490717:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:3.0:1616513084.490723:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513084.490726:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:3.0:1616513084.490728:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:3.0:1616513084.490734:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513084.490738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:25.2:1616513084.490744:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:27.0:1616513084.490748:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513084.490753:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513084.490756:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513084.490758:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:2.0:1616513084.490760:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000400:00000200:27.0:1616513084.490763:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:2.0:1616513084.490767:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000100:00000200:27.0:1616513084.490769:0:6476:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bf8900 x1693067725926624/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513084.490771:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.490777:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:27.0:1616513084.490781:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd27f8 00000800:00000200:27.0:1616513084.490785:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:2.0:1616513084.490787:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc566300 x1693067725926608/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513084.490799:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3b430 00000800:00000200:2.0:1616513084.490803:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:27.2:1616513084.490805:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:2.0:1616513084.490806:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:27.0:1616513084.490814:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.490819:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513084.490821:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513084.490827:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:27.2:1616513084.490834:0:6475:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:27.0:1616513084.490841:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6270c0 00000800:00000200:31.0:1616513084.490842:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513084.490848:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513084.490849:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166d6d [1] + 192 00000800:00000200:3.0:1616513084.490851:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513084.490855:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:31.0:1616513084.490856:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000100:00000200:27.0:1616513084.490859:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c8a2f980 x1693067725926592/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513084.490864:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6270e0 00000800:00000200:27.0:1616513084.490874:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:31.0:1616513084.490876:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166d8d [1] + 192 00000800:00000200:27.0:1616513084.490876:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513084.490878:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:31.0:1616513084.490881:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513084.490884:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c8a2f980 x1693067725926592/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513084.490894:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c8a2f980 x1693067725926592/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513084.490902:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83540 00000100:00000200:31.0:1616513084.490906:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bf8900 x1693067725926624/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513084.490921:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513084.490923:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:25.2:1616513084.490923:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:31.0:1616513084.490925:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:3.0:1616513084.490927:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bf8900 x1693067725926624/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513084.490940:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bf8900 x1693067725926624/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513084.490950:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:18.0:1616513084.490977:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513084.490986:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513084.490995:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513084.491003:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6270d0 00000400:00000200:18.0:1616513084.491010:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cb269 [1] + 192 00000400:00000200:18.0:1616513084.491016:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513084.491021:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc566300 x1693067725926608/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513084.491030:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513084.491036:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513084.491039:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513084.491041:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:11.0:1616513084.491047:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc566300 x1693067725926608/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513084.491060:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc566300 x1693067725926608/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513084.491071:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bc28 00000800:00000200:11.2:1616513084.493646:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.493704:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.493718:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513084.493724:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.493726:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.247@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.493736:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.247@tcp1 of length 224 into portal 26 MB=0x603ff4c782c00 00000400:00000200:12.0:1616513084.493743:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.247@tcp1 of length 224/224 into md 0xf97caba9 [1] + 0 00000400:00000200:12.0:1616513084.493747:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd7fdebb0 00000400:00000200:12.0:1616513084.493750:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.493753:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513084.493757:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd7fdebb0 00000800:00000200:12.0:1616513084.493760:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.493763:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.493765:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.493820:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693244894751744 00010000:00000200:24.0:1616513084.493830:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1693244894751744/t0(0) o400->e7756e9c-dba1-c850-c958-48168c7b56e6@10.31.132.247@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.493843:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693244894751744, offset 192 00000400:00000200:24.0:1616513084.493849:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.247@tcp1 00000400:00000200:24.0:1616513084.493857:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.247@tcp1 routed destination 00000400:00000200:24.0:1616513084.493871:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.247@tcp1(10.31.132.247@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513084.493876:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.493877:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.493883:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.493888:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.493890:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513084.493894:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:24.0:1616513084.493900:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513084.493929:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.493938:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.493947:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.247@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.493956:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.247@tcp1 of length 224 into portal 12 MB=0x603ff4c782dc0 00000400:00000200:2.0:1616513084.493965:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.247@tcp1 of length 224/224 into md 0xf964d519 [1] + 37064 00000400:00000200:2.0:1616513084.493970:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.493972:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.493979:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513084.493982:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513084.493982:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513084.493989:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513084.493991:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513084.493992:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.493993:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.494000:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.494006:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2f68 00000800:00000200:26.0:1616513084.494009:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.494056:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693244894752192 00010000:00000200:22.0:1616513084.494068:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d1680 x1693244894752192/t0(0) o400->465fd619-410e-d762-b94c-7f7bd9c3a2d0@10.31.132.247@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.494083:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693244894752192, offset 192 00000400:00000200:22.0:1616513084.494089:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.247@tcp1 00000400:00000200:22.0:1616513084.494096:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.247@tcp1 routed destination 00000400:00000200:22.0:1616513084.494112:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.247@tcp1(10.31.132.247@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.494118:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.494124:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.494127:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.494129:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.494133:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.494139:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.494192:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.494199:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.494202:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.494203:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.494209:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.494212:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a990 00000800:00000200:20.0:1616513084.494218:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.540522:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.540578:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.540587:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513084.540596:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.145@tcp1 : PUT - for me 00000400:00000200:18.0:1616513084.540606:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.145@tcp1 of length 224 into portal 26 MB=0x5ffe88b3c4c80 00000400:00000200:18.0:1616513084.540615:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.145@tcp1 of length 224/224 into md 0xf97cabb9 [1] + 0 00000400:00000200:18.0:1616513084.540620:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b33a8 00000400:00000200:18.0:1616513084.540623:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513084.540626:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513084.540630:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b33a8 00000800:00000200:18.0:1616513084.540633:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.540633:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.540636:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.540640:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.540643:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.540644:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.540653:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.145@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.540661:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.145@tcp1 of length 224 into portal 12 MB=0x5ffe88b3c4d00 00000400:00000200:2.0:1616513084.540668:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.145@tcp1 of length 224/224 into md 0xf964d519 [1] + 37288 00000400:00000200:2.0:1616513084.540673:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.540676:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.540682:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.540686:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.540687:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.540705:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688749117033600 00010000:00000200:24.0:1616513084.540716:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688749117033600/t0(0) o400->50e306d5-471f-34d1-7e23-6527cf7930c8@10.31.141.145@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.540730:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688749117033600, offset 192 00000400:00000200:24.0:1616513084.540736:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.145@tcp1 00000400:00000200:24.0:1616513084.540744:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.145@tcp1 routed destination 00000100:00000200:22.0:1616513084.540746:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688749117033728 00010000:00000200:22.0:1616513084.540758:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d5a00 x1688749117033728/t0(0) o400->915c9eb5-afe5-cf8a-e32e-ef64e18848f4@10.31.141.145@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513084.540759:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.145@tcp1(10.31.141.145@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.540770:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.540776:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.540779:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.540781:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513084.540784:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688749117033728, offset 192 00000800:00000200:24.0:1616513084.540786:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513084.540790:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.145@tcp1 00000800:00000200:7.2:1616513084.540791:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.540793:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513084.540798:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.145@tcp1 routed destination 00000400:00000200:22.0:1616513084.540812:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.145@tcp1(10.31.141.145@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.540819:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.540825:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.540828:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.540830:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.540835:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513084.540844:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.540851:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.540854:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513084.540855:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513084.540861:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.540866:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3210 00000800:00000200:14.0:1616513084.540870:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.540888:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.540944:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.540951:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.540954:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.540960:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.540964:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a3b8 00000800:00000200:28.0:1616513084.540968:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.540971:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.546725:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.546740:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.546748:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.546757:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.129@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.546766:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.129@tcp1 of length 224 into portal 26 MB=0x5ffdbe256fd00 00000400:00000200:8.0:1616513084.546773:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.129@tcp1 of length 224/224 into md 0xf97cabc9 [1] + 0 00000400:00000200:8.0:1616513084.546777:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2d48 00000400:00000200:8.0:1616513084.546780:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.546782:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.546787:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2d48 00000800:00000200:8.0:1616513084.546790:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.546792:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.546794:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513084.546802:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.546805:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513084.546805:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.546814:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.129@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.546823:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.129@tcp1 of length 224 into portal 12 MB=0x5ffdbe256fd80 00000400:00000200:20.0:1616513084.546829:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.129@tcp1 of length 224/224 into md 0x865bbec1 [1] + 70672 00000400:00000200:20.0:1616513084.546835:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.546837:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513084.546844:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.546847:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.546849:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.546850:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694743825664 00010000:00000200:24.0:1616513084.546861:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688694743825664/t0(0) o400->7b7305df-cfd0-aa2f-5788-7dd9f0ddb261@10.31.141.129@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.546875:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694743825664, offset 192 00000400:00000200:24.0:1616513084.546882:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.129@tcp1 00000400:00000200:24.0:1616513084.546889:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.129@tcp1 routed destination 00000400:00000200:24.0:1616513084.546904:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.129@tcp1(10.31.141.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513084.546908:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694743825792 00000800:00000200:24.0:1616513084.546910:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.546916:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.546919:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.546921:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513084.546921:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce9b00 x1688694743825792/t0(0) o400->384055e8-6834-e70c-3f9f-fcb977540ab6@10.31.141.129@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513084.546926:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.546932:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.546933:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513084.546936:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694743825792, offset 192 00000400:00000200:0.0:1616513084.546942:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.129@tcp1 00000400:00000200:0.0:1616513084.546949:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.129@tcp1 routed destination 00000400:00000200:0.0:1616513084.546966:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.129@tcp1(10.31.141.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.546973:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.546978:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.546981:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.546983:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.546988:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513084.546991:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513084.546993:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513084.546998:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.547001:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513084.547006:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.547007:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.547013:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf34b8 00000800:00000200:2.0:1616513084.547016:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513084.547047:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.547054:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.547057:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.547057:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.547064:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513084.547067:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05cb0 00000800:00000200:12.0:1616513084.547071:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.557581:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.557636:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.557645:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513084.557654:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.155@tcp1 : PUT - for me 00000400:00000200:18.0:1616513084.557664:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.155@tcp1 of length 224 into portal 26 MB=0x5ffe358f9edc0 00000400:00000200:18.0:1616513084.557671:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.155@tcp1 of length 224/224 into md 0xf97cac09 [1] + 0 00000400:00000200:18.0:1616513084.557676:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2440 00000400:00000200:18.0:1616513084.557678:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513084.557681:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513084.557685:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2440 00000800:00000200:18.0:1616513084.557688:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.557690:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.557691:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.557697:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.557700:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.557701:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.557709:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.155@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.557719:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.155@tcp1 of length 224 into portal 12 MB=0x5ffe358f9ee40 00000400:00000200:14.0:1616513084.557725:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.155@tcp1 of length 224/224 into md 0xf964d519 [1] + 37512 00000400:00000200:14.0:1616513084.557730:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.557733:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.557740:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.557743:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.557745:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.557760:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726798986688 00010000:00000200:24.0:1616513084.557770:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688726798986688/t0(0) o400->9836b5e7-a8c9-8e97-e65e-3e874f85d582@10.31.140.155@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.557785:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726798986688, offset 192 00000400:00000200:24.0:1616513084.557791:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.155@tcp1 00000400:00000200:24.0:1616513084.557799:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.155@tcp1 routed destination 00000100:00000200:22.0:1616513084.557804:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726798986816 00000400:00000200:24.0:1616513084.557814:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.155@tcp1(10.31.140.155@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:22.0:1616513084.557816:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464620480 x1688726798986816/t0(0) o400->182ad635-c79e-d332-b3bd-72d30b9607d0@10.31.140.155@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513084.557820:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.557826:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.557829:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:22.0:1616513084.557830:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726798986816, offset 192 00000400:00000200:22.0:1616513084.557836:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.155@tcp1 00000800:00000200:24.0:1616513084.557838:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.557843:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513084.557844:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.155@tcp1 routed destination 00000800:00000200:7.2:1616513084.557848:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.557850:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513084.557859:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.155@tcp1(10.31.140.155@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.557865:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.557871:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.557874:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.557876:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.557880:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513084.557904:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.557911:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.557914:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.557914:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.557921:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.557926:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:11.2:1616513084.557927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513084.557930:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513084.557986:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.557993:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.557996:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.558002:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.558006:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4add0 00000800:00000200:20.0:1616513084.558010:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.558013:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.564735:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.564790:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.564798:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.564806:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.36@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.564815:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.36@tcp1 of length 224 into portal 26 MB=0x6018c6c560ec0 00000400:00000200:8.0:1616513084.564821:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.36@tcp1 of length 224/224 into md 0xf97cac29 [1] + 0 00000400:00000200:8.0:1616513084.564825:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3ed0 00000400:00000200:8.0:1616513084.564828:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.564830:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.564834:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3ed0 00000800:00000200:8.0:1616513084.564837:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.564840:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.564842:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.564846:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.564856:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.564865:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.36@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.564874:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.36@tcp1 of length 224 into portal 12 MB=0x6018c6c560fc0 00000400:00000200:12.0:1616513084.564882:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.36@tcp1 of length 224/224 into md 0x865bbec1 [1] + 70896 00000400:00000200:12.0:1616513084.564887:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.564890:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:24.0:1616513084.564896:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690552484892352 00000800:00000200:12.0:1616513084.564896:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.564899:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.564901:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.564904:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:24.0:1616513084.564906:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1690552484892352/t0(0) o400->22726cbc-5513-da65-6d9c-7ec03619cffd@10.31.142.36@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.564920:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690552484892352, offset 192 00000400:00000200:24.0:1616513084.564926:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.36@tcp1 00000400:00000200:24.0:1616513084.564933:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.36@tcp1 routed destination 00000400:00000200:24.0:1616513084.564947:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.36@tcp1(10.31.142.36@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.564953:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.564958:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.564960:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.564962:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.564967:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.564973:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513084.564981:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690552484892608 00000800:00000200:7.2:1616513084.564983:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513084.564994:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0de780 x1690552484892608/t0(0) o400->0ff634a3-f3d2-f57f-9612-3e8341b15e7f@10.31.142.36@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.565008:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690552484892608, offset 192 00000400:00000200:0.0:1616513084.565015:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.36@tcp1 00000400:00000200:0.0:1616513084.565023:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.36@tcp1 routed destination 00000400:00000200:0.0:1616513084.565038:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.36@tcp1(10.31.142.36@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.565044:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513084.565047:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513084.565050:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.565052:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513084.565054:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513084.565055:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513084.565057:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513084.565059:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513084.565062:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513084.565065:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513084.565065:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513084.565071:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000800:00000200:14.0:1616513084.565074:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513084.565118:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.565125:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.565128:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.565129:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.565135:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.565139:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb058f8 00000800:00000200:28.0:1616513084.565143:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.573503:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.573560:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513084.573563:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513084.573569:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.573578:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.159@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.573586:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.159@tcp1 of length 224 into portal 12 MB=0x5ffe59a460b40 00000400:00000200:12.0:1616513084.573593:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.159@tcp1 of length 224/224 into md 0x865bbec1 [1] + 71120 00000400:00000200:12.0:1616513084.573598:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.573600:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.573607:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.573610:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.573612:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.573614:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513084.573633:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.573643:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513084.573652:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.159@tcp1 : PUT - for me 00000400:00000200:18.0:1616513084.573660:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.159@tcp1 of length 224 into portal 26 MB=0x5ffe59a4608c0 00000400:00000200:18.0:1616513084.573667:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.159@tcp1 of length 224/224 into md 0xf97cac39 [1] + 0 00000400:00000200:18.0:1616513084.573671:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2c38 00000100:00000200:0.0:1616513084.573671:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688736484428608 00000400:00000200:18.0:1616513084.573674:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513084.573677:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00010000:00000200:0.0:1616513084.573682:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d9680 x1688736484428608/t0(0) o400->41f813b7-72d7-ddaf-fe15-980b3e36230d@10.31.141.159@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:18.0:1616513084.573683:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2c38 00000800:00000200:18.0:1616513084.573686:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.573687:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.573689:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.573691:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513084.573696:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688736484428608, offset 192 00000400:00000200:0.0:1616513084.573701:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.159@tcp1 00000400:00000200:0.0:1616513084.573708:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.159@tcp1 routed destination 00000400:00000200:0.0:1616513084.573722:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.159@tcp1(10.31.141.159@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.573728:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513084.573733:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.573736:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.573746:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.573750:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688736484427968 00000800:00000200:0.0:1616513084.573750:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.573756:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:24.0:1616513084.573760:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688736484427968/t0(0) o400->704fd715-5992-6218-a574-7568df98135a@10.31.141.159@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.573773:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688736484427968, offset 192 00000400:00000200:24.0:1616513084.573779:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.159@tcp1 00000400:00000200:24.0:1616513084.573786:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.159@tcp1 routed destination 00000400:00000200:24.0:1616513084.573800:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.159@tcp1(10.31.141.159@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.573806:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.573811:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.573814:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.573816:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513084.573817:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513084.573820:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513084.573824:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:24.0:1616513084.573826:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.573826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513084.573827:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.573834:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.573838:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb044c8 00000800:00000200:26.0:1616513084.573842:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.573845:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513084.573893:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.573899:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.573902:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.573908:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.573913:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:8.0:1616513084.573916:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.573919:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.594422:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.594478:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.594487:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.594496:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.142@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.594505:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.142@tcp1 of length 224 into portal 26 MB=0x5ffdea104a700 00000400:00000200:12.0:1616513084.594511:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.142@tcp1 of length 224/224 into md 0xf97cac49 [1] + 0 00000400:00000200:12.0:1616513084.594516:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3b18 00000400:00000200:12.0:1616513084.594519:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.594522:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513084.594526:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3b18 00000800:00000200:12.0:1616513084.594529:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.594531:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.594532:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513084.594539:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.594542:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513084.594542:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.594551:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.142@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.594560:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.142@tcp1 of length 224 into portal 12 MB=0x5ffdea104aa80 00000400:00000200:20.0:1616513084.594566:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.142@tcp1 of length 224/224 into md 0x865bbec1 [1] + 71344 00000400:00000200:20.0:1616513084.594571:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.594574:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513084.594581:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.594584:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706532812544 00000800:00000200:20.0:1616513084.594584:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.594586:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:24.0:1616513084.594594:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688706532812544/t0(0) o400->947f30cc-97cd-7c4c-86f6-4e43cb2d9352@10.31.142.142@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.594607:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706532812544, offset 192 00000400:00000200:24.0:1616513084.594613:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.142@tcp1 00000400:00000200:24.0:1616513084.594619:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.142@tcp1 routed destination 00000400:00000200:24.0:1616513084.594632:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.142@tcp1(10.31.142.142@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.594638:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.594642:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513084.594643:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706532813440 00000800:00000200:24.0:1616513084.594645:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.594647:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.594651:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513084.594654:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8480 x1688706532813440/t0(0) o400->d14b771c-09d8-eb7e-80ae-fb8563eff29b@10.31.142.142@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513084.594657:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513084.594667:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706532813440, offset 192 00000800:00000200:7.2:1616513084.594668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513084.594673:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.142@tcp1 00000400:00000200:0.0:1616513084.594680:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.142@tcp1 routed destination 00000400:00000200:0.0:1616513084.594693:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.142@tcp1(10.31.142.142@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.594699:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.594705:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.594707:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.594709:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.594713:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.594719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513084.594723:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.594730:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.594733:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.594741:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.594747:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:2.0:1616513084.594750:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.594753:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513084.594772:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.594779:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.594782:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.594782:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.594789:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.594793:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04c38 00000800:00000200:28.0:1616513084.594796:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.596034:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.596089:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.596098:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.596108:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.64@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.596117:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.64@tcp1 of length 224 into portal 26 MB=0x5ffdc47ac4180 00000400:00000200:12.0:1616513084.596124:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.64@tcp1 of length 224/224 into md 0xf97cac59 [1] + 0 00000400:00000200:12.0:1616513084.596128:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3540 00000400:00000200:12.0:1616513084.596131:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.596134:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513084.596138:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3540 00000800:00000200:12.0:1616513084.596141:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.596142:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.596143:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.596149:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.596152:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.596153:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.596162:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.64@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.596170:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.64@tcp1 of length 224 into portal 12 MB=0x5ffdc47ac4240 00000400:00000200:20.0:1616513084.596178:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.64@tcp1 of length 224/224 into md 0x865bbec1 [1] + 71568 00000400:00000200:20.0:1616513084.596183:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.596186:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513084.596192:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.596196:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.596197:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.596200:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696443912576 00010000:00000200:24.0:1616513084.596211:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688696443912576/t0(0) o400->510acc90-cb83-2f25-e86b-ba352c3d68d1@10.31.141.64@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.596225:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696443912576, offset 192 00000400:00000200:24.0:1616513084.596231:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.64@tcp1 00000400:00000200:24.0:1616513084.596239:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.64@tcp1 routed destination 00000100:00000200:0.0:1616513084.596252:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696443912768 00000400:00000200:24.0:1616513084.596253:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.64@tcp1(10.31.141.64@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.596260:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:0.0:1616513084.596263:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceb180 x1688696443912768/t0(0) o400->80580cc7-733f-b8f5-3ed7-078823fece8a@10.31.141.64@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513084.596265:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.596268:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.596270:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.596275:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513084.596275:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696443912768, offset 192 00000800:00000200:7.2:1616513084.596281:0:49938:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513084.596281:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.64@tcp1 00000100:00000200:24.0:1616513084.596282:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513084.596288:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.64@tcp1 routed destination 00000400:00000200:0.0:1616513084.596301:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.64@tcp1(10.31.141.64@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.596307:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.596311:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.596314:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.596316:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.596320:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.596325:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513084.596348:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.596355:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.596357:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.596364:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.596369:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:26.0:1616513084.596373:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.596376:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513084.596379:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.596386:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.596388:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.596388:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.596395:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.596399:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000800:00000200:28.0:1616513084.596403:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.596718:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.596728:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.596733:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513084.596737:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.596741:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.167@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.596748:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.167@tcp1 of length 224 into portal 26 MB=0x5ffe584604840 00000400:00000200:12.0:1616513084.596753:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.167@tcp1 of length 224/224 into md 0xf97cac69 [1] + 0 00000400:00000200:12.0:1616513084.596757:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2000 00000800:00000200:11.2:1616513084.596757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513084.596758:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.596761:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513084.596765:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2000 00000800:00000200:12.0:1616513084.596767:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.596768:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.596769:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.596772:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.596775:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.596776:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:24.0:1616513084.596777:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688736117049408 00010000:00000200:24.0:1616513084.596783:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688736117049408/t0(0) o400->1f23294d-5d21-c2bd-3dd9-477a8c0d9721@10.31.141.167@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513084.596784:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.167@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.596793:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.167@tcp1 of length 224 into portal 12 MB=0x5ffe5846048c0 00000100:00000200:24.0:1616513084.596794:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688736117049408, offset 192 00000400:00000200:8.0:1616513084.596799:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.167@tcp1 of length 224/224 into md 0x865bbec1 [1] + 71792 00000400:00000200:8.0:1616513084.596804:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513084.596806:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.167@tcp1 00000100:00000200:8.0:1616513084.596808:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513084.596812:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.167@tcp1 routed destination 00000800:00000200:8.0:1616513084.596818:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.596821:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.596823:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.596825:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688736117049536 00000400:00000200:24.0:1616513084.596827:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.167@tcp1(10.31.141.167@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513084.596831:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8f080 x1688736117049536/t0(0) o400->68f66aa7-a858-42e8-53ec-530329159415@10.31.141.167@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513084.596833:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.596837:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.596840:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513084.596841:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688736117049536, offset 192 00000800:00000200:24.0:1616513084.596842:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.596845:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513084.596847:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.167@tcp1 00000100:00000200:24.0:1616513084.596849:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.596850:0:49938:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513084.596852:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.167@tcp1 routed destination 00000400:00000200:0.0:1616513084.596864:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.167@tcp1(10.31.141.167@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.596869:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.596873:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.596875:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.596876:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.596879:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.596884:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.596893:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.596897:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.596899:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513084.596901:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.596904:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513084.596907:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513084.596908:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a90 00000800:00000200:20.0:1616513084.596911:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513084.596913:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.596916:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.596923:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.596928:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000800:00000200:2.0:1616513084.596932:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.596935:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.596941:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513084.596943:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.596944:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.596946:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.596947:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.598325:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.598379:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.598388:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.598397:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.146@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.598406:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.146@tcp1 of length 224 into portal 26 MB=0x5ffdc9f8c3e80 00000400:00000200:14.0:1616513084.598412:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.146@tcp1 of length 224/224 into md 0xf97cac79 [1] + 0 00000400:00000200:14.0:1616513084.598417:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2bb0 00000400:00000200:14.0:1616513084.598420:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.598423:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513084.598427:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2bb0 00000800:00000200:14.0:1616513084.598430:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.598432:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.598446:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.598448:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.598490:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697918209664 00010000:00000200:24.0:1616513084.598501:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688697918209664/t0(0) o400->aca9f91f-ce79-422e-3110-22bfa409339f@10.31.141.146@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.598515:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697918209664, offset 192 00000400:00000200:24.0:1616513084.598521:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.146@tcp1 00000400:00000200:24.0:1616513084.598528:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.146@tcp1 routed destination 00000400:00000200:24.0:1616513084.598544:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.146@tcp1(10.31.141.146@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.598550:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.598556:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.598558:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.598561:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.598565:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.598571:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.598572:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.598572:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513084.598623:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.598630:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.598633:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.598640:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513084.598641:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513084.598645:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:2.0:1616513084.598649:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513084.598650:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.598651:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513084.598652:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513084.598660:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.146@tcp1 : PUT - for me 00000400:00000200:28.0:1616513084.598668:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.146@tcp1 of length 224 into portal 12 MB=0x5ffdc9f8c41c0 00000400:00000200:28.0:1616513084.598675:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.146@tcp1 of length 224/224 into md 0x865bbec1 [1] + 72016 00000400:00000200:28.0:1616513084.598680:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513084.598682:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513084.598689:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.598694:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.598696:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.598714:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697918210496 00010000:00000200:0.0:1616513084.598726:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0e8480 x1688697918210496/t0(0) o400->fd6216b0-12a5-9774-c532-03aaffd54ce3@10.31.141.146@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.598741:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697918210496, offset 192 00000400:00000200:0.0:1616513084.598747:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.146@tcp1 00000400:00000200:0.0:1616513084.598755:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.146@tcp1 routed destination 00000400:00000200:0.0:1616513084.598769:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.146@tcp1(10.31.141.146@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.598776:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.598782:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.598796:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.598798:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.598802:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.598808:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.598862:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.598868:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.598871:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513084.598878:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513084.598882:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb045d8 00000800:00000200:12.0:1616513084.598886:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.598889:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513084.602582:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926640, portal 4 00000100:00000200:4.0:1616513084.602587:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926640, offset 0 00000400:00000200:4.0:1616513084.602594:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:4.0:1616513084.602600:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:4.0:1616513084.602612:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513084.602619:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:4.0:1616513084.602625:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513084.602628:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:4.0:1616513084.602630:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513084.602635:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513084.602691:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513084.602711:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513084.602715:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513084.602717:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513084.602721:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:8.0:1616513084.602724:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513084.602730:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4557cee780 x1693067725926640/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513084.602742:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbb18 00000800:00000200:8.0:1616513084.602745:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513084.602828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:12.0:1616513084.602882:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513084.602894:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513084.602896:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:12.0:1616513084.602903:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513084.602911:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6270f0 00000400:00000200:12.0:1616513084.602918:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d7f11 [1] + 192 00000400:00000200:12.0:1616513084.602923:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.602928:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4557cee780 x1693067725926640/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513084.602943:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513084.602946:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513084.602948:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:4.0:1616513084.602950:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4557cee780 x1693067725926640/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513084.602963:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4557cee780 x1693067725926640/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513084.602973:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00000800:00000200:7.2:1616513084.604805:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.604858:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.604868:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.604877:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.193@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.604886:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.193@tcp1 of length 224 into portal 12 MB=0x6001c313b0300 00000800:00000200:11.2:1616513084.604888:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513084.604892:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.193@tcp1 of length 224/224 into md 0xf964d519 [1] + 37736 00000400:00000200:26.0:1616513084.604902:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.604905:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.604911:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.604912:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.604915:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.604917:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513084.604957:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.604967:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513084.604967:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.604976:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.193@tcp1 : PUT - for me 00000100:00000200:22.0:1616513084.604979:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688970945299200 00000400:00000200:28.0:1616513084.604985:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.193@tcp1 of length 224 into portal 26 MB=0x6001c313b0280 00000400:00000200:28.0:1616513084.604991:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.193@tcp1 of length 224/224 into md 0xf97cac89 [1] + 0 00010000:00000200:22.0:1616513084.604991:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cd9488d80 x1688970945299200/t0(0) o400->dd72207b-68b3-934f-999e-3d796c054e33@10.31.138.193@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513084.604996:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3a90 00000400:00000200:28.0:1616513084.604999:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513084.605001:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513084.605005:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3a90 00000800:00000200:28.0:1616513084.605008:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513084.605008:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688970945299200, offset 192 00000800:00000200:28.0:1616513084.605011:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.605013:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513084.605015:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.193@tcp1 00000400:00000200:22.0:1616513084.605022:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.193@tcp1 routed destination 00000400:00000200:22.0:1616513084.605037:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.193@tcp1(10.31.138.193@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.605048:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513084.605054:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.605057:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.605059:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.605064:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.605069:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688970945299072 00000800:00000200:7.2:1616513084.605070:0:49939:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:24.0:1616513084.605079:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688970945299072/t0(0) o400->e0075874-f29d-47ee-2aa7-527ff094f90e@10.31.138.193@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.605094:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688970945299072, offset 192 00000400:00000200:24.0:1616513084.605100:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.193@tcp1 00000400:00000200:24.0:1616513084.605107:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.193@tcp1 routed destination 00000400:00000200:24.0:1616513084.605122:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.193@tcp1(10.31.138.193@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.605128:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.605134:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513084.605135:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513084.605138:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.605140:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513084.605142:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513084.605145:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513084.605145:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513084.605150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.605152:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513084.605152:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.605156:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4acc0 00000800:00000200:2.0:1616513084.605160:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.605163:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513084.605215:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.605222:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.605225:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.605231:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.605237:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:20.0:1616513084.605240:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.605243:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.610257:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.610268:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513084.610269:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513084.610273:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.610280:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.68@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.610288:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.68@tcp1 of length 224 into portal 12 MB=0x5ffe47a2428c0 00000400:00000200:8.0:1616513084.610294:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.68@tcp1 of length 224/224 into md 0x865bbec1 [1] + 72240 00000400:00000200:8.0:1616513084.610298:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.610300:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513084.610306:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.610308:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.610310:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.610312:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513084.610342:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.610351:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513084.610357:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513084.610360:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.68@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.610370:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.68@tcp1 of length 224 into portal 26 MB=0x5ffe47a242580 00000100:00000200:0.0:1616513084.610373:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731650369728 00000400:00000200:14.0:1616513084.610376:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.68@tcp1 of length 224/224 into md 0xf97cacb9 [1] + 0 00000400:00000200:14.0:1616513084.610381:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3c28 00000400:00000200:14.0:1616513084.610384:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:0.0:1616513084.610384:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8d580 x1688731650369728/t0(0) o400->8c5a20a7-c0d9-3668-fa29-9d190014dd15@10.31.129.68@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513084.610387:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513084.610391:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3c28 00000800:00000200:14.0:1616513084.610394:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.610396:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513084.610396:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688731650369728, offset 192 00000800:00000200:14.0:1616513084.610398:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513084.610402:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.68@tcp1 00000400:00000200:0.0:1616513084.610408:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.68@tcp1 routed destination 00000400:00000200:0.0:1616513084.610422:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.68@tcp1(10.31.129.68@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.610427:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513084.610432:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.610437:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.610450:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.610455:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513084.610463:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731650368896 00000800:00000200:7.2:1616513084.610467:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:24.0:1616513084.610473:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688731650368896/t0(0) o400->016aa434-a1f3-4ded-b493-4aa683b23fa7@10.31.129.68@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.610487:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688731650368896, offset 192 00000400:00000200:24.0:1616513084.610492:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.68@tcp1 00000400:00000200:24.0:1616513084.610499:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.68@tcp1 routed destination 00000400:00000200:24.0:1616513084.610513:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.68@tcp1(10.31.129.68@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.610519:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.610525:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.610527:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.610529:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.610533:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513084.610539:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.610539:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513084.610547:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.610554:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.610556:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.610563:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.610566:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb054b8 00000800:00000200:2.0:1616513084.610570:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.610578:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513084.610591:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.610598:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.610600:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.610606:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.610612:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:20.0:1616513084.610615:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.610617:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.632607:0:49943:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.632662:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.632672:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.632681:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.165@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.632690:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.165@tcp1 of length 224 into portal 26 MB=0x60183639b05c0 00000400:00000200:26.0:1616513084.632696:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.165@tcp1 of length 224/224 into md 0xf97cace9 [1] + 0 00000400:00000200:26.0:1616513084.632701:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3210 00000400:00000200:26.0:1616513084.632704:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.632706:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513084.632710:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3210 00000800:00000200:26.0:1616513084.632714:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.632715:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.632716:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.632719:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.632775:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513683711424 00010000:00000200:24.0:1616513084.632785:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1690513683711424/t0(0) o400->216ec965-7383-01a3-a318-5eea332f771a@10.31.136.165@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.632800:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513683711424, offset 192 00000400:00000200:24.0:1616513084.632806:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.165@tcp1 00000400:00000200:24.0:1616513084.632813:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.165@tcp1 routed destination 00000400:00000200:24.0:1616513084.632828:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.165@tcp1(10.31.136.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.632835:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.632841:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.632844:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.632846:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.632851:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.632855:0:49943:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.632857:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513084.632909:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.632916:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.632919:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513084.632925:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.632931:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:2.0:1616513084.632934:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.632937:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.632943:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513084.632945:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.632951:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.165@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.632971:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.165@tcp1 of length 224 into portal 12 MB=0x60183639b0680 00000400:00000200:2.0:1616513084.632977:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.165@tcp1 of length 224/224 into md 0xf964d519 [1] + 37960 00000400:00000200:2.0:1616513084.632981:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.632984:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.632989:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.632992:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.632993:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.633054:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513683711616 00010000:00000200:22.0:1616513084.633066:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dd67d2400 x1690513683711616/t0(0) o400->6ddf1e51-30a9-1516-c425-606b0b09110a@10.31.136.165@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.633080:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513683711616, offset 192 00000400:00000200:22.0:1616513084.633087:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.165@tcp1 00000400:00000200:22.0:1616513084.633094:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.165@tcp1 routed destination 00000400:00000200:22.0:1616513084.633109:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.165@tcp1(10.31.136.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.633116:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.633128:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.633131:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.633133:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.633138:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.633184:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.633197:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.633202:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.633204:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.633210:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.633213:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bd38 00000800:00000200:8.0:1616513084.633217:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.633219:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513084.634487:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926656, portal 4 00000100:00000200:10.0:1616513084.634492:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926656, offset 0 00000400:00000200:10.0:1616513084.634497:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:10.0:1616513084.634502:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:10.0:1616513084.634512:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513084.634517:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:10.0:1616513084.634522:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513084.634525:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:10.0:1616513084.634527:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:10.0:1616513084.634531:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513084.634580:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513084.634600:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513084.634607:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513084.634610:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513084.634618:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513084.634626:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b278480 x1693067725926656/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513084.634638:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82c38 00000800:00000200:1.0:1616513084.634643:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513084.634647:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513084.634672:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513084.634681:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513084.634687:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513084.634690:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513084.634696:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513084.634704:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627100 00000400:00000200:1.0:1616513084.634711:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffd15 [1] + 192 00000400:00000200:1.0:1616513084.634717:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513084.634721:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b278480 x1693067725926656/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513084.634736:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513084.634739:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513084.634740:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:10.0:1616513084.634741:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b278480 x1693067725926656/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513084.634751:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b278480 x1693067725926656/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513084.634759:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83540 00000800:00000200:11.2:1616513084.642866:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513084.642922:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.642932:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.642941:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.35@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.642951:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.35@tcp1 of length 224 into portal 26 MB=0x60230841bd680 00000400:00000200:20.0:1616513084.642957:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.35@tcp1 of length 224/224 into md 0xf97cacf9 [1] + 0 00000400:00000200:20.0:1616513084.642962:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2ff0 00000400:00000200:20.0:1616513084.642967:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.642970:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513084.642974:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2ff0 00000800:00000200:20.0:1616513084.642977:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.642978:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.642980:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.642985:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.642988:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.642988:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.642997:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.35@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.643006:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.35@tcp1 of length 224 into portal 12 MB=0x60230841bd700 00000400:00000200:12.0:1616513084.643012:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.35@tcp1 of length 224/224 into md 0x865bbec1 [1] + 72464 00000400:00000200:12.0:1616513084.643018:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.643020:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.643027:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.643031:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.643032:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.643038:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691257258366592 00010000:00000200:24.0:1616513084.643048:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1691257258366592/t0(0) o400->320d7aba-351f-df9b-13b9-49cbb7064d72@10.31.142.35@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.643063:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691257258366592, offset 192 00000400:00000200:24.0:1616513084.643069:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.35@tcp1 00000400:00000200:24.0:1616513084.643076:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.35@tcp1 routed destination 00000400:00000200:24.0:1616513084.643091:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.35@tcp1(10.31.142.35@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513084.643092:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691257258366720 00000800:00000200:24.0:1616513084.643098:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.643104:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513084.643105:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0da400 x1691257258366720/t0(0) o400->dfce5fda-5fbc-95b0-9436-17e8bac1ed73@10.31.142.35@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513084.643107:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.643109:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.643113:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513084.643119:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691257258366720, offset 192 00000100:00000200:24.0:1616513084.643120:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.643120:0:49944:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513084.643125:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.35@tcp1 00000400:00000200:0.0:1616513084.643133:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.35@tcp1 routed destination 00000400:00000200:0.0:1616513084.643148:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.35@tcp1(10.31.142.35@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.643155:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513084.643160:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.643163:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.643165:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.643170:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513084.643174:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513084.643175:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513084.643181:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.643184:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.643191:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.643196:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:26.0:1616513084.643200:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.643203:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513084.643230:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.643237:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.643251:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.643255:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.643258:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.643262:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a08 00000800:00000200:28.0:1616513084.643265:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.650416:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513084.650471:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.650481:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.650490:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.650499:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x60399133f45c0 00000400:00000200:20.0:1616513084.650505:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0x865bbec1 [1] + 72688 00000400:00000200:20.0:1616513084.650511:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.650513:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513084.650520:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.650523:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.650525:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.650528:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513084.650545:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926672, portal 4 00000100:00000200:30.0:1616513084.650550:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926672, offset 0 00000400:00000200:30.0:1616513084.650557:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:30.0:1616513084.650563:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:30.0:1616513084.650576:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513084.650583:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000100:00000200:0.0:1616513084.650584:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805848057280 00000800:00000200:30.0:1616513084.650589:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513084.650592:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:30.0:1616513084.650594:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:30.0:1616513084.650599:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513084.650647:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00010000:00000200:0.0:1616513084.650652:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8000 x1692805848057280/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:710/0 lens 336/336 e 0 to 0 dl 1616513090 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513084.650667:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805848057280, offset 192 00000800:00000200:27.0:1616513084.650669:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:0.0:1616513084.650673:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000800:00000200:27.0:1616513084.650674:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513084.650677:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513084.650681:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:27.0:1616513084.650684:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513084.650691:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b27b180 x1693067725926672/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513084.650696:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:27.0:1616513084.650703:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee908 00000800:00000200:0.0:1616513084.650703:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:27.0:1616513084.650708:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:0.0:1616513084.650709:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513084.650710:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:0.0:1616513084.650711:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.650713:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.650718:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.650724:0:49945:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.650791:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.650798:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.650801:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513084.650804:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513084.650808:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:27.2:1616513084.650809:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000400:00000200:18.0:1616513084.650812:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05188 00000800:00000200:18.0:1616513084.650816:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:27.0:1616513084.650818:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513084.650824:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513084.650832:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513084.650840:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627110 00000400:00000200:27.0:1616513084.650847:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166dad [1] + 192 00000400:00000200:27.0:1616513084.650851:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513084.650856:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b27b180 x1693067725926672/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513084.650870:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513084.650870:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513084.650873:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513084.650875:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:30.0:1616513084.650878:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b27b180 x1693067725926672/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513084.650889:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b27b180 x1693067725926672/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513084.650899:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eecc0 00000100:00000200:9.0:1616513084.667001:0:6488:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926688, portal 4 00000100:00000200:9.0:1616513084.667008:0:6488:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926688, offset 0 00000400:00000200:9.0:1616513084.667014:0:6488:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:9.0:1616513084.667020:0:6488:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:9.0:1616513084.667034:0:6488:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:9.0:1616513084.667041:0:6488:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:9.0:1616513084.667047:0:6488:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:9.0:1616513084.667051:0:6488:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:9.0:1616513084.667053:0:6488:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:9.0:1616513084.667058:0:6488:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513084.667066:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513084.667079:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513084.667086:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513084.667090:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.667097:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513084.667103:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cab101b00 x1693067725926688/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513084.667115:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35980 00000800:00000200:8.0:1616513084.667120:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513084.667123:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513084.667232:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:12.0:1616513084.667286:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513084.667304:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:12.0:1616513084.667313:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513084.667321:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627120 00000400:00000200:12.0:1616513084.667328:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d7fa1 [1] + 192 00000400:00000200:12.0:1616513084.667333:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.667339:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cab101b00 x1693067725926688/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513084.667351:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513084.667354:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513084.667357:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513084.667359:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:9.0:1616513084.667361:0:6488:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cab101b00 x1693067725926688/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:9.0:1616513084.667374:0:6488:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cab101b00 x1693067725926688/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:9.0:1616513084.667385:0:6488:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35430 00000800:00000200:7.2:1616513084.674758:0:49951:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.674815:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.674825:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.674834:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.157@tcp1 : PUT - for me 00000800:00000200:14.0:1616513084.674843:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513084.674844:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.157@tcp1 of length 224 into portal 26 MB=0x604b89b7d8240 00000400:00000200:26.0:1616513084.674850:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.157@tcp1 of length 224/224 into md 0xf97cad19 [1] + 0 00000800:00000200:14.0:1616513084.674853:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.674855:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2088 00000400:00000200:26.0:1616513084.674858:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.674861:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.157@tcp1 : PUT - for me 00000100:00000200:26.0:1616513084.674863:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513084.674868:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2088 00000800:00000200:26.0:1616513084.674871:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513084.674873:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.157@tcp1 of length 224 into portal 12 MB=0x604b89b7d8500 00000800:00000200:26.0:1616513084.674874:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513084.674876:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.674879:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513084.674879:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.157@tcp1 of length 224/224 into md 0xf964d519 [1] + 38184 00000400:00000200:14.0:1616513084.674884:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.674887:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.674895:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.674898:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.674900:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.674909:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694040789452032 00010000:00000200:22.0:1616513084.674917:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464620900 x1694040789452032/t0(0) o400->2933f40a-4cf4-864b-ec97-c2fe0663bed7@10.31.140.157@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.674929:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694040789452032, offset 192 00000400:00000200:22.0:1616513084.674934:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.157@tcp1 00000100:00000200:24.0:1616513084.674940:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694040789451328 00000400:00000200:22.0:1616513084.674940:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.157@tcp1 routed destination 00010000:00000200:24.0:1616513084.674952:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1694040789451328/t0(0) o400->2596627c-655e-0109-95d0-16b19adb0e72@10.31.140.157@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513084.674953:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.157@tcp1(10.31.140.157@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.674959:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.674964:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:24.0:1616513084.674966:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694040789451328, offset 192 00000800:00000200:22.0:1616513084.674966:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.674968:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.674971:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513084.674973:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.157@tcp1 00000400:00000200:24.0:1616513084.674980:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.157@tcp1 routed destination 00000400:00000200:24.0:1616513084.674995:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.157@tcp1(10.31.140.157@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.675002:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.675008:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.675011:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.675013:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.675018:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513084.675018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513084.675022:0:49951:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.675025:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513084.675033:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513084.675036:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513084.675038:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.675040:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513084.675043:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.675046:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513084.675046:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.675046:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.675050:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bba0 00000400:00000200:18.0:1616513084.675053:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:8.0:1616513084.675053:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.675056:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513084.675058:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:18.0:1616513084.675062:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.679911:0:49955:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.679966:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.679976:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513084.679979:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513084.679984:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513084.679985:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.126@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.679995:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.126@tcp1 of length 224 into portal 12 MB=0x60184abc47340 00000400:00000200:2.0:1616513084.680001:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.126@tcp1 of length 224/224 into md 0xf964d519 [1] + 38408 00000400:00000200:2.0:1616513084.680006:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.680009:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.680016:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.680019:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.680021:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513084.680041:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.680050:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513084.680054:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.680059:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.126@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.680069:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.126@tcp1 of length 224 into portal 26 MB=0x60184abc47240 00000400:00000200:20.0:1616513084.680076:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.126@tcp1 of length 224/224 into md 0xf97cad49 [1] + 0 00000400:00000200:20.0:1616513084.680080:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2aa0 00000400:00000200:20.0:1616513084.680083:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513084.680084:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519189353280 00000100:00000200:20.0:1616513084.680086:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513084.680090:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2aa0 00000800:00000200:20.0:1616513084.680093:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513084.680096:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebba80 x1690519189353280/t0(0) o400->4cc4b408-44e7-3b0f-af3a-b7c92da2fe01@10.31.132.126@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513084.680096:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.680098:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513084.680111:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690519189353280, offset 192 00000400:00000200:22.0:1616513084.680117:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.126@tcp1 00000400:00000200:22.0:1616513084.680125:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.126@tcp1 routed destination 00000400:00000200:22.0:1616513084.680140:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.126@tcp1(10.31.132.126@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.680146:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.680152:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.680155:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.680157:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.680162:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513084.680163:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519189353024 00000800:00000200:11.2:1616513084.680167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513084.680174:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1690519189353024/t0(0) o400->902eb782-2be7-9328-b03b-de94cda09c74@10.31.132.126@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:8.0:1616513084.680176:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.680180:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.680181:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.680187:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:24.0:1616513084.680188:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690519189353024, offset 192 00000400:00000200:8.0:1616513084.680190:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b5c8 00000800:00000200:8.0:1616513084.680193:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513084.680196:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.126@tcp1 00000800:00000200:8.0:1616513084.680196:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513084.680204:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.126@tcp1 routed destination 00000400:00000200:24.0:1616513084.680219:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.126@tcp1(10.31.132.126@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.680226:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.680231:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.680234:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.680236:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.680241:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.680246:0:49955:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.680248:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513084.680257:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.680264:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.680267:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.680274:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.680279:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:26.0:1616513084.680283:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.680286:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.710818:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.710874:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.710885:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.710895:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.8@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.710905:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.8@tcp1 of length 224 into portal 12 MB=0x5ffd45279e980 00000400:00000200:14.0:1616513084.710912:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.8@tcp1 of length 224/224 into md 0xf964d519 [1] + 38632 00000400:00000200:14.0:1616513084.710918:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.710922:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.710930:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.710931:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.710933:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.710935:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.710946:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513084.711000:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662265424256 00000800:00000200:12.0:1616513084.711004:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.711013:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00010000:00000200:22.0:1616513084.711015:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464625a00 x1688662265424256/t0(0) o400->a272332d-223b-ec53-5bc6-916fb70f1d67@10.31.133.8@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513084.711023:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.8@tcp1 : PUT - for me 00000100:00000200:22.0:1616513084.711031:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662265424256, offset 192 00000400:00000200:12.0:1616513084.711032:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.8@tcp1 of length 224 into portal 26 MB=0x5ffd45279e900 00000400:00000200:22.0:1616513084.711037:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.8@tcp1 00000400:00000200:12.0:1616513084.711043:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.8@tcp1 of length 224/224 into md 0xf97cad79 [1] + 0 00000400:00000200:22.0:1616513084.711045:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.8@tcp1 routed destination 00000400:00000200:12.0:1616513084.711047:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2dd0 00000400:00000200:12.0:1616513084.711050:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.711053:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513084.711058:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2dd0 00000400:00000200:22.0:1616513084.711060:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.8@tcp1(10.31.133.8@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513084.711062:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.711065:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513084.711066:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513084.711068:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.711070:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513084.711072:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.711075:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.711077:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.711083:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.711088:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.711098:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.711104:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.711106:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.711112:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.711116:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b760 00000800:00000200:8.0:1616513084.711119:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.711122:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.711126:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662265424128 00010000:00000200:24.0:1616513084.711138:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688662265424128/t0(0) o400->ef985711-7ad0-6ca6-1162-304659e1198c@10.31.133.8@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.711152:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662265424128, offset 192 00000400:00000200:24.0:1616513084.711158:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.8@tcp1 00000400:00000200:24.0:1616513084.711166:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.8@tcp1 routed destination 00000400:00000200:24.0:1616513084.711181:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.8@tcp1(10.31.133.8@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.711188:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.711194:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.711197:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.711199:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.711204:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.711210:0:49959:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.711211:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513084.711223:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.711230:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.711233:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.711240:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.711245:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:26.0:1616513084.711249:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.711252:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.724816:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513084.724873:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.724884:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.724893:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.37@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.724903:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.37@tcp1 of length 224 into portal 26 MB=0x6048d6c92f1c0 00000400:00000200:20.0:1616513084.724909:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.37@tcp1 of length 224/224 into md 0xf97cad89 [1] + 0 00000400:00000200:20.0:1616513084.724914:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b22a8 00000400:00000200:20.0:1616513084.724917:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.724920:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513084.724924:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b22a8 00000800:00000200:20.0:1616513084.724927:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.724928:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.724930:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.724932:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.724987:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855318733248 00010000:00000200:24.0:1616513084.724998:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1693855318733248/t0(0) o400->c0a29d73-39f1-6b48-3b4f-10f7ac400205@10.31.141.37@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.725013:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855318733248, offset 192 00000400:00000200:24.0:1616513084.725019:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.37@tcp1 00000400:00000200:24.0:1616513084.725026:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.37@tcp1 routed destination 00000400:00000200:24.0:1616513084.725041:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.37@tcp1(10.31.141.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.725048:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.725055:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:7.2:1616513084.725055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.725058:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.725060:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.725065:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.725071:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.725072:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513084.725080:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.725085:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.725088:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.725093:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.725098:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3210 00000800:00000200:8.0:1616513084.725101:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.725104:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513084.725114:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.725124:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.725133:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.37@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.725142:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.37@tcp1 of length 224 into portal 12 MB=0x6048d6c92f400 00000400:00000200:2.0:1616513084.725151:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.37@tcp1 of length 224/224 into md 0xf964d519 [1] + 38856 00000400:00000200:2.0:1616513084.725156:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.725159:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.725166:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.725168:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.725169:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.725172:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.725233:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855318733824 00010000:00000200:22.0:1616513084.725246:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebde80 x1693855318733824/t0(0) o400->029994ca-dcea-ba83-ff7d-d026f07fa218@10.31.141.37@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.725260:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855318733824, offset 192 00000400:00000200:22.0:1616513084.725267:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.37@tcp1 00000400:00000200:22.0:1616513084.725274:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.37@tcp1 routed destination 00000400:00000200:22.0:1616513084.725291:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.37@tcp1(10.31.141.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.725298:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513084.725304:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.725307:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.725309:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.725314:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.725319:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.725344:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.725350:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.725353:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513084.725357:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513084.725360:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.725364:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bc28 00000800:00000200:26.0:1616513084.725368:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.738752:0:49962:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.738809:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.738818:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.738827:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.64@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.738837:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.64@tcp1 of length 224 into portal 26 MB=0x601827c9bf7c0 00000400:00000200:14.0:1616513084.738844:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.64@tcp1 of length 224/224 into md 0xf97cad99 [1] + 0 00000400:00000200:14.0:1616513084.738849:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2f68 00000400:00000200:14.0:1616513084.738852:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.738854:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513084.738858:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2f68 00000800:00000200:14.0:1616513084.738862:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.738863:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.738865:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.738867:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.738923:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509808236480 00010000:00000200:24.0:1616513084.738934:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1690509808236480/t0(0) o400->974d9da2-3caa-c2b9-b79c-0f34e7985d76@10.31.128.64@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.738949:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690509808236480, offset 192 00000400:00000200:24.0:1616513084.738955:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.64@tcp1 00000400:00000200:24.0:1616513084.738963:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.64@tcp1 routed destination 00000400:00000200:24.0:1616513084.738978:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.64@tcp1(10.31.128.64@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.738985:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.738990:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.738993:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.738995:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513084.738996:0:49962:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.739000:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513084.739007:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.739007:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513084.739052:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.739062:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513084.739070:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.64@tcp1 : PUT - for me 00000800:00000200:12.0:1616513084.739070:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.739076:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513084.739079:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.64@tcp1 of length 224 into portal 12 MB=0x601827c9bfb40 00000400:00000200:12.0:1616513084.739079:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.739086:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.64@tcp1 of length 224/224 into md 0xf964d519 [1] + 39080 00000400:00000200:12.0:1616513084.739086:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.739092:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:12.0:1616513084.739092:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000100:00000200:18.0:1616513084.739094:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.739095:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.739098:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513084.739102:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.739105:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.739106:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.739123:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.739178:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509808237376 00010000:00000200:22.0:1616513084.739191:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfad00 x1690509808237376/t0(0) o400->516ad39e-9c50-57a0-0df3-86884a98b2fc@10.31.128.64@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.739205:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690509808237376, offset 192 00000400:00000200:22.0:1616513084.739212:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.64@tcp1 00000400:00000200:22.0:1616513084.739219:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.64@tcp1 routed destination 00000400:00000200:22.0:1616513084.739234:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.64@tcp1(10.31.128.64@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.739240:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513084.739246:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.739249:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.739251:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.739256:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.739260:0:49962:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.739271:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.739277:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.739279:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513084.739281:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.739286:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.739290:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b078 00000800:00000200:2.0:1616513084.739294:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513084.762590:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926704, portal 4 00000100:00000200:30.0:1616513084.762597:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926704, offset 0 00000400:00000200:30.0:1616513084.762603:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513084.762609:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513084.762634:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513084.762641:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513084.762646:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513084.762649:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513084.762651:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513084.762657:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513084.762702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513084.762716:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513084.762722:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513084.762725:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.762731:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513084.762736:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b27de80 x1693067725926704/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513084.762748:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef3a8 00000800:00000200:8.0:1616513084.762751:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513084.762754:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513084.762873:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:12.0:1616513084.762927:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513084.762936:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:12.0:1616513084.762945:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513084.762954:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627130 00000400:00000200:12.0:1616513084.762960:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d8001 [1] + 192 00000400:00000200:12.0:1616513084.762965:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.762971:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b27de80 x1693067725926704/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513084.762980:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513084.762985:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513084.762988:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513084.762990:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513084.762991:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b27de80 x1693067725926704/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513084.763003:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b27de80 x1693067725926704/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513084.763012:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eecc0 00000800:00000200:11.2:1616513084.810945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.810967:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.810977:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.810986:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.167@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.810997:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.167@tcp1 of length 224 into portal 26 MB=0x6018289c86fc0 00000400:00000200:8.0:1616513084.811004:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.167@tcp1 of length 224/224 into md 0xf97cada9 [1] + 0 00000400:00000200:8.0:1616513084.811009:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b34b8 00000400:00000200:8.0:1616513084.811012:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.811015:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.811020:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b34b8 00000800:00000200:28.0:1616513084.811021:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.811024:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.811027:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.811029:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.811089:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510029254592 00010000:00000200:24.0:1616513084.811101:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1690510029254592/t0(0) o400->03f526ff-b6e1-d121-5080-f2cf5cd8babe@10.31.140.167@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.811117:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510029254592, offset 192 00000400:00000200:24.0:1616513084.811124:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.167@tcp1 00000400:00000200:24.0:1616513084.811131:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.167@tcp1 routed destination 00000400:00000200:24.0:1616513084.811147:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.167@tcp1(10.31.140.167@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.811153:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.811160:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.811163:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.811165:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.811170:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.811175:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.811192:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513084.811235:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.811242:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.811245:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.811252:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.811258:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:20.0:1616513084.811261:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.811264:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.811372:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513084.811383:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.811391:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513084.811391:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.811400:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.167@tcp1 : PUT - for me 00000400:00000200:28.0:1616513084.811409:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.167@tcp1 of length 224 into portal 12 MB=0x6018289c87240 00000400:00000200:28.0:1616513084.811416:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.167@tcp1 of length 224/224 into md 0x865bbec1 [1] + 73024 00000400:00000200:28.0:1616513084.811421:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513084.811424:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513084.811432:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.811448:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.811450:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.811504:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510029255232 00010000:00000200:0.0:1616513084.811518:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ea880 x1690510029255232/t0(0) o400->d55d6c35-3979-d354-3cbc-4e5a78774f67@10.31.140.167@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.811543:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510029255232, offset 192 00000400:00000200:0.0:1616513084.811550:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.167@tcp1 00000400:00000200:0.0:1616513084.811557:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.167@tcp1 routed destination 00000400:00000200:0.0:1616513084.811573:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.167@tcp1(10.31.140.167@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.811579:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513084.811586:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.811589:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.811591:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.811597:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.811604:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.811660:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.811668:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.811671:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.811678:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.811681:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb056d8 00000800:00000200:26.0:1616513084.811685:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.811688:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.813755:0:49970:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.813811:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.813833:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.813842:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.15@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.813851:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.15@tcp1 of length 224 into portal 26 MB=0x5ffe555f18e40 00000400:00000200:14.0:1616513084.813857:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.15@tcp1 of length 224/224 into md 0xf97cadb9 [1] + 0 00000400:00000200:14.0:1616513084.813861:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3a08 00000400:00000200:14.0:1616513084.813864:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.813867:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513084.813870:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3a08 00000800:00000200:2.0:1616513084.813871:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.813874:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.813876:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513084.813878:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.813881:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.813882:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.813890:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.15@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.813899:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.15@tcp1 of length 224 into portal 12 MB=0x5ffe555f19080 00000400:00000200:2.0:1616513084.813906:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.15@tcp1 of length 224/224 into md 0xf964d519 [1] + 39304 00000400:00000200:2.0:1616513084.813911:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.813913:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.813921:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.813924:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.813926:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.813933:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688735338040896 00010000:00000200:24.0:1616513084.813944:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688735338040896/t0(0) o400->63723392-9570-6d87-73a8-13ece5fed182@10.31.142.15@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.813958:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688735338040896, offset 192 00000400:00000200:24.0:1616513084.813964:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.15@tcp1 00000400:00000200:24.0:1616513084.813972:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.15@tcp1 routed destination 00000400:00000200:24.0:1616513084.813986:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.15@tcp1(10.31.142.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513084.813989:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688735338041472 00000800:00000200:24.0:1616513084.813992:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.813998:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.814001:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:22.0:1616513084.814002:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb8000 x1688735338041472/t0(0) o400->cf883b47-fb5e-e39b-be6c-c67b34d57e1c@10.31.142.15@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513084.814003:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.814007:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513084.814014:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513084.814016:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688735338041472, offset 192 00000400:00000200:22.0:1616513084.814023:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.15@tcp1 00000400:00000200:22.0:1616513084.814030:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.15@tcp1 routed destination 00000400:00000200:22.0:1616513084.814045:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.15@tcp1(10.31.142.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.814051:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513084.814052:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.814057:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.814060:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.814064:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.814068:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.814073:0:49970:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513084.814107:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.814114:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.814116:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.814123:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.814129:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:18.0:1616513084.814130:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513084.814132:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.814135:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513084.814137:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.814140:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513084.814141:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513084.814146:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.814150:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a5d8 00000800:00000200:18.0:1616513084.814154:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.816030:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513084.816055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.816065:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.816070:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.816077:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.121@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.816083:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.121@tcp1 of length 224 into portal 12 MB=0x602acbf77b300 00000800:00000200:2.0:1616513084.816085:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513084.816087:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.121@tcp1 of length 224/224 into md 0x865bbec1 [1] + 73248 00000400:00000200:8.0:1616513084.816091:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.816093:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.816094:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:8.0:1616513084.816098:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.816100:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.816101:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513084.816103:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.121@tcp1 : PUT - for me 00000800:00000200:8.0:1616513084.816104:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513084.816112:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.121@tcp1 of length 224 into portal 26 MB=0x602acbf77b0c0 00000400:00000200:2.0:1616513084.816118:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.121@tcp1 of length 224/224 into md 0xf97cade9 [1] + 0 00000400:00000200:2.0:1616513084.816123:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3298 00000400:00000200:2.0:1616513084.816125:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.816128:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.816132:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3298 00000800:00000200:2.0:1616513084.816137:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.816140:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.816142:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.816144:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513084.816173:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691790830187264 00010000:00000200:0.0:1616513084.816186:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8ba80 x1691790830187264/t0(0) o400->e4b309b9-80dd-c068-7a3f-b909946569fd@10.31.141.121@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513084.816200:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691790830186688 00000100:00000200:0.0:1616513084.816200:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691790830187264, offset 192 00000400:00000200:0.0:1616513084.816206:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.121@tcp1 00010000:00000200:24.0:1616513084.816211:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1691790830186688/t0(0) o400->a6557b23-1e42-9b7f-aaff-db56d6ee06ed@10.31.141.121@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513084.816214:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.121@tcp1 routed destination 00000100:00000200:24.0:1616513084.816225:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691790830186688, offset 192 00000400:00000200:0.0:1616513084.816229:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.121@tcp1(10.31.141.121@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513084.816232:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.121@tcp1 00000800:00000200:0.0:1616513084.816235:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513084.816239:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.121@tcp1 routed destination 00000800:00000200:0.0:1616513084.816246:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.816249:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.816251:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513084.816254:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.121@tcp1(10.31.141.121@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.816256:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513084.816261:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513084.816261:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.816266:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.816269:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.816271:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.816276:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.816281:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.816283:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513084.816314:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.816321:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.816323:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.816330:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.816334:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04550 00000800:00000200:14.0:1616513084.816335:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513084.816338:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.816341:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513084.816342:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.816344:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.816351:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.816356:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:14.0:1616513084.816359:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.816362:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.824409:0:49971:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.824465:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.824475:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.824483:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.87@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.824492:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.87@tcp1 of length 224 into portal 12 MB=0x601830592f340 00000400:00000200:2.0:1616513084.824499:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.87@tcp1 of length 224/224 into md 0xf964d519 [1] + 39528 00000400:00000200:2.0:1616513084.824504:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.824507:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.824513:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.824516:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.824517:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.824520:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.824525:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513084.824530:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.824532:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.87@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.824537:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.87@tcp1 of length 224 into portal 26 MB=0x601830592ef80 00000400:00000200:2.0:1616513084.824543:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.87@tcp1 of length 224/224 into md 0xf97cadf9 [1] + 0 00000400:00000200:2.0:1616513084.824547:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3188 00000400:00000200:2.0:1616513084.824549:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.824551:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513084.824554:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3188 00000800:00000200:2.0:1616513084.824556:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.824558:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.824560:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513084.824576:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512106123136 00000100:00000200:22.0:1616513084.824578:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512106124096 00010000:00000200:24.0:1616513084.824587:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1690512106123136/t0(0) o400->437f8dc6-8b83-6715-e539-f4e4d4a49eef@10.31.128.87@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:22.0:1616513084.824590:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb8900 x1690512106124096/t0(0) o400->6b966e5f-230d-3792-a632-15ad7eedeea7@10.31.128.87@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513084.824601:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512106123136, offset 192 00000100:00000200:22.0:1616513084.824604:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512106124096, offset 192 00000400:00000200:24.0:1616513084.824609:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.87@tcp1 00000400:00000200:22.0:1616513084.824611:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.87@tcp1 00000400:00000200:24.0:1616513084.824617:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.87@tcp1 routed destination 00000400:00000200:22.0:1616513084.824618:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.87@tcp1 routed destination 00000400:00000200:24.0:1616513084.824632:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.87@tcp1(10.31.128.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513084.824633:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.87@tcp1(10.31.128.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.824639:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513084.824639:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513084.824645:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.824645:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.824647:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513084.824648:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513084.824649:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513084.824650:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513084.824654:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513084.824655:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.824660:0:49971:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513084.824661:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.824700:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.824711:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.824715:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513084.824716:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:8.0:1616513084.824717:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.824723:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513084.824725:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.824727:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.824727:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:26.0:1616513084.824729:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513084.824730:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.824732:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513084.824734:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.824738:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a2a8 00000800:00000200:18.0:1616513084.824742:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.825581:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513084.825638:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.825649:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.825658:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.112@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.825667:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.112@tcp1 of length 224 into portal 26 MB=0x60392c1cbf6c0 00000400:00000200:20.0:1616513084.825674:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.112@tcp1 of length 224/224 into md 0xf97cae19 [1] + 0 00000400:00000200:20.0:1616513084.825679:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2a18 00000400:00000200:20.0:1616513084.825681:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.825684:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513084.825689:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2a18 00000800:00000200:20.0:1616513084.825692:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.825693:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.825695:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:24.0:1616513084.825700:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692778711742144 00000800:00000200:20.0:1616513084.825702:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:24.0:1616513084.825705:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1692778711742144/t0(0) o400->0c8b6679-b130-dafb-7b5f-dbf71bed7bcb@10.31.143.112@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513084.825705:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.825705:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513084.825714:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.112@tcp1 : PUT - for me 00000100:00000200:24.0:1616513084.825716:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692778711742144, offset 192 00000400:00000200:24.0:1616513084.825721:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.112@tcp1 00000400:00000200:12.0:1616513084.825723:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.112@tcp1 of length 224 into portal 12 MB=0x60392c1cbf740 00000400:00000200:24.0:1616513084.825726:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.112@tcp1 routed destination 00000400:00000200:12.0:1616513084.825729:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.112@tcp1 of length 224/224 into md 0x865bbec1 [1] + 73472 00000400:00000200:12.0:1616513084.825734:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.825736:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513084.825739:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.112@tcp1(10.31.143.112@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.825744:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513084.825744:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.825747:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513084.825748:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513084.825749:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513084.825750:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.825752:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.825755:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513084.825758:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.825760:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.825771:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.825779:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.825782:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513084.825782:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513084.825788:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.825793:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:28.0:1616513084.825797:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.825807:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692778711742272 00010000:00000200:0.0:1616513084.825820:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d204800 x1692778711742272/t0(0) o400->0b121622-fc39-ff86-202f-ac0adabcc9b8@10.31.143.112@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.825834:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692778711742272, offset 192 00000400:00000200:0.0:1616513084.825841:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.112@tcp1 00000400:00000200:0.0:1616513084.825848:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.112@tcp1 routed destination 00000400:00000200:0.0:1616513084.825863:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.112@tcp1(10.31.143.112@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.825871:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513084.825877:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.825879:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.825882:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.825886:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.825894:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.825948:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.825955:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.825957:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.825963:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.825967:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb042a8 00000800:00000200:14.0:1616513084.825971:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.825974:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.835075:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513084.835140:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.835149:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.835158:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.164@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.835167:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.164@tcp1 of length 224 into portal 26 MB=0x5ffd5673f3280 00000400:00000200:20.0:1616513084.835174:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.164@tcp1 of length 224/224 into md 0xf97cae49 [1] + 0 00000400:00000200:20.0:1616513084.835179:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2198 00000400:00000200:20.0:1616513084.835181:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.835184:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513084.835189:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2198 00000800:00000200:20.0:1616513084.835192:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.835192:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.835194:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.835197:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.835253:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666908865152 00010000:00000200:24.0:1616513084.835263:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688666908865152/t0(0) o400->4cd2e5de-093d-f818-a753-fd42f3b2d316@10.31.132.164@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.835278:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666908865152, offset 192 00000400:00000200:24.0:1616513084.835284:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.164@tcp1 00000400:00000200:24.0:1616513084.835291:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.164@tcp1 routed destination 00000400:00000200:24.0:1616513084.835306:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.164@tcp1(10.31.132.164@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.835318:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.835323:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.835326:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.835328:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.835333:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.835338:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.835340:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513084.835346:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.835351:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513084.835353:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513084.835358:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513084.835361:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:8.0:1616513084.835364:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.835367:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.836237:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.836248:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.836255:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513084.836257:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.836264:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.164@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.836273:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.164@tcp1 of length 224 into portal 12 MB=0x5ffd5673f3300 00000400:00000200:12.0:1616513084.836285:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.164@tcp1 of length 224/224 into md 0x865bbec1 [1] + 73696 00000400:00000200:12.0:1616513084.836290:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.836293:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.836300:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.836302:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.836304:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.836362:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666908865280 00010000:00000200:0.0:1616513084.836374:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d205100 x1688666908865280/t0(0) o400->68d1b68d-7b8b-47d7-6484-3d119635f68d@10.31.132.164@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.836387:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666908865280, offset 192 00000400:00000200:0.0:1616513084.836393:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.164@tcp1 00000400:00000200:0.0:1616513084.836400:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.164@tcp1 routed destination 00000400:00000200:0.0:1616513084.836414:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.164@tcp1(10.31.132.164@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.836420:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513084.836425:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.836428:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.836430:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.836448:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.836454:0:49972:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.836514:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.836520:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.836523:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.836530:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.836534:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb043b8 00000800:00000200:26.0:1616513084.836538:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.836541:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.837740:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.837750:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.837757:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.837764:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.235@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.837771:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.235@tcp1 of length 224 into portal 26 MB=0x5ffdd5a342e80 00000400:00000200:8.0:1616513084.837777:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.235@tcp1 of length 224/224 into md 0xf97cae59 [1] + 0 00000400:00000200:8.0:1616513084.837781:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3320 00000400:00000200:8.0:1616513084.837783:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.837785:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.837788:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3320 00000800:00000200:8.0:1616513084.837790:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513084.837793:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.837794:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.837796:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.837859:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701049777792 00010000:00000200:24.0:1616513084.837870:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688701049777792/t0(0) o400->fba11e7f-c88e-9616-616a-28015efcb7cd@10.31.128.235@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.837884:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701049777792, offset 192 00000400:00000200:24.0:1616513084.837890:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.235@tcp1 00000400:00000200:24.0:1616513084.837898:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.235@tcp1 routed destination 00000400:00000200:24.0:1616513084.837913:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.235@tcp1(10.31.128.235@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.837919:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.837925:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.837927:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.837930:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.837934:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.837939:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513084.837941:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513084.837992:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.837999:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.838002:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513084.838008:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.838014:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5ed0 00000800:00000200:20.0:1616513084.838017:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.838020:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.838488:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.838496:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.838502:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513084.838506:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.838509:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.235@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.838514:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.235@tcp1 of length 224 into portal 12 MB=0x5ffdd5a343080 00000400:00000200:8.0:1616513084.838519:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.235@tcp1 of length 224/224 into md 0x865bbec1 [1] + 73920 00000400:00000200:8.0:1616513084.838523:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.838525:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513084.838529:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.838531:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.838533:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.838598:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701049778304 00010000:00000200:0.0:1616513084.838610:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89b00 x1688701049778304/t0(0) o400->57dec56d-00ff-ecf9-161a-ca07f9cc3f5f@10.31.128.235@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.838625:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701049778304, offset 192 00000400:00000200:0.0:1616513084.838632:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.235@tcp1 00000400:00000200:0.0:1616513084.838639:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.235@tcp1 routed destination 00000400:00000200:0.0:1616513084.838654:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.235@tcp1(10.31.128.235@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.838660:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513084.838666:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.838669:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.838671:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.838676:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.838682:0:49972:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.838757:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.838764:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.838766:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.838773:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.838776:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05980 00000800:00000200:14.0:1616513084.838782:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.838785:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.838790:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513084.838792:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.838794:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.838795:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.838797:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.840239:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513084.840308:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.840319:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.840328:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.109@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.840337:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.109@tcp1 of length 224 into portal 26 MB=0x6001e5c95c940 00000400:00000200:20.0:1616513084.840343:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.109@tcp1 of length 224/224 into md 0xf97cae89 [1] + 0 00000400:00000200:20.0:1616513084.840348:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b38f8 00000400:00000200:20.0:1616513084.840351:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.840354:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513084.840358:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b38f8 00000800:00000200:20.0:1616513084.840361:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.840363:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513084.840364:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.840367:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.840420:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688980262603072 00010000:00000200:24.0:1616513084.840431:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688980262603072/t0(0) o400->0533e04c-8d12-3243-ec90-6ccbd2b41756@10.31.142.109@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513084.840433:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513084.840447:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.840455:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:24.0:1616513084.840457:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688980262603072, offset 192 00000800:00000200:8.0:1616513084.840457:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513084.840463:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.109@tcp1 00000400:00000200:28.0:1616513084.840464:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.109@tcp1 : PUT - for me 00000400:00000200:24.0:1616513084.840471:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.109@tcp1 routed destination 00000400:00000200:24.0:1616513084.840485:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.109@tcp1(10.31.142.109@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.840492:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513084.840494:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.109@tcp1 of length 224 into portal 12 MB=0x6001e5c95cd00 00000800:00000200:24.0:1616513084.840498:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513084.840500:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.109@tcp1 of length 224/224 into md 0x865bbec1 [1] + 74144 00000800:00000200:24.0:1616513084.840500:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.840504:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513084.840506:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513084.840508:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513084.840509:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:11.2:1616513084.840514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.840515:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513084.840516:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513084.840519:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513084.840521:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513084.840523:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.840527:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513084.840528:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.840530:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.840535:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.840539:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5a90 00000800:00000200:28.0:1616513084.840542:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.840574:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688980262604032 00010000:00000200:0.0:1616513084.840585:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0e9f80 x1688980262604032/t0(0) o400->b4ec5c35-4773-1823-ed1c-4e27c229fd73@10.31.142.109@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.840598:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688980262604032, offset 192 00000400:00000200:0.0:1616513084.840603:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.109@tcp1 00000400:00000200:0.0:1616513084.840609:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.109@tcp1 routed destination 00000400:00000200:0.0:1616513084.840622:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.109@tcp1(10.31.142.109@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.840628:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513084.840633:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.840635:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.840637:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.840641:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.840648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.840703:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.840709:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.840711:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513084.840717:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.840721:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05298 00000800:00000200:18.0:1616513084.840725:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.840728:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.840905:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.840915:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.840921:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513084.840923:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513084.840929:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.144@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.840949:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.144@tcp1 of length 224 into portal 26 MB=0x5ffd6ff08b5c0 00000400:00000200:8.0:1616513084.840954:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.144@tcp1 of length 224/224 into md 0xf97cae99 [1] + 0 00000400:00000200:8.0:1616513084.840958:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2770 00000400:00000200:8.0:1616513084.840960:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.840962:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.840965:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2770 00000800:00000200:8.0:1616513084.840968:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.840971:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.840972:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513084.840978:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673750398400 00010000:00000200:24.0:1616513084.840984:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688673750398400/t0(0) o400->1ae5286a-5bb2-7c5d-0a95-5e76401a29c9@10.31.129.144@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513084.840995:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688673750398400, offset 192 00000400:00000200:24.0:1616513084.841001:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.144@tcp1 00000400:00000200:24.0:1616513084.841007:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.144@tcp1 routed destination 00000400:00000200:24.0:1616513084.841020:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.144@tcp1(10.31.129.144@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513084.841026:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513084.841031:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513084.841033:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513084.841035:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513084.841038:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513084.841042:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.841043:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.841052:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.841056:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513084.841058:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513084.841061:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513084.841064:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513084.841069:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4b28 00000800:00000200:20.0:1616513084.841071:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.841140:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513084.841194:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.841204:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.841213:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.144@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.841221:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.144@tcp1 of length 224 into portal 12 MB=0x5ffd6ff08b9c0 00000400:00000200:2.0:1616513084.841227:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.144@tcp1 of length 224/224 into md 0xf964d519 [1] + 39752 00000400:00000200:2.0:1616513084.841232:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.841237:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.841243:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.841246:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.841246:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.841250:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.841305:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673750399424 00010000:00000200:22.0:1616513084.841317:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebda00 x1688673750399424/t0(0) o400->aa3c2567-8412-14e8-cf88-2c155475f4a3@10.31.129.144@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.841330:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688673750399424, offset 192 00000400:00000200:22.0:1616513084.841336:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.144@tcp1 00000400:00000200:22.0:1616513084.841343:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.144@tcp1 routed destination 00000400:00000200:22.0:1616513084.841356:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.144@tcp1(10.31.129.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.841362:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513084.841367:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.841370:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.841372:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.841376:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.841381:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.841390:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.841393:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.841395:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513084.841398:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513084.841400:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.841402:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4abb0 00000800:00000200:18.0:1616513084.841405:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:7.0:1616513084.842602:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1693067725926720, portal 25 00000100:00000200:7.0:1616513084.842606:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1693067725926720, offset 0 00000400:00000200:7.0:1616513084.842611:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:7.0:1616513084.842617:0:6507:0:(lib-move.c:4150:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:7.0:1616513084.842623:0:6507:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x603d60c627140 00000400:00000200:7.0:1616513084.842628:0:6507:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0xf97caea9 [1] + 0 00000400:00000200:7.0:1616513084.842632:0:6507:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2990 00000400:00000200:7.0:1616513084.842636:0:6507:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:7.0:1616513084.842638:0:6507:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:7.0:1616513084.842643:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2990 00000400:00000200:7.0:1616513084.842646:0:6507:0:(lib-msg.c:806:lnet_is_health_check()) msg ffff8da53f87ca00 failed too early to retry and send 00000100:00000200:7.0:1616513084.842648:0:6507:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd604c80 x1693067725926720/t0(0) o400->MGC10.242.62.139@tcp5@0@lo:26/25 lens 224/224 e 0 to 0 dl 1616513177 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513084.842658:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637ecc0 00000100:00000200:15.0:1616513084.842665:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1693067725926752, portal 25 00000100:00000200:19.0:1616513084.842667:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1693067725926736, portal 10 00000100:00000200:15.0:1616513084.842670:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1693067725926752, offset 0 00000100:00000200:19.0:1616513084.842672:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1693067725926736, offset 0 00000400:00000200:15.0:1616513084.842675:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:19.0:1616513084.842681:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:15.0:1616513084.842681:0:6509:0:(lib-move.c:4150:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:19.0:1616513084.842688:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:15.0:1616513084.842694:0:6509:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-0@lo of length 224 into portal 26 MB=0x603d60c627160 00000400:00000200:15.0:1616513084.842698:0:6509:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-0@lo of length 224/224 into md 0xf97caeb9 [1] + 0 00000400:00000200:19.0:1616513084.842702:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000400:00000200:15.0:1616513084.842703:0:6509:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b27f8 00000400:00000200:15.0:1616513084.842705:0:6509:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513084.842707:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693067725926720 00000100:00000200:15.0:1616513084.842708:0:6509:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:19.0:1616513084.842709:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000400:00000200:15.0:1616513084.842715:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b27f8 00000800:00000200:19.0:1616513084.842716:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00010000:00000200:24.0:1616513084.842717:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1693067725926720/t0(0) o400->4bddedab-bc70-f779-5410-318453d3d68a@0@lo:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:15.0:1616513084.842718:0:6509:0:(lib-msg.c:806:lnet_is_health_check()) msg ffff8d5b18e97c00 failed too early to retry and send 00000800:00000200:19.0:1616513084.842719:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000100:00000200:15.0:1616513084.842720:0:6509:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd604380 x1693067725926752/t0(0) o400->MGC10.242.46.233@o2ib1@0@lo:26/25 lens 224/224 e 0 to 0 dl 1616513177 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:19.0:1616513084.842721:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:19.0:1616513084.842727:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:15.0:1616513084.842731:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b760 00000100:00000200:24.0:1616513084.842732:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693067725926720, offset 192 00000400:00000200:24.0:1616513084.842737:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:24.0:1616513084.842744:0:245737:0:(lib-move.c:4150:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:24.0:1616513084.842750:0:245737:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-0@lo of length 192 into portal 25 MB=0x603d60c627140 00000400:00000200:24.0:1616513084.842756:0:245737:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 192/192 into md 0x866d8101 [1] + 192 00000400:00000200:24.0:1616513084.842760:0:245737:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513084.842764:0:245737:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd604c80 x1693067725926720/t0(0) o400->MGC10.242.62.139@tcp5@0@lo:26/25 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513084.842771:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693067725926752 00000800:00000200:31.2:1616513084.842773:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000400:00000200:24.0:1616513084.842779:0:245737:0:(lib-msg.c:806:lnet_is_health_check()) msg ffff8d5cc479da00 failed too early to retry and send 00000400:00000200:24.0:1616513084.842781:0:245737:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4990 00010000:00000200:16.0:1616513084.842781:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1693067725926752/t0(0) o400->80a7964a-b90a-b048-bdcc-4a7cb892b6fd@0@lo:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:7.0:1616513084.842785:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd604c80 x1693067725926720/t0(0) o400->MGC10.242.62.139@tcp5@0@lo:26/25 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513084.842793:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd604c80 x1693067725926720/t0(0) o400->MGC10.242.62.139@tcp5@0@lo:26/25 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:24.0:1616513084.842795:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:7.0:1616513084.842802:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f7e8 00000100:00000200:16.0:1616513084.842804:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693067725926752, offset 192 00000400:00000200:16.0:1616513084.842809:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-0@lo 00000400:00000200:16.0:1616513084.842815:0:252364:0:(lib-move.c:4150:lnet_parse()) TRACE: 0@lo(0@lo) <- 0@lo : PUT - for me 00000400:00000200:16.0:1616513084.842820:0:252364:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-0@lo of length 192 into portal 25 MB=0x603d60c627160 00000400:00000200:16.0:1616513084.842826:0:252364:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 19 from 12345-0@lo of length 192/192 into md 0x866d8111 [1] + 192 00000400:00000200:16.0:1616513084.842831:0:252364:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513084.842835:0:252364:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd604380 x1693067725926752/t0(0) o400->MGC10.242.46.233@o2ib1@0@lo:26/25 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513084.842837:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:26.0:1616513084.842844:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:26.0:1616513084.842847:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513084.842849:0:252364:0:(lib-msg.c:806:lnet_is_health_check()) msg ffff8d5cdffc9e00 failed too early to retry and send 00000400:00000200:16.0:1616513084.842851:0:252364:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2880 00000400:00000200:26.0:1616513084.842853:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000100:00000200:15.0:1616513084.842857:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd604380 x1693067725926752/t0(0) o400->MGC10.242.46.233@o2ib1@0@lo:26/25 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513084.842859:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600d80 x1693067725926736/t0(0) o400->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:12/10 lens 224/224 e 0 to 0 dl 1616513091 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513084.842859:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:15.0:1616513084.842867:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd604380 x1693067725926752/t0(0) o400->MGC10.242.46.233@o2ib1@0@lo:26/25 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513084.842872:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706dd0 00000800:00000200:26.0:1616513084.842875:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:26.0:1616513084.842878:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000400:00000200:15.0:1616513084.842878:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000800:00000200:31.2:1616513084.842929:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:14.0:1616513084.842982:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:14.0:1616513084.842991:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.234@o2ib1 00000400:00000200:14.0:1616513084.842999:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513084.843006:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 192 into portal 10 MB=0x603d60c627150 00000400:00000200:14.0:1616513084.843013:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.242.46.234@o2ib1 of length 192/192 into md 0xf97cb539 [1] + 192 00000400:00000200:14.0:1616513084.843018:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.843022:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600d80 x1693067725926736/t0(0) o400->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:12/10 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513084.843036:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:14.0:1616513084.843039:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:14.0:1616513084.843041:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:18.0:1616513084.843043:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:19.0:1616513084.843045:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600d80 x1693067725926736/t0(0) o400->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:12/10 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513084.843058:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600d80 x1693067725926736/t0(0) o400->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:12/10 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513084.843069:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000800:00000200:7.2:1616513084.854170:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.854224:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513084.854229:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513084.854234:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.854242:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.90@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.854251:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.90@tcp1 of length 224 into portal 26 MB=0x5f908f7b93200 00000400:00000200:26.0:1616513084.854257:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.90@tcp1 of length 224/224 into md 0xf97caec9 [1] + 0 00000400:00000200:26.0:1616513084.854262:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2330 00000400:00000200:26.0:1616513084.854264:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.854267:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513084.854271:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2330 00000800:00000200:26.0:1616513084.854274:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.854277:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.854278:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.854281:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513084.854299:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.854309:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513084.854309:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.854318:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.90@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.854327:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.90@tcp1 of length 224 into portal 12 MB=0x5f908f7b93280 00000400:00000200:12.0:1616513084.854333:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.90@tcp1 of length 224/224 into md 0x865bbec1 [1] + 74368 00000100:00000200:16.0:1616513084.854338:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681191794717184 00000400:00000200:12.0:1616513084.854338:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.854340:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.854347:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:16.0:1616513084.854349:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1681191794717184/t0(0) o400->b085d14b-da24-9ac1-418d-cc1d564e5a4d@10.31.131.90@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513084.854350:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.854354:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513084.854363:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681191794717184, offset 192 00000400:00000200:16.0:1616513084.854370:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.90@tcp1 00000400:00000200:16.0:1616513084.854377:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.90@tcp1 routed destination 00000400:00000200:16.0:1616513084.854392:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.90@tcp1(10.31.131.90@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.854399:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.854405:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.854407:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.854412:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.854417:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513084.854417:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681191794717312 00000800:00000200:11.2:1616513084.854422:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.854424:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:0.0:1616513084.854429:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d202400 x1681191794717312/t0(0) o400->ba8859bc-f0c1-703c-0f5e-e89d37b8f814@10.31.131.90@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.854459:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681191794717312, offset 192 00000400:00000200:0.0:1616513084.854465:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.90@tcp1 00000400:00000200:0.0:1616513084.854471:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.90@tcp1 routed destination 00000800:00000200:28.0:1616513084.854484:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513084.854484:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.90@tcp1(10.31.131.90@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.854490:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513084.854491:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.854494:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513084.854495:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.854497:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.854499:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513084.854501:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513084.854504:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513084.854506:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b35c8 00000800:00000200:7.2:1616513084.854509:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513084.854510:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.854513:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513084.854564:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.854571:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.854573:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.854579:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.854583:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04198 00000800:00000200:14.0:1616513084.854587:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.854589:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:27.0:1616513084.870578:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:27.0:1616513084.870586:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000100:00000200:31.0:1616513084.874532:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926768, portal 4 00000100:00000200:31.0:1616513084.874538:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926768, offset 0 00000400:00000200:31.0:1616513084.874545:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:31.0:1616513084.874551:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:31.0:1616513084.874564:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513084.874571:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:31.0:1616513084.874578:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513084.874581:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:31.0:1616513084.874583:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:31.0:1616513084.874589:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513084.874635:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513084.874692:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513084.874699:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513084.874702:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.874709:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513084.874718:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded4c80 x1693067725926768/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513084.874729:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c3e48 00000800:00000200:26.0:1616513084.874733:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513084.874736:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513084.874762:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513084.874817:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513084.874826:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513084.874835:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513084.874843:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627170 00000400:00000200:14.0:1616513084.874849:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cb579 [1] + 192 00000400:00000200:14.0:1616513084.874854:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.874859:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded4c80 x1693067725926768/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513084.874871:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513084.874875:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513084.874878:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513084.874880:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:31.0:1616513084.874885:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded4c80 x1693067725926768/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513084.874898:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded4c80 x1693067725926768/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513084.874909:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000100:00000200:27.0:1616513084.890585:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926784, portal 4 00000100:00000200:27.0:1616513084.890590:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926784, offset 0 00000400:00000200:27.0:1616513084.890600:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:27.0:1616513084.890605:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:27.0:1616513084.890619:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513084.890625:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:27.0:1616513084.890632:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513084.890635:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:27.0:1616513084.890637:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:27.0:1616513084.890642:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513084.890690:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513084.890709:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513084.890715:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513084.890717:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513084.890724:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513084.890733:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456e9200 x1693067725926784/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513084.890745:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249ed0 00000800:00000200:1.0:1616513084.890750:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513084.890752:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513084.890877:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513084.890887:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513084.890893:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513084.890897:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513084.890902:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513084.890910:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627180 00000400:00000200:1.0:1616513084.890916:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffd35 [1] + 192 00000400:00000200:1.0:1616513084.890921:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513084.890926:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456e9200 x1693067725926784/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513084.890939:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513084.890942:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513084.890943:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:27.0:1616513084.890947:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456e9200 x1693067725926784/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513084.890960:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456e9200 x1693067725926784/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513084.890970:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248660 00000800:00000200:11.2:1616513084.903493:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513084.903507:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513084.903515:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513084.903525:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.102@tcp1 : PUT - for me 00000400:00000200:8.0:1616513084.903534:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.102@tcp1 of length 224 into portal 26 MB=0x6021712a2b440 00000400:00000200:8.0:1616513084.903541:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.102@tcp1 of length 224/224 into md 0xf97caed9 [1] + 0 00000400:00000200:8.0:1616513084.903546:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3430 00000400:00000200:8.0:1616513084.903549:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.903551:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513084.903556:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3430 00000800:00000200:8.0:1616513084.903559:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.903560:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513084.903562:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513084.903565:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513084.903622:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691147980420160 00010000:00000200:16.0:1616513084.903635:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1691147980420160/t0(0) o400->2a0257cf-a9d8-edd7-828a-f9b153dc40fd@10.31.140.102@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.903650:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691147980420160, offset 192 00000400:00000200:16.0:1616513084.903657:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.102@tcp1 00000400:00000200:16.0:1616513084.903665:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.102@tcp1 routed destination 00000400:00000200:16.0:1616513084.903681:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.102@tcp1(10.31.140.102@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.903688:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.903694:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.903697:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.903699:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.903705:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.903709:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.903712:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513084.903765:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.903771:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.903774:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.903781:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.903789:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b36d8 00000800:00000200:28.0:1616513084.903792:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.903795:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513084.904013:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.904025:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.904033:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513084.904033:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.904042:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.102@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.904050:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.102@tcp1 of length 224 into portal 12 MB=0x6021712a2b700 00000400:00000200:12.0:1616513084.904057:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.102@tcp1 of length 224/224 into md 0x865bbec1 [1] + 74592 00000400:00000200:12.0:1616513084.904062:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.904065:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.904072:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.904075:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.904077:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513084.904138:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691147980420864 00010000:00000200:0.0:1616513084.904152:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d201b00 x1691147980420864/t0(0) o400->d39399ec-1065-5748-935b-e3359d138667@10.31.140.102@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513084.904166:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691147980420864, offset 192 00000400:00000200:0.0:1616513084.904173:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.102@tcp1 00000400:00000200:0.0:1616513084.904180:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.102@tcp1 routed destination 00000400:00000200:0.0:1616513084.904198:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.102@tcp1(10.31.140.102@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.904204:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513084.904211:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.904214:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.904216:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.904222:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.904229:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.904285:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.904292:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.904295:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513084.904302:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.904306:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04000 00000800:00000200:26.0:1616513084.904309:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513084.904312:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:11.0:1616513084.906486:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926800, portal 4 00000100:00000200:11.0:1616513084.906490:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926800, offset 0 00000400:00000200:11.0:1616513084.906496:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:11.0:1616513084.906502:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:11.0:1616513084.906513:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513084.906519:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:11.0:1616513084.906524:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513084.906527:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:11.0:1616513084.906529:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:11.0:1616513084.906534:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513084.906579:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513084.906636:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513084.906643:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513084.906647:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.906653:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513084.906660:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8f30a8b180 x1693067725926800/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513084.906672:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3ac38 00000800:00000200:28.0:1616513084.906676:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513084.906679:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513084.906753:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513084.906762:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513084.906766:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513084.906773:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513084.906778:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627190 00000400:00000200:8.0:1616513084.906784:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d8181 [1] + 192 00000400:00000200:8.0:1616513084.906788:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513084.906792:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8f30a8b180 x1693067725926800/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513084.906805:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513084.906807:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:8.0:1616513084.906808:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513084.906811:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:11.0:1616513084.906812:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8f30a8b180 x1693067725926800/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513084.906834:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8f30a8b180 x1693067725926800/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513084.906843:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bc28 00000800:00000200:7.2:1616513084.911019:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513084.911038:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513084.911048:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513084.911057:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.70@tcp1 : PUT - for me 00000400:00000200:18.0:1616513084.911066:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.70@tcp1 of length 224 into portal 26 MB=0x603ce39aaa700 00000400:00000200:18.0:1616513084.911073:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.70@tcp1 of length 224/224 into md 0xf97caef9 [1] + 0 00000400:00000200:18.0:1616513084.911077:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2220 00000400:00000200:18.0:1616513084.911080:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513084.911083:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513084.911087:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2220 00000800:00000200:18.0:1616513084.911091:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.911092:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.911093:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513084.911096:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513084.911152:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693034125895424 00010000:00000200:16.0:1616513084.911164:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1693034125895424/t0(0) o400->faaa96ec-7d4b-b447-6395-4a3d9478bc70@10.31.136.70@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.911179:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693034125895424, offset 192 00000400:00000200:16.0:1616513084.911185:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.70@tcp1 00000400:00000200:16.0:1616513084.911193:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.70@tcp1 routed destination 00000400:00000200:16.0:1616513084.911208:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.70@tcp1(10.31.136.70@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.911215:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.911222:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.911224:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.911227:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.911232:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513084.911238:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.911239:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.911293:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.911300:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513084.911302:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.911309:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.911314:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b23b8 00000800:00000200:28.0:1616513084.911318:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.911321:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.911381:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.911447:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.911456:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.911465:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.70@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.911474:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.70@tcp1 of length 224 into portal 12 MB=0x603ce39aaa840 00000400:00000200:26.0:1616513084.911480:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.70@tcp1 of length 224/224 into md 0xf964d519 [1] + 39976 00000400:00000200:26.0:1616513084.911486:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.911488:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513084.911497:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.911500:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.911501:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.911505:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.911567:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693034125895744 00010000:00000200:22.0:1616513084.911580:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b27c800 x1693034125895744/t0(0) o400->bd382f90-46ca-2d28-27b3-14b3183a7ad0@10.31.136.70@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.911594:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693034125895744, offset 192 00000400:00000200:22.0:1616513084.911603:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.70@tcp1 00000400:00000200:22.0:1616513084.911610:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.70@tcp1 routed destination 00000400:00000200:22.0:1616513084.911625:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.70@tcp1(10.31.136.70@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.911632:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513084.911638:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.911640:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.911642:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.911647:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.911653:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.911664:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.911670:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.911673:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513084.911674:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513084.911679:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.911683:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a880 00000800:00000200:14.0:1616513084.911687:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.911861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513084.911917:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.911928:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.911930:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513084.911937:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.13@tcp1 : PUT - for me 00000400:00000200:12.0:1616513084.911947:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.13@tcp1 of length 224 into portal 12 MB=0x5ffde44f2c480 00000400:00000200:12.0:1616513084.911953:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.13@tcp1 of length 224/224 into md 0x865bbec1 [1] + 74816 00000400:00000200:12.0:1616513084.911958:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513084.911961:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513084.911967:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513084.911970:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.911972:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.912031:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513084.912031:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704988136576 00000800:00000200:2.0:1616513084.912043:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513084.912044:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d204c80 x1688704988136576/t0(0) o400->0d1f0a3e-fe1c-0733-8cfd-443ee18fc76f@10.31.140.13@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513084.912051:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513084.912052:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513084.912058:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704988136576, offset 192 00000400:00000200:2.0:1616513084.912060:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.13@tcp1 : PUT - for me 00000400:00000200:0.0:1616513084.912065:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.13@tcp1 00000400:00000200:2.0:1616513084.912069:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.13@tcp1 of length 224 into portal 26 MB=0x5ffde44f2c200 00000400:00000200:0.0:1616513084.912073:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.13@tcp1 routed destination 00000400:00000200:2.0:1616513084.912075:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.13@tcp1 of length 224/224 into md 0xf97caf19 [1] + 0 00000400:00000200:2.0:1616513084.912080:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3870 00000400:00000200:2.0:1616513084.912082:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.912085:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513084.912087:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.13@tcp1(10.31.140.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513084.912090:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3870 00000800:00000200:2.0:1616513084.912093:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.912094:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513084.912096:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.912097:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513084.912100:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:16.0:1616513084.912102:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704988135936 00000800:00000200:0.0:1616513084.912102:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513084.912104:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513084.912109:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:16.0:1616513084.912110:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688704988135936/t0(0) o400->4be958e1-8f86-cef9-1a3e-86d4789bc137@10.31.140.13@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513084.912114:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.912121:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704988135936, offset 192 00000800:00000200:28.0:1616513084.912124:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513084.912127:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513084.912127:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.13@tcp1 00000400:00000200:28.0:1616513084.912129:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513084.912133:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513084.912133:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.13@tcp1 routed destination 00000400:00000200:28.0:1616513084.912134:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513084.912137:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:28.0:1616513084.912140:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:16.0:1616513084.912146:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.13@tcp1(10.31.140.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.912152:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513084.912157:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.912160:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513084.912161:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513084.912165:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.912169:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513084.912170:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513084.912178:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513084.912184:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513084.912186:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513084.912188:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513084.912192:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513084.912197:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b37e8 00000800:00000200:18.0:1616513084.912200:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.913590:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.913644:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.913653:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.913662:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.31@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.913671:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.31@tcp1 of length 224 into portal 26 MB=0x6041a0ab4a2c0 00000400:00000200:26.0:1616513084.913678:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.31@tcp1 of length 224/224 into md 0xf97caf49 [1] + 0 00000400:00000200:26.0:1616513084.913683:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf37e8 00000400:00000200:26.0:1616513084.913686:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.913688:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513084.913692:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf37e8 00000800:00000200:26.0:1616513084.913697:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.913700:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.913702:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.913704:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513084.913755:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359755535040 00010000:00000200:16.0:1616513084.913766:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1693359755535040/t0(0) o400->051eb6c7-59f4-6251-60e1-7114b2fcda51@10.31.129.31@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.913780:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693359755535040, offset 192 00000400:00000200:16.0:1616513084.913786:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.31@tcp1 00000400:00000200:16.0:1616513084.913794:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.31@tcp1 routed destination 00000400:00000200:16.0:1616513084.913809:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.31@tcp1(10.31.129.31@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.913816:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.913821:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.913824:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.913826:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.913831:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513084.913837:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513084.913837:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.913853:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513084.913857:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513084.913859:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513084.913865:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513084.913868:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3f58 00000800:00000200:16.0:1616513084.913871:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513084.913875:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.913897:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.913951:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.913960:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.913969:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.31@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.913978:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.31@tcp1 of length 224 into portal 12 MB=0x6041a0ab4a700 00000400:00000200:14.0:1616513084.913984:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.31@tcp1 of length 224/224 into md 0xf964d519 [1] + 40200 00000400:00000200:14.0:1616513084.913994:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.913997:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.914003:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.914006:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.914008:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.914011:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.914066:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359755536128 00010000:00000200:22.0:1616513084.914078:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464627500 x1693359755536128/t0(0) o400->cb533122-96a0-7b26-cd5f-17dbcfd78aa9@10.31.129.31@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.914092:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693359755536128, offset 192 00000400:00000200:22.0:1616513084.914098:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.31@tcp1 00000400:00000200:22.0:1616513084.914106:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.31@tcp1 routed destination 00000400:00000200:22.0:1616513084.914120:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.31@tcp1(10.31.129.31@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.914127:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513084.914132:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.914135:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.914137:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.914142:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.914147:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.914155:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.914160:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513084.914162:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513084.914164:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513084.914167:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513084.914170:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b980 00000800:00000200:26.0:1616513084.914173:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513084.952826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513084.952885:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513084.952895:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513084.952905:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.211@tcp1 : PUT - for me 00000400:00000200:20.0:1616513084.952915:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.211@tcp1 of length 224 into portal 26 MB=0x5ffe1da32c280 00000400:00000200:20.0:1616513084.952922:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.211@tcp1 of length 224/224 into md 0xf97caf59 [1] + 0 00000400:00000200:20.0:1616513084.952927:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3ed0 00000400:00000200:20.0:1616513084.952930:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.952933:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513084.952938:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ed0 00000800:00000200:28.0:1616513084.952940:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.952941:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513084.952944:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513084.952948:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513084.952951:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.952957:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513084.952966:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.211@tcp1 : PUT - for me 00000400:00000200:28.0:1616513084.952976:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.211@tcp1 of length 224 into portal 12 MB=0x5ffe1da32c300 00000400:00000200:28.0:1616513084.952982:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.211@tcp1 of length 224/224 into md 0x865bbec1 [1] + 75040 00000400:00000200:28.0:1616513084.952988:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513084.952990:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513084.952998:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.953003:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.953006:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513084.953014:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688720377037440 00010000:00000200:16.0:1616513084.953026:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688720377037440/t0(0) o400->05255509-50e2-52de-2f8c-3f7419b39acf@10.31.142.211@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.953040:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688720377037440, offset 192 00000400:00000200:16.0:1616513084.953046:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.211@tcp1 00000400:00000200:16.0:1616513084.953053:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.211@tcp1 routed destination 00000400:00000200:16.0:1616513084.953068:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.211@tcp1(10.31.142.211@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.953075:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.953080:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.953083:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513084.953084:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688720377037568 00000800:00000200:16.0:1616513084.953085:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.953090:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.953096:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.953097:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:0.0:1616513084.953097:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ef500 x1688720377037568/t0(0) o400->47a5b412-03e4-d3b1-2e9b-2580d2f5d660@10.31.142.211@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513084.953112:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513084.953112:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688720377037568, offset 192 00000400:00000200:0.0:1616513084.953118:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.211@tcp1 00000800:00000200:12.0:1616513084.953119:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.953122:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513084.953123:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513084.953126:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.211@tcp1 routed destination 00000400:00000200:12.0:1616513084.953128:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513084.953134:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3f58 00000800:00000200:12.0:1616513084.953137:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513084.953142:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.211@tcp1(10.31.142.211@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513084.953148:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513084.953155:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513084.953158:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513084.953161:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513084.953167:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.953175:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.953232:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.953240:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.953243:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.953250:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.953254:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb047f8 00000800:00000200:14.0:1616513084.953258:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.953261:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513084.954527:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926816, portal 4 00000100:00000200:26.0:1616513084.954533:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926816, offset 0 00000400:00000200:26.0:1616513084.954539:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:26.0:1616513084.954545:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:26.0:1616513084.954558:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513084.954564:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:26.0:1616513084.954570:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513084.954573:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:26.0:1616513084.954575:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:26.0:1616513084.954580:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513084.954626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513084.954682:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513084.954689:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513084.954692:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513084.954698:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513084.954704:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf85e80 x1693067725926816/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513084.954717:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af4b8 00000800:00000200:28.0:1616513084.954721:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513084.954724:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513084.954816:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513084.954870:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513084.954880:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513084.954890:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513084.954897:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6271a0 00000400:00000200:20.0:1616513084.954904:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d81f1 [1] + 192 00000400:00000200:20.0:1616513084.954909:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513084.954914:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf85e80 x1693067725926816/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513084.954929:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513084.954932:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:20.0:1616513084.954934:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:26.0:1616513084.954937:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf85e80 x1693067725926816/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513084.954937:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:26.0:1616513084.954949:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf85e80 x1693067725926816/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513091 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513084.954958:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa08 00000800:00000200:11.2:1616513084.986738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513084.986794:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513084.986804:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513084.986814:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.40@tcp1 : PUT - for me 00000400:00000200:28.0:1616513084.986824:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.40@tcp1 of length 224 into portal 26 MB=0x5ffdeacb345c0 00000400:00000200:28.0:1616513084.986831:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.40@tcp1 of length 224/224 into md 0xf97caf69 [1] + 0 00000400:00000200:28.0:1616513084.986836:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3ba0 00000400:00000200:28.0:1616513084.986838:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513084.986841:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513084.986846:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ba0 00000800:00000200:28.0:1616513084.986849:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513084.986850:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513084.986852:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513084.986854:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513084.986913:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706728805824 00010000:00000200:16.0:1616513084.986926:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688706728805824/t0(0) o400->1ab930a0-6f20-82f0-27bc-e3b9a5d8d277@10.31.140.40@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.986940:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706728805824, offset 192 00000400:00000200:16.0:1616513084.986947:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.40@tcp1 00000400:00000200:16.0:1616513084.986955:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.40@tcp1 routed destination 00000400:00000200:16.0:1616513084.986971:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.40@tcp1(10.31.140.40@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.986977:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.986983:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.986986:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.986988:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.986993:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.986999:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.987001:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513084.987005:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513084.987052:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513084.987060:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513084.987062:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513084.987069:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513084.987074:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513084.987075:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3cb0 00000800:00000200:12.0:1616513084.987080:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513084.987083:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513084.987083:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513084.987092:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.40@tcp1 : PUT - for me 00000400:00000200:2.0:1616513084.987101:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.40@tcp1 of length 224 into portal 12 MB=0x5ffdeacb34680 00000400:00000200:2.0:1616513084.987108:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.40@tcp1 of length 224/224 into md 0xf964d519 [1] + 40424 00000400:00000200:2.0:1616513084.987114:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513084.987116:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513084.987123:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.987127:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.987128:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513084.987131:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.987190:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706728806016 00010000:00000200:22.0:1616513084.987204:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9200 x1688706728806016/t0(0) o400->db15e613-e214-2bff-620b-5fecfa40ff1e@10.31.140.40@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.987218:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706728806016, offset 192 00000400:00000200:22.0:1616513084.987225:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.40@tcp1 00000400:00000200:22.0:1616513084.987232:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.40@tcp1 routed destination 00000400:00000200:22.0:1616513084.987247:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.40@tcp1(10.31.140.40@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.987253:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513084.987260:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.987265:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.987267:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.987272:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.987278:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.987334:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.987342:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513084.987345:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513084.987351:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513084.987357:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b870 00000800:00000200:14.0:1616513084.987361:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513084.987364:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513084.995557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513084.995612:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513084.995621:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513084.995630:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.83@tcp1 : PUT - for me 00000400:00000200:26.0:1616513084.995638:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.83@tcp1 of length 224 into portal 26 MB=0x5ffd4d3ec8680 00000400:00000200:26.0:1616513084.995644:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.83@tcp1 of length 224/224 into md 0xf97caf79 [1] + 0 00000400:00000200:26.0:1616513084.995648:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3d38 00000400:00000200:26.0:1616513084.995651:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513084.995653:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513084.995659:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3d38 00000800:00000200:26.0:1616513084.995663:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513084.995665:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513084.995665:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513084.995670:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513084.995724:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664437196416 00010000:00000200:16.0:1616513084.995736:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688664437196416/t0(0) o400->f952ae3d-868d-f2e7-27d5-e838dfa52bfe@10.31.136.83@tcp1:41/0 lens 224/224 e 0 to 0 dl 1616513176 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513084.995750:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664437196416, offset 192 00000400:00000200:16.0:1616513084.995758:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.83@tcp1 00000400:00000200:16.0:1616513084.995765:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.83@tcp1 routed destination 00000400:00000200:16.0:1616513084.995780:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.83@tcp1(10.31.136.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513084.995787:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513084.995792:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513084.995795:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513084.995797:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513084.995802:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513084.995807:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513084.995809:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513084.995864:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513084.995873:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513084.995876:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513084.995882:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513084.995888:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3078 00000800:00000200:24.0:1616513084.995891:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513084.995895:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513084.995945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513084.995998:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513084.996008:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513084.996016:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.83@tcp1 : PUT - for me 00000400:00000200:14.0:1616513084.996025:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.83@tcp1 of length 224 into portal 12 MB=0x5ffd4d3ec8780 00000400:00000200:14.0:1616513084.996031:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.83@tcp1 of length 224/224 into md 0xf964d519 [1] + 40648 00000400:00000200:14.0:1616513084.996037:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513084.996039:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513084.996050:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513084.996054:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513084.996056:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513084.996058:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513084.996112:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664437196672 00010000:00000200:22.0:1616513084.996124:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621200 x1688664437196672/t0(0) o400->1566879a-2f7b-a44c-0cf0-20240a774ca4@10.31.136.83@tcp1:710/0 lens 224/224 e 0 to 0 dl 1616513090 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513084.996137:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664437196672, offset 192 00000400:00000200:22.0:1616513084.996143:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.83@tcp1 00000400:00000200:22.0:1616513084.996150:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.83@tcp1 routed destination 00000400:00000200:22.0:1616513084.996163:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.83@tcp1(10.31.136.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513084.996169:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513084.996175:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513084.996177:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513084.996179:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513084.996184:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513084.996189:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513084.996200:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513084.996207:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513084.996209:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513084.996210:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513084.996216:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513084.996220:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a550 00000800:00000200:2.0:1616513084.996224:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.002722:0:49989:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513085.002776:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513085.002778:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513085.002785:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.002794:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.65@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.002803:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.65@tcp1 of length 224 into portal 26 MB=0x6018349abab40 00000400:00000200:14.0:1616513085.002810:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.65@tcp1 of length 224/224 into md 0xf97caf89 [1] + 0 00000400:00000200:14.0:1616513085.002815:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2a18 00000400:00000200:14.0:1616513085.002817:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513085.002820:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513085.002825:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2a18 00000800:00000200:14.0:1616513085.002827:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.002830:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.002830:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513085.002834:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513085.002851:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.002861:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.002870:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.65@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.002878:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.65@tcp1 of length 224 into portal 12 MB=0x6018349abab80 00000400:00000200:12.0:1616513085.002885:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.65@tcp1 of length 224/224 into md 0x865bbec1 [1] + 75264 00000400:00000200:12.0:1616513085.002897:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.002899:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:16.0:1616513085.002904:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513248594752 00000800:00000200:12.0:1616513085.002906:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.002909:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.002911:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.002914:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:16.0:1616513085.002915:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1690513248594752/t0(0) o400->5ea9221c-531f-7f94-2b74-32b71112ec7e@10.31.131.65@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.002929:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513248594752, offset 192 00000400:00000200:16.0:1616513085.002936:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.65@tcp1 00000400:00000200:16.0:1616513085.002943:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.65@tcp1 routed destination 00000400:00000200:16.0:1616513085.002958:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.65@tcp1(10.31.131.65@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.002964:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.002970:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.002973:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.002975:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.002980:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.002985:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513085.002986:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513248594816 00000100:00000200:16.0:1616513085.002987:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:0.0:1616513085.002998:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d200000 x1690513248594816/t0(0) o400->66084537-93de-2afb-2d43-05b2c9e50684@10.31.131.65@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.003011:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513248594816, offset 192 00000400:00000200:0.0:1616513085.003017:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.65@tcp1 00000400:00000200:0.0:1616513085.003024:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.65@tcp1 routed destination 00000400:00000200:0.0:1616513085.003038:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.65@tcp1(10.31.131.65@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513085.003040:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513085.003044:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513085.003046:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513085.003049:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513085.003049:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.003052:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.003054:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513085.003056:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513085.003059:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513085.003061:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3100 00000800:00000200:7.2:1616513085.003063:0:49989:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513085.003065:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.003068:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513085.003118:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.003125:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.003127:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.003133:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.003137:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:26.0:1616513085.003141:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.003144:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.019238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513085.019293:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.019303:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.019311:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.62@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.019321:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.62@tcp1 of length 224 into portal 26 MB=0x601830db33e40 00000400:00000200:18.0:1616513085.019328:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.62@tcp1 of length 224/224 into md 0xf97cafa9 [1] + 0 00000400:00000200:18.0:1616513085.019332:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2550 00000400:00000200:18.0:1616513085.019335:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.019338:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513085.019342:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2550 00000800:00000200:18.0:1616513085.019345:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.019345:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.019348:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.019350:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.019404:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512242458176 00010000:00000200:16.0:1616513085.019414:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1690512242458176/t0(0) o400->e8137361-c011-a934-ccc6-2e2693e09702@10.31.138.62@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.019428:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512242458176, offset 192 00000400:00000200:16.0:1616513085.019447:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.62@tcp1 00000400:00000200:16.0:1616513085.019455:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.62@tcp1 routed destination 00000400:00000200:16.0:1616513085.019468:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.62@tcp1(10.31.138.62@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.019474:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.019481:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.019483:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.019485:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.019490:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513085.019497:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513085.019535:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.019590:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.019597:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.019599:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513085.019606:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.019612:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3100 00000800:00000200:12.0:1616513085.019615:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.019618:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.019635:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513085.019645:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.019659:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513085.019668:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.62@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.019677:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.62@tcp1 of length 224 into portal 12 MB=0x601830db34000 00000400:00000200:20.0:1616513085.019683:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.62@tcp1 of length 224/224 into md 0x865bbec1 [1] + 75488 00000400:00000200:20.0:1616513085.019688:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.019691:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513085.019697:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.019700:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.019702:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.019705:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.019762:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512242458624 00010000:00000200:0.0:1616513085.019774:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceba80 x1690512242458624/t0(0) o400->292be5d4-6f83-d741-790c-9eb6a102c6b7@10.31.138.62@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.019789:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512242458624, offset 192 00000400:00000200:0.0:1616513085.019795:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.62@tcp1 00000400:00000200:0.0:1616513085.019803:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.62@tcp1 routed destination 00000400:00000200:0.0:1616513085.019818:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.62@tcp1(10.31.138.62@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.019824:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.019830:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.019833:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.019835:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.019840:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.019845:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.019897:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.019903:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.019905:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.019911:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.019914:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05430 00000800:00000200:26.0:1616513085.019918:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.019921:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.028718:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513085.028773:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.028783:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.028792:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.79@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.028800:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.79@tcp1 of length 224 into portal 26 MB=0x60188948be480 00000400:00000200:14.0:1616513085.028807:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.79@tcp1 of length 224/224 into md 0xf97cafc9 [1] + 0 00000400:00000200:14.0:1616513085.028812:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2d48 00000400:00000200:14.0:1616513085.028816:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513085.028819:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513085.028823:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2d48 00000800:00000200:18.0:1616513085.028826:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.028826:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.028829:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513085.028831:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.028887:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690535979639936 00000800:00000200:11.2:1616513085.028895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:16.0:1616513085.028898:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1690535979639936/t0(0) o400->2bd142fc-b1dc-1c39-29c3-b60e8122e7f6@10.31.132.79@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.028912:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690535979639936, offset 192 00000400:00000200:16.0:1616513085.028919:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.79@tcp1 00000400:00000200:16.0:1616513085.028927:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.79@tcp1 routed destination 00000400:00000200:16.0:1616513085.028941:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.79@tcp1(10.31.132.79@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.028948:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.028954:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513085.028954:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.028957:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.028959:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513085.028964:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513085.028964:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:16.0:1616513085.028970:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513085.028972:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.79@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.028981:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.79@tcp1 of length 224 into portal 12 MB=0x60188948be880 00000400:00000200:12.0:1616513085.028987:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.79@tcp1 of length 224/224 into md 0x865bbec1 [1] + 75712 00000400:00000200:12.0:1616513085.028992:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.028995:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513085.029001:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513085.029004:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513085.029006:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.029009:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.029017:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.029020:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.029027:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.029032:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3cb0 00000800:00000200:20.0:1616513085.029036:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.029039:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.029064:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690535979640960 00010000:00000200:0.0:1616513085.029076:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d207080 x1690535979640960/t0(0) o400->25fba9c9-d035-e2d8-049e-a10e695afdbb@10.31.132.79@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.029090:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690535979640960, offset 192 00000400:00000200:0.0:1616513085.029098:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.79@tcp1 00000400:00000200:0.0:1616513085.029104:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.79@tcp1 routed destination 00000400:00000200:0.0:1616513085.029118:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.79@tcp1(10.31.132.79@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.029124:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.029129:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.029132:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.029134:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.029139:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.029144:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.029198:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.029205:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.029208:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.029215:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.029219:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04b28 00000800:00000200:26.0:1616513085.029223:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.029226:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513085.034533:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926832, portal 4 00000100:00000200:2.0:1616513085.034539:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926832, offset 0 00000400:00000200:2.0:1616513085.034545:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:2.0:1616513085.034551:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:2.0:1616513085.034564:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513085.034570:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:2.0:1616513085.034576:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513085.034580:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:2.0:1616513085.034582:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:2.0:1616513085.034588:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513085.034636:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513085.034658:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513085.034664:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513085.034667:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513085.034674:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513085.034680:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bdde80 x1693067725926832/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513085.034693:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed078 00000800:00000200:27.0:1616513085.034698:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513085.034701:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513085.034761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513085.034771:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513085.034778:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513085.034786:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513085.034794:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6271b0 00000400:00000200:27.0:1616513085.034800:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166e1d [1] + 192 00000400:00000200:27.0:1616513085.034806:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513085.034810:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bdde80 x1693067725926832/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513085.034824:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513085.034825:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513085.034830:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513085.034832:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:2.0:1616513085.034836:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bdde80 x1693067725926832/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513085.034849:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bdde80 x1693067725926832/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513085.034860:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec440 00000800:00000200:11.2:1616513085.055051:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513085.055108:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.055118:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.055127:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.146@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.055137:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.146@tcp1 of length 224 into portal 26 MB=0x5ffd6ff675a40 00000400:00000200:24.0:1616513085.055143:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.146@tcp1 of length 224/224 into md 0xf97cafd9 [1] + 0 00000400:00000200:24.0:1616513085.055148:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf27f8 00000400:00000200:24.0:1616513085.055151:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.055153:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513085.055158:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf27f8 00000800:00000200:24.0:1616513085.055161:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.055163:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.055164:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.055167:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.055222:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673756600896 00010000:00000200:16.0:1616513085.055233:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688673756600896/t0(0) o400->1ce1064e-871d-4905-c63e-66201011a2b6@10.31.129.146@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.055247:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688673756600896, offset 192 00000400:00000200:16.0:1616513085.055254:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.146@tcp1 00000400:00000200:16.0:1616513085.055261:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.146@tcp1 routed destination 00000400:00000200:16.0:1616513085.055276:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.146@tcp1(10.31.129.146@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.055283:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.055289:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.055291:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.055293:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.055298:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.055304:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.055307:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513085.055372:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.055378:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.055381:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.055388:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.055395:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b37e8 00000800:00000200:20.0:1616513085.055399:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.055402:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.055408:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513085.055410:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513085.055415:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.146@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.055424:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.146@tcp1 of length 224 into portal 12 MB=0x5ffd6ff675e80 00000400:00000200:20.0:1616513085.055430:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.146@tcp1 of length 224/224 into md 0x865bbec1 [1] + 75936 00000400:00000200:20.0:1616513085.055458:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.055460:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513085.055467:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.055470:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.055472:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.055530:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673756601984 00010000:00000200:0.0:1616513085.055542:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cee780 x1688673756601984/t0(0) o400->645dc6ff-e79d-057a-8c5c-462f8e75bac1@10.31.129.146@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.055556:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688673756601984, offset 192 00000400:00000200:0.0:1616513085.055562:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.146@tcp1 00000400:00000200:0.0:1616513085.055569:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.146@tcp1 routed destination 00000400:00000200:0.0:1616513085.055582:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.146@tcp1(10.31.129.146@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.055589:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.055594:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.055597:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.055599:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.055603:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.055610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.055665:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.055671:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.055674:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513085.055681:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.055685:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04088 00000800:00000200:18.0:1616513085.055689:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.055692:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513085.066602:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926848, portal 4 00000100:00000200:6.0:1616513085.066606:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926848, offset 0 00000400:00000200:6.0:1616513085.066611:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513085.066615:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:6.0:1616513085.066624:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513085.066628:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:6.0:1616513085.066632:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513085.066634:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:6.0:1616513085.066636:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:6.0:1616513085.066640:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513085.066687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513085.066743:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513085.066749:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513085.066752:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513085.066755:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:26.0:1616513085.066758:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513085.066763:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc4eb9b00 x1693067725926848/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513085.066774:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01540 00000800:00000200:26.0:1616513085.066778:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513085.066854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513085.066866:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513085.066875:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:14.0:1616513085.066880:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513085.066884:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513085.066892:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6271c0 00000400:00000200:18.0:1616513085.066898:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cb699 [1] + 192 00000400:00000200:18.0:1616513085.066903:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.066910:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc4eb9b00 x1693067725926848/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513085.066925:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513085.066928:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:6.0:1616513085.066928:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc4eb9b00 x1693067725926848/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513085.066930:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:6.0:1616513085.066937:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc4eb9b00 x1693067725926848/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513085.066944:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:11.2:1616513085.082008:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513085.082066:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.082076:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.082086:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.101@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.082095:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.101@tcp1 of length 224 into portal 26 MB=0x5ffd2f5703f00 00000400:00000200:24.0:1616513085.082102:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.101@tcp1 of length 224/224 into md 0xf97cafe9 [1] + 0 00000400:00000200:24.0:1616513085.082107:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2dd0 00000400:00000200:24.0:1616513085.082110:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.082113:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513085.082120:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000800:00000200:12.0:1616513085.082122:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.082123:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.082126:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.082128:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.082129:0:49997:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513085.082186:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656409542400 00000800:00000200:2.0:1616513085.082192:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:16.0:1616513085.082198:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688656409542400/t0(0) o400->47469bbc-e5e2-2b06-a8bb-64d0b91a3a5c@10.31.128.101@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513085.082202:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.082211:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.101@tcp1 : PUT - for me 00000100:00000200:16.0:1616513085.082213:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688656409542400, offset 192 00000400:00000200:16.0:1616513085.082219:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.101@tcp1 00000400:00000200:2.0:1616513085.082220:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.101@tcp1 of length 224 into portal 12 MB=0x5ffd2f5704300 00000400:00000200:16.0:1616513085.082227:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.101@tcp1 routed destination 00000400:00000200:2.0:1616513085.082227:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.101@tcp1 of length 224/224 into md 0xf964d519 [1] + 40872 00000400:00000200:2.0:1616513085.082232:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.082235:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513085.082242:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.101@tcp1(10.31.128.101@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513085.082243:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.082246:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.082247:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513085.082249:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513085.082253:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513085.082255:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.082257:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.082260:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.082265:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.082270:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.082272:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513085.082309:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656409543424 00010000:00000200:22.0:1616513085.082320:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb8d80 x1688656409543424/t0(0) o400->5e3e1a94-75c4-d6b2-6d40-2d82ff9e7e15@10.31.128.101@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513085.082325:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.082332:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:22.0:1616513085.082334:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688656409543424, offset 192 00000400:00000200:28.0:1616513085.082335:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513085.082340:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.101@tcp1 00000400:00000200:28.0:1616513085.082341:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.082347:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2220 00000400:00000200:22.0:1616513085.082347:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.101@tcp1 routed destination 00000800:00000200:28.0:1616513085.082351:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.082354:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513085.082360:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.101@tcp1(10.31.128.101@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.082366:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.082372:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.082374:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.082376:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.082381:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.082386:0:49997:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.082452:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.082464:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.082467:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513085.082473:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.082477:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba90 00000800:00000200:14.0:1616513085.082481:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.082484:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.088161:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513085.088192:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513085.088217:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.088226:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.088235:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.123@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.088244:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.123@tcp1 of length 224 into portal 26 MB=0x603ff6ee17200 00000800:00000200:12.0:1616513085.088248:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513085.088251:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.123@tcp1 of length 224/224 into md 0xf97caff9 [1] + 0 00000400:00000200:26.0:1616513085.088255:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3298 00000800:00000200:12.0:1616513085.088257:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513085.088258:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.088261:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513085.088265:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3298 00000400:00000200:12.0:1616513085.088266:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.123@tcp1 : PUT - for me 00000800:00000200:26.0:1616513085.088268:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.088271:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.088272:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513085.088275:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.123@tcp1 of length 224 into portal 12 MB=0x603ff6ee17540 00000800:00000200:2.0:1616513085.088275:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513085.088281:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.123@tcp1 of length 224/224 into md 0x865bbec1 [1] + 76160 00000400:00000200:12.0:1616513085.088286:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.088289:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513085.088296:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.088299:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.088300:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.088303:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.088335:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245472076288 00010000:00000200:16.0:1616513085.088345:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1693245472076288/t0(0) o400->ae652b7c-0ff2-86a0-7525-de7b26fe2fa0@10.31.140.123@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.088360:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245472076288, offset 192 00000400:00000200:16.0:1616513085.088366:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.123@tcp1 00000100:00000200:0.0:1616513085.088366:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245472077120 00000400:00000200:16.0:1616513085.088374:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.123@tcp1 routed destination 00010000:00000200:0.0:1616513085.088378:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d206c00 x1693245472077120/t0(0) o400->490a6ac5-52b5-108a-21ae-40240c6bc72f@10.31.140.123@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513085.088389:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.123@tcp1(10.31.140.123@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513085.088393:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245472077120, offset 192 00000800:00000200:16.0:1616513085.088395:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513085.088400:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.123@tcp1 00000800:00000200:16.0:1616513085.088401:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.088404:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.088406:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513085.088408:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.123@tcp1 routed destination 00000800:00000200:16.0:1616513085.088411:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.088415:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.088418:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513085.088423:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.123@tcp1(10.31.140.123@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.088430:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.088451:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.088456:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.088458:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.088463:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.088467:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513085.088469:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.088476:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.088478:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513085.088485:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.088492:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3430 00000800:00000200:28.0:1616513085.088496:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.088499:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513085.088521:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.088528:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.088530:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513085.088537:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.088541:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:14.0:1616513085.088545:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.088548:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.102190:0:49999:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.102246:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.102256:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.102265:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.44@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.102275:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.44@tcp1 of length 224 into portal 26 MB=0x5ffd498de0300 00000400:00000200:26.0:1616513085.102282:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.44@tcp1 of length 224/224 into md 0xf97cb009 [1] + 0 00000400:00000200:26.0:1616513085.102287:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2b28 00000400:00000200:26.0:1616513085.102290:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.102292:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513085.102297:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2b28 00000800:00000200:26.0:1616513085.102300:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.102303:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.102304:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513085.102311:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.102314:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.102315:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.102323:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.44@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.102332:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.44@tcp1 of length 224 into portal 12 MB=0x5ffd498de0380 00000400:00000200:2.0:1616513085.102339:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.44@tcp1 of length 224/224 into md 0xf964d519 [1] + 41096 00000400:00000200:2.0:1616513085.102344:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.102347:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513085.102354:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.102357:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.102359:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.102361:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663446389504 00010000:00000200:16.0:1616513085.102372:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688663446389504/t0(0) o400->208dc24b-9c52-72be-63fb-6c0ceb83342e@10.31.136.44@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.102387:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663446389504, offset 192 00000400:00000200:16.0:1616513085.102395:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.44@tcp1 00000400:00000200:16.0:1616513085.102402:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.44@tcp1 routed destination 00000400:00000200:16.0:1616513085.102417:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.44@tcp1(10.31.136.44@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513085.102419:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663446389632 00000800:00000200:16.0:1616513085.102423:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.102429:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513085.102432:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebad00 x1688663446389632/t0(0) o400->efb32473-39ef-d786-9741-a39ab5787a8b@10.31.136.44@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.102432:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.102448:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.102453:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.102460:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.102461:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513085.102476:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663446389632, offset 192 00000400:00000200:22.0:1616513085.102483:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.44@tcp1 00000400:00000200:22.0:1616513085.102490:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.44@tcp1 routed destination 00000400:00000200:22.0:1616513085.102504:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.44@tcp1(10.31.136.44@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.102511:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513085.102516:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513085.102517:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.102520:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.102522:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513085.102522:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.102525:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513085.102526:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.102531:0:49999:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513085.102532:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.102537:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2330 00000800:00000200:12.0:1616513085.102541:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.102544:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513085.102595:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.102602:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.102605:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513085.102606:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.102611:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.102615:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a000 00000800:00000200:18.0:1616513085.102618:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.107419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.107474:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.107483:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.107492:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.191@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.107501:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.191@tcp1 of length 224 into portal 26 MB=0x602183647c340 00000400:00000200:2.0:1616513085.107507:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.191@tcp1 of length 224/224 into md 0xf97cb019 [1] + 0 00000400:00000200:2.0:1616513085.107512:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf24c8 00000400:00000200:2.0:1616513085.107515:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.107517:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.107521:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf24c8 00000800:00000200:2.0:1616513085.107524:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.107527:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.107527:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.107531:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.107600:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691152873407296 00010000:00000200:16.0:1616513085.107610:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1691152873407296/t0(0) o400->ec682b7a-7eaf-6d3b-3944-5a99d0d02ff3@10.31.140.191@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.107623:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691152873407296, offset 192 00000400:00000200:16.0:1616513085.107629:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.191@tcp1 00000400:00000200:16.0:1616513085.107636:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.191@tcp1 routed destination 00000400:00000200:16.0:1616513085.107650:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.191@tcp1(10.31.140.191@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.107656:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.107661:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.107663:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.107665:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.107669:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513085.107675:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513085.107676:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.107731:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.107738:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.107741:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513085.107747:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.107753:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2990 00000800:00000200:28.0:1616513085.107756:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.107759:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.108118:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.108171:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.108180:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.108188:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.191@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.108196:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.191@tcp1 of length 224 into portal 12 MB=0x602183647c9c0 00000400:00000200:26.0:1616513085.108201:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.191@tcp1 of length 224/224 into md 0xf964d519 [1] + 41320 00000400:00000200:26.0:1616513085.108206:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.108211:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.108217:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.108220:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.108222:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.108224:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.108281:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691152873408960 00010000:00000200:22.0:1616513085.108293:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b279b00 x1691152873408960/t0(0) o400->d8dcab46-3100-9cb6-f406-ddbe04de3447@10.31.140.191@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.108307:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691152873408960, offset 192 00000400:00000200:22.0:1616513085.108314:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.191@tcp1 00000400:00000200:22.0:1616513085.108321:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.191@tcp1 routed destination 00000400:00000200:22.0:1616513085.108336:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.191@tcp1(10.31.140.191@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.108342:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.108348:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.108351:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.108353:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.108357:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.108362:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.108372:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.108378:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.108380:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513085.108383:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513085.108387:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.108391:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a908 00000800:00000200:14.0:1616513085.108395:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.130168:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513085.130223:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.130233:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.130242:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.109@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.130251:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.109@tcp1 of length 224 into portal 26 MB=0x6000799938dc0 00000400:00000200:18.0:1616513085.130257:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.109@tcp1 of length 224/224 into md 0xf97cb069 [1] + 0 00000400:00000200:18.0:1616513085.130262:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2aa0 00000400:00000200:18.0:1616513085.130265:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.130268:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513085.130272:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2aa0 00000800:00000200:18.0:1616513085.130275:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.130278:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.130278:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.130282:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.130338:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688882501619136 00010000:00000200:16.0:1616513085.130349:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688882501619136/t0(0) o400->42416453-d195-ffde-5c9d-80ed984aa78b@10.31.136.109@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.130364:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688882501619136, offset 192 00000400:00000200:16.0:1616513085.130370:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.109@tcp1 00000400:00000200:16.0:1616513085.130377:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.109@tcp1 routed destination 00000400:00000200:16.0:1616513085.130392:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.109@tcp1(10.31.136.109@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.130403:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.130408:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.130411:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.130413:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.130418:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513085.130425:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513085.130426:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.130480:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.130487:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.130489:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513085.130496:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.130501:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b38f8 00000800:00000200:12.0:1616513085.130505:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.130508:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.130514:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513085.130516:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.130518:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.130520:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.130522:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513085.130618:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926864, portal 4 00000100:00000200:18.0:1616513085.130622:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926864, offset 0 00000400:00000200:18.0:1616513085.130627:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:18.0:1616513085.130632:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:18.0:1616513085.130644:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513085.130650:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513085.130656:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513085.130658:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513085.130660:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513085.130666:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513085.130712:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513085.130727:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513085.130733:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513085.130736:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513085.130743:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513085.130749:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc4eba400 x1693067725926864/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513085.130761:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cd980 00000800:00000200:2.0:1616513085.130765:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513085.130768:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513085.130803:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513085.130816:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513085.130824:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513085.130826:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:26.0:1616513085.130833:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513085.130842:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6271d0 00000400:00000200:26.0:1616513085.130848:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cb739 [1] + 192 00000400:00000200:26.0:1616513085.130853:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.130858:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc4eba400 x1693067725926864/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513085.130873:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513085.130876:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513085.130878:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:18.0:1616513085.130879:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc4eba400 x1693067725926864/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:18.0:1616513085.130892:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc4eba400 x1693067725926864/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513085.130902:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc330 00000800:00000200:7.2:1616513085.130995:0:50002:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.131007:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.131015:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513085.131018:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513085.131022:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.109@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.131030:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.109@tcp1 of length 224 into portal 12 MB=0x6000799938ec0 00000400:00000200:2.0:1616513085.131036:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.109@tcp1 of length 224/224 into md 0xf964d519 [1] + 41544 00000400:00000200:2.0:1616513085.131041:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.131044:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513085.131050:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.131052:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.131054:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.131115:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688882501619392 00010000:00000200:22.0:1616513085.131128:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebc800 x1688882501619392/t0(0) o400->a9e02152-6382-569c-52ae-286e1bd4e6db@10.31.136.109@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.131142:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688882501619392, offset 192 00000400:00000200:22.0:1616513085.131148:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.109@tcp1 00000400:00000200:22.0:1616513085.131155:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.109@tcp1 routed destination 00000400:00000200:22.0:1616513085.131170:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.109@tcp1(10.31.136.109@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.131178:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.131184:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.131186:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.131189:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.131193:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.131197:0:50002:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.131207:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.131211:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.131213:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513085.131215:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.131219:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.131222:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a4c8 00000800:00000200:18.0:1616513085.131224:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.141238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513085.141253:0:50003:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513085.141266:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.141275:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.141284:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.228@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.141293:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.228@tcp1 of length 224 into portal 12 MB=0x5ffdd26644dc0 00000800:00000200:20.0:1616513085.141294:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513085.141300:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.228@tcp1 of length 224/224 into md 0xf964d519 [1] + 41768 00000800:00000200:20.0:1616513085.141303:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:14.0:1616513085.141305:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.141311:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.228@tcp1 : PUT - for me 00000100:00000200:14.0:1616513085.141316:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513085.141320:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.228@tcp1 of length 224 into portal 26 MB=0x5ffdd26644ac0 00000800:00000200:14.0:1616513085.141322:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.141325:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513085.141327:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.228@tcp1 of length 224/224 into md 0xf97cb089 [1] + 0 00000800:00000200:14.0:1616513085.141327:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.141330:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513085.141331:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2908 00000400:00000200:20.0:1616513085.141334:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.141337:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513085.141341:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2908 00000800:00000200:20.0:1616513085.141344:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.141347:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.141349:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.141352:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513085.141387:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700180516288 00010000:00000200:22.0:1616513085.141398:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623180 x1688700180516288/t0(0) o400->6638e1b1-43e9-4892-f6f8-28da6ab89e4c@10.31.140.228@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513085.141409:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700180515520 00000100:00000200:22.0:1616513085.141411:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700180516288, offset 192 00000400:00000200:22.0:1616513085.141417:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.228@tcp1 00010000:00000200:16.0:1616513085.141420:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688700180515520/t0(0) o400->c4845fe9-8f67-2549-b895-840d878e13e7@10.31.140.228@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513085.141424:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.228@tcp1 routed destination 00000100:00000200:16.0:1616513085.141446:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700180515520, offset 192 00000400:00000200:22.0:1616513085.141451:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.228@tcp1(10.31.140.228@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513085.141452:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.228@tcp1 00000800:00000200:22.0:1616513085.141457:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513085.141460:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.228@tcp1 routed destination 00000800:00000200:22.0:1616513085.141463:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.141465:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.141467:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.141472:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513085.141475:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.228@tcp1(10.31.140.228@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513085.141477:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.141481:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.141487:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.141490:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.141492:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.141497:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.141501:0:50003:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.141504:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513085.141513:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.141520:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.141523:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.141529:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513085.141532:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513085.141535:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2198 00000800:00000200:26.0:1616513085.141538:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513085.141539:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513085.141541:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513085.141541:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.141548:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.141552:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a110 00000800:00000200:24.0:1616513085.141556:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.141559:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.145309:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513085.145363:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.145372:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513085.145382:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.251@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.145391:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.251@tcp1 of length 224 into portal 26 MB=0x5ffd307287ec0 00000400:00000200:20.0:1616513085.145397:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.251@tcp1 of length 224/224 into md 0xf97cb099 [1] + 0 00000400:00000200:20.0:1616513085.145402:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2e58 00000400:00000200:20.0:1616513085.145405:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.145409:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513085.145413:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2e58 00000800:00000200:20.0:1616513085.145416:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.145419:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.145421:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.145424:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.145477:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656706830016 00010000:00000200:16.0:1616513085.145488:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688656706830016/t0(0) o400->acb06aed-df4c-9ec3-61d6-3d3abcfb0f49@10.31.130.251@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.145502:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688656706830016, offset 192 00000400:00000200:16.0:1616513085.145509:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.251@tcp1 00000400:00000200:16.0:1616513085.145517:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.251@tcp1 routed destination 00000400:00000200:16.0:1616513085.145531:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.251@tcp1(10.31.130.251@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.145538:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.145544:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.145546:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.145550:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.145555:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.145559:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.145562:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513085.145613:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.145619:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513085.145622:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.145629:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.145635:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3188 00000800:00000200:24.0:1616513085.145638:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.145641:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.145810:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513085.145863:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.145872:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.145880:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.251@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.145888:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.251@tcp1 of length 224 into portal 12 MB=0x5ffd307288340 00000400:00000200:14.0:1616513085.145894:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.251@tcp1 of length 224/224 into md 0xf964d519 [1] + 41992 00000400:00000200:14.0:1616513085.145899:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513085.145901:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513085.145907:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.145910:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513085.145912:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.145914:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.145970:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656706831168 00010000:00000200:22.0:1616513085.145982:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464625100 x1688656706831168/t0(0) o400->4af366b3-6eea-7ae8-e7c3-cb5e6eca4f08@10.31.130.251@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.145997:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688656706831168, offset 192 00000400:00000200:22.0:1616513085.146004:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.251@tcp1 00000400:00000200:22.0:1616513085.146011:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.251@tcp1 routed destination 00000400:00000200:22.0:1616513085.146026:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.251@tcp1(10.31.130.251@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.146032:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.146038:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.146041:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.146043:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.146048:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.146052:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.146063:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.146070:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.146072:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.146078:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.146082:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba08 00000800:00000200:26.0:1616513085.146086:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.146088:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.160528:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513085.160583:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.160593:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513085.160602:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.174@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.160613:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.174@tcp1 of length 224 into portal 26 MB=0x5ffd3d44f0ec0 00000400:00000200:20.0:1616513085.160620:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.174@tcp1 of length 224/224 into md 0xf97cb0b9 [1] + 0 00000400:00000200:20.0:1616513085.160624:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3a08 00000400:00000200:20.0:1616513085.160627:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.160630:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513085.160634:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a08 00000800:00000200:20.0:1616513085.160637:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.160637:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.160640:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.160642:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.160698:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660148686528 00010000:00000200:16.0:1616513085.160709:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688660148686528/t0(0) o400->0eefc17d-b60b-2b81-2a5e-44f47ca2a3ba@10.31.128.174@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.160724:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660148686528, offset 192 00000400:00000200:16.0:1616513085.160730:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.174@tcp1 00000400:00000200:16.0:1616513085.160738:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.174@tcp1 routed destination 00000400:00000200:16.0:1616513085.160754:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.174@tcp1(10.31.128.174@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.160761:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.160766:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.160769:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.160771:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.160776:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.160782:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.160783:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513085.160836:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.160843:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513085.160846:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.160852:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.160858:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3a08 00000800:00000200:24.0:1616513085.160861:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.160864:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.160870:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513085.160872:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513085.160878:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.174@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.160887:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.174@tcp1 of length 224 into portal 12 MB=0x5ffd3d44f1380 00000400:00000200:24.0:1616513085.160893:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.174@tcp1 of length 224/224 into md 0x865bbec1 [1] + 76384 00000400:00000200:24.0:1616513085.160898:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.160901:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513085.160907:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.160910:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.160911:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.160973:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660148687744 00010000:00000200:0.0:1616513085.160986:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd703f00 x1688660148687744/t0(0) o400->de1934e3-02e6-8ac0-44e7-63e071511ac5@10.31.128.174@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.161000:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660148687744, offset 192 00000400:00000200:0.0:1616513085.161006:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.174@tcp1 00000400:00000200:0.0:1616513085.161014:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.174@tcp1 routed destination 00000400:00000200:0.0:1616513085.161029:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.174@tcp1(10.31.128.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.161035:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.161042:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.161045:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.161047:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.161052:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.161058:0:50005:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.161112:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.161119:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513085.161121:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513085.161127:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513085.161131:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05c28 00000800:00000200:2.0:1616513085.161135:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.161138:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513085.178563:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926896, portal 4 00000100:00000200:15.0:1616513085.178568:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926896, offset 0 00000400:00000200:15.0:1616513085.178575:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:19.0:1616513085.178577:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926880, portal 4 00000400:00000200:15.0:1616513085.178582:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000100:00000200:19.0:1616513085.178583:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926880, offset 0 00000400:00000200:19.0:1616513085.178592:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:15.0:1616513085.178596:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513085.178603:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:19.0:1616513085.178604:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:15.0:1616513085.178612:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513085.178615:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000400:00000200:19.0:1616513085.178617:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513085.178617:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:15.0:1616513085.178623:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:19.0:1616513085.178624:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:19.0:1616513085.178631:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:27.2:1616513085.178632:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:19.0:1616513085.178634:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:19.0:1616513085.178636:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:19.0:1616513085.178642:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:27.0:1616513085.178646:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513085.178651:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513085.178654:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513085.178658:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513085.178661:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513085.178667:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600d80 x1693067725926896/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513085.178679:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b760 00000800:00000200:27.0:1616513085.178683:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:5.2:1616513085.178688:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513085.178745:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513085.178753:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513085.178756:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.178763:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513085.178770:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be42d00 x1693067725926880/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513085.178782:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706990 00000800:00000200:20.0:1616513085.178786:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513085.178789:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.2:1616513085.178841:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513085.178850:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:5.2:1616513085.178855:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:27.0:1616513085.178857:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513085.178865:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513085.178873:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6271f0 00000400:00000200:27.0:1616513085.178879:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166e2d [1] + 192 00000400:00000200:27.0:1616513085.178884:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513085.178888:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600d80 x1693067725926896/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513085.178902:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513085.178905:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513085.178905:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:28.0:1616513085.178909:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:15.0:1616513085.178909:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600d80 x1693067725926896/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513085.178910:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:28.0:1616513085.178919:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000100:00000200:15.0:1616513085.178922:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600d80 x1693067725926896/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513085.178929:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:15.0:1616513085.178932:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000400:00000200:28.0:1616513085.178937:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6271e0 00000400:00000200:28.0:1616513085.178944:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d8341 [1] + 192 00000400:00000200:28.0:1616513085.178949:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513085.178955:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be42d00 x1693067725926880/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:24.0:1616513085.178965:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513085.178970:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513085.178973:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513085.178975:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:19.0:1616513085.178980:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be42d00 x1693067725926880/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513085.178993:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be42d00 x1693067725926880/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513085.179004:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000800:00000200:11.2:1616513085.180839:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513085.180896:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.180906:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513085.180914:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.188@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.180923:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.188@tcp1 of length 224 into portal 26 MB=0x5ffe3526bf640 00000400:00000200:20.0:1616513085.180930:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.188@tcp1 of length 224/224 into md 0xf97cb0d9 [1] + 0 00000400:00000200:20.0:1616513085.180934:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3430 00000400:00000200:20.0:1616513085.180937:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.180942:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513085.180947:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3430 00000800:00000200:12.0:1616513085.180949:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.180950:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.180953:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.180955:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.181011:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726689019456 00010000:00000200:16.0:1616513085.181022:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688726689019456/t0(0) o400->0aed7d57-f365-1b66-da7b-c3275cb95c49@10.31.131.188@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.181035:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726689019456, offset 192 00000400:00000200:16.0:1616513085.181041:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.188@tcp1 00000400:00000200:16.0:1616513085.181048:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.188@tcp1 routed destination 00000400:00000200:16.0:1616513085.181062:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.188@tcp1(10.31.131.188@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.181069:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.181074:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.181077:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.181079:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.181083:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.181087:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.181090:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513085.181141:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.181148:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513085.181151:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.181157:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.181163:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2f68 00000800:00000200:24.0:1616513085.181167:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.181170:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.181655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513085.181667:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.181675:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513085.181677:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513085.181684:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.188@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.181693:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.188@tcp1 of length 224 into portal 12 MB=0x5ffe3526bf780 00000400:00000200:12.0:1616513085.181699:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.188@tcp1 of length 224/224 into md 0x865bbec1 [1] + 76608 00000400:00000200:12.0:1616513085.181704:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.181707:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513085.181714:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.181717:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.181719:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.181781:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726689019776 00010000:00000200:0.0:1616513085.181794:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d202d00 x1688726689019776/t0(0) o400->2bec88a0-927a-da9f-d4c8-1fcc2b212239@10.31.131.188@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.181808:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726689019776, offset 192 00000400:00000200:0.0:1616513085.181814:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.188@tcp1 00000400:00000200:0.0:1616513085.181821:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.188@tcp1 routed destination 00000400:00000200:0.0:1616513085.181836:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.188@tcp1(10.31.131.188@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.181843:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.181849:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.181852:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.181854:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.181859:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.181866:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.181880:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.181887:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.181891:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513085.181893:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513085.181898:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.181902:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05f58 00000800:00000200:26.0:1616513085.181906:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.191668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513085.191732:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.191742:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.191755:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.145@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.191764:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.145@tcp1 of length 224 into portal 12 MB=0x60107029d0f00 00000400:00000200:24.0:1616513085.191771:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.145@tcp1 of length 224/224 into md 0x865bbec1 [1] + 76832 00000400:00000200:24.0:1616513085.191776:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.191779:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513085.191786:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.191789:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513085.191791:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.191795:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.191848:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689979480510208 00000800:00000200:7.2:1616513085.191854:0:50010:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:0.0:1616513085.191860:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd707500 x1689979480510208/t0(0) o400->550ab18c-29da-e6bd-542c-1ba9cf50258b@10.31.143.145@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.191884:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689979480510208, offset 192 00000400:00000200:0.0:1616513085.191890:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.145@tcp1 00000400:00000200:0.0:1616513085.191897:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.145@tcp1 routed destination 00000400:00000200:0.0:1616513085.191910:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.145@tcp1(10.31.143.145@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513085.191912:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.191916:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513085.191922:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.191922:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.191924:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.191926:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513085.191931:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.145@tcp1 : PUT - for me 00000800:00000200:0.0:1616513085.191931:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.191936:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513085.191940:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.145@tcp1 of length 224 into portal 26 MB=0x60107029d0e80 00000400:00000200:2.0:1616513085.191947:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.145@tcp1 of length 224/224 into md 0xf97cb119 [1] + 0 00000400:00000200:2.0:1616513085.191952:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2990 00000400:00000200:2.0:1616513085.191955:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.191957:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.191962:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2990 00000800:00000200:2.0:1616513085.191965:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.191968:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.191968:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.191972:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513085.191991:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.191998:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.192001:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.192008:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.192012:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05078 00000800:00000200:20.0:1616513085.192016:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.192019:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.192030:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689979480510080 00010000:00000200:16.0:1616513085.192041:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1689979480510080/t0(0) o400->cbea4e91-f54c-4b11-44e2-a83cf9afa317@10.31.143.145@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.192058:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689979480510080, offset 192 00000400:00000200:16.0:1616513085.192064:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.145@tcp1 00000400:00000200:16.0:1616513085.192071:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.145@tcp1 routed destination 00000400:00000200:16.0:1616513085.192086:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.145@tcp1(10.31.143.145@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.192093:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.192099:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.192102:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.192104:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.192109:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.192114:0:50010:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.192116:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513085.192124:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.192129:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.192132:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513085.192138:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.192142:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2dd0 00000800:00000200:18.0:1616513085.192145:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.192148:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513085.194683:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926912, portal 4 00000100:00000200:10.0:1616513085.194689:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926912, offset 0 00000400:00000200:10.0:1616513085.194695:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:10.0:1616513085.194700:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:10.0:1616513085.194712:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513085.194718:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:10.0:1616513085.194724:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513085.194727:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513085.194729:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513085.194733:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513085.194780:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513085.194836:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513085.194844:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513085.194846:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513085.194849:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513085.194853:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513085.194859:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c8a2e780 x1693067725926912/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513085.194871:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82198 00000800:00000200:14.0:1616513085.194875:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513085.194952:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513085.194961:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513085.194965:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513085.194972:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513085.194978:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627200 00000400:00000200:18.0:1616513085.194983:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cb7d9 [1] + 192 00000400:00000200:18.0:1616513085.194987:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.194991:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c8a2e780 x1693067725926912/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513085.195003:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513085.195005:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:18.0:1616513085.195006:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513085.195008:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:10.0:1616513085.195010:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c8a2e780 x1693067725926912/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513085.195022:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c8a2e780 x1693067725926912/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513085.195031:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83540 00000400:00000080:7.0:1616513085.203676:0:50012:0:(module.c:90:libcfs_ioctl()) libcfs ioctl cmd 3233310049 00000400:00000080:7.0:1616513085.203709:0:50012:0:(module.c:90:libcfs_ioctl()) libcfs ioctl cmd 3233310049 00000400:00000080:7.0:1616513085.203719:0:50012:0:(module.c:90:libcfs_ioctl()) libcfs ioctl cmd 3233310049 00000800:00000200:11.2:1616513085.214015:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513085.214071:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.214081:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513085.214090:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.22@tcp1 : PUT - for me 00000400:00000200:28.0:1616513085.214099:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.22@tcp1 of length 224 into portal 26 MB=0x602431cc43e40 00000400:00000200:28.0:1616513085.214106:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.22@tcp1 of length 224/224 into md 0xf97cb129 [1] + 0 00000400:00000200:28.0:1616513085.214111:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2880 00000400:00000200:28.0:1616513085.214113:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513085.214116:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513085.214121:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2880 00000800:00000200:28.0:1616513085.214124:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.214126:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.214129:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513085.214133:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513085.214136:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.214137:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.214145:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.22@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.214154:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.22@tcp1 of length 224 into portal 12 MB=0x602431cc43ec0 00000400:00000200:24.0:1616513085.214160:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.22@tcp1 of length 224/224 into md 0x865bbec1 [1] + 77056 00000400:00000200:24.0:1616513085.214166:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.214168:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513085.214176:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.214179:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.214181:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.214195:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691337128951360 00010000:00000200:16.0:1616513085.214206:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1691337128951360/t0(0) o400->7c0e9199-2c26-6917-7bde-9276024db9c4@10.31.143.22@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.214221:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691337128951360, offset 192 00000400:00000200:16.0:1616513085.214228:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.22@tcp1 00000400:00000200:16.0:1616513085.214235:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.22@tcp1 routed destination 00000100:00000200:0.0:1616513085.214242:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691337128951488 00000400:00000200:16.0:1616513085.214250:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.22@tcp1(10.31.143.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513085.214255:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd705580 x1691337128951488/t0(0) o400->5ead7c50-d94c-7006-008a-a25d5475b423@10.31.143.22@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.214256:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.214262:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.214265:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.214280:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513085.214282:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691337128951488, offset 192 00000800:00000200:16.0:1616513085.214284:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513085.214288:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.22@tcp1 00000800:00000200:11.2:1616513085.214290:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.214291:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513085.214294:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.22@tcp1 routed destination 00000400:00000200:0.0:1616513085.214308:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.22@tcp1(10.31.143.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.214314:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.214320:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.214323:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.214325:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.214329:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.214338:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513085.214343:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.214351:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.214353:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513085.214355:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513085.214360:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.214366:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2088 00000800:00000200:12.0:1616513085.214369:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513085.214394:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.214401:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513085.214404:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513085.214411:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513085.214417:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:2.0:1616513085.214422:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.214425:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.224311:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.224327:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.224337:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.224346:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.248@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.224356:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.248@tcp1 of length 224 into portal 26 MB=0x603ff61b7c280 00000400:00000200:26.0:1616513085.224362:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.248@tcp1 of length 224/224 into md 0xf97cb149 [1] + 0 00000400:00000200:26.0:1616513085.224367:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2110 00000400:00000200:26.0:1616513085.224370:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.224373:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513085.224377:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2110 00000800:00000200:26.0:1616513085.224380:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.224383:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.224384:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513085.224391:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.224394:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.224395:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.224403:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.248@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.224412:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.248@tcp1 of length 224 into portal 12 MB=0x603ff61b7c300 00000400:00000200:18.0:1616513085.224419:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.248@tcp1 of length 224/224 into md 0xf964d519 [1] + 42216 00000400:00000200:18.0:1616513085.224424:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.224427:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513085.224463:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.224465:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513085.224466:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245251240576 00000800:00000200:18.0:1616513085.224467:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:16.0:1616513085.224477:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1693245251240576/t0(0) o400->317f70d8-c7d9-71d1-8d81-723cddf0528b@10.31.132.248@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.224491:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245251240576, offset 192 00000400:00000200:16.0:1616513085.224497:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.248@tcp1 00000400:00000200:16.0:1616513085.224505:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.248@tcp1 routed destination 00000100:00000200:22.0:1616513085.224509:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245251240704 00010000:00000200:22.0:1616513085.224522:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfc380 x1693245251240704/t0(0) o400->01222aab-b634-c954-89b6-7798000b20e5@10.31.132.248@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513085.224522:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.248@tcp1(10.31.132.248@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.224528:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.224534:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:22.0:1616513085.224537:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245251240704, offset 192 00000800:00000200:16.0:1616513085.224537:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.224539:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513085.224543:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.248@tcp1 00000800:00000200:16.0:1616513085.224544:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513085.224551:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.248@tcp1 routed destination 00000100:00000200:16.0:1616513085.224551:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513085.224551:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513085.224566:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.248@tcp1(10.31.132.248@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.224572:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.224578:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.224581:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.224583:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.224587:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.224593:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513085.224605:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.224612:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.224615:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513085.224618:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.224621:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.224627:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3210 00000800:00000200:28.0:1616513085.224630:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513085.224665:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.224672:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.224675:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513085.224675:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513085.224682:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.224686:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b320 00000800:00000200:14.0:1616513085.224690:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513085.226590:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926928, portal 4 00000100:00000200:10.0:1616513085.226596:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926928, offset 0 00000400:00000200:10.0:1616513085.226605:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:10.0:1616513085.226612:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:10.0:1616513085.226624:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513085.226631:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:10.0:1616513085.226638:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513085.226641:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:10.0:1616513085.226643:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:10.0:1616513085.226648:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513085.226696:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.226716:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513085.226722:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513085.226726:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513085.226733:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513085.226741:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c8a2a880 x1693067725926928/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513085.226753:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83980 00000800:00000200:1.0:1616513085.226759:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513085.226762:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513085.226889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513085.226900:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.226906:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513085.226910:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513085.226914:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513085.226923:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627210 00000400:00000200:1.0:1616513085.226930:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffd45 [1] + 192 00000400:00000200:1.0:1616513085.226935:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513085.226942:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c8a2a880 x1693067725926928/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513085.226957:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.226960:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513085.226961:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:10.0:1616513085.226966:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c8a2a880 x1693067725926928/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513085.226980:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c8a2a880 x1693067725926928/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513085.226990:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83540 00000800:00000200:7.2:1616513085.251275:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.251289:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.251299:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.251320:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.3@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.251330:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.3@tcp1 of length 224 into portal 26 MB=0x5ffd5b9ecca80 00000400:00000200:2.0:1616513085.251336:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.3@tcp1 of length 224/224 into md 0xf97cb159 [1] + 0 00000400:00000200:2.0:1616513085.251340:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf22a8 00000400:00000200:2.0:1616513085.251343:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.251345:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.251350:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf22a8 00000800:00000200:2.0:1616513085.251353:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.251355:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.251356:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.251360:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.251415:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668295973504 00010000:00000200:16.0:1616513085.251426:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688668295973504/t0(0) o400->cf20eb6d-0c45-439f-8602-9b39e6b6a342@10.31.130.3@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513085.251439:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513085.251454:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668295973504, offset 192 00000400:00000200:16.0:1616513085.251460:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.3@tcp1 00000400:00000200:16.0:1616513085.251468:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.3@tcp1 routed destination 00000400:00000200:16.0:1616513085.251484:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.3@tcp1(10.31.130.3@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.251491:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513085.251492:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.251496:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.251499:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513085.251501:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.251501:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.251506:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513085.251509:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.3@tcp1 : PUT - for me 00000800:00000200:11.2:1616513085.251512:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.251513:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513085.251517:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.3@tcp1 of length 224 into portal 12 MB=0x5ffd5b9eccbc0 00000400:00000200:26.0:1616513085.251523:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.3@tcp1 of length 224/224 into md 0xf964d519 [1] + 42440 00000400:00000200:26.0:1616513085.251528:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.251531:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.251539:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.251542:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.251544:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.251546:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513085.251569:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.251576:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.251579:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.251586:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.251591:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3a90 00000800:00000200:20.0:1616513085.251599:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.251602:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513085.251613:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668295973824 00010000:00000200:22.0:1616513085.251626:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b27de80 x1688668295973824/t0(0) o400->98e6d596-2ffa-61e1-218d-b7e2b721b871@10.31.130.3@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.251640:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668295973824, offset 192 00000400:00000200:22.0:1616513085.251647:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.3@tcp1 00000400:00000200:22.0:1616513085.251655:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.3@tcp1 routed destination 00000400:00000200:22.0:1616513085.251670:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.3@tcp1(10.31.130.3@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.251676:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.251682:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.251685:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.251687:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.251692:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.251696:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.251707:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.251712:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.251715:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513085.251716:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.251721:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.251725:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aaa0 00000800:00000200:18.0:1616513085.251729:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513085.274578:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926960, portal 4 00000100:00000200:30.0:1616513085.274584:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926960, offset 0 00000100:00000200:15.0:1616513085.274588:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926944, portal 4 00000400:00000200:30.0:1616513085.274591:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000100:00000200:15.0:1616513085.274594:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926944, offset 0 00000400:00000200:30.0:1616513085.274596:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:15.0:1616513085.274602:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:15.0:1616513085.274608:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513085.274609:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513085.274615:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:30.0:1616513085.274623:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:15.0:1616513085.274623:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513085.274626:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:30.0:1616513085.274628:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:15.0:1616513085.274631:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513085.274633:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:7.2:1616513085.274635:0:50020:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:15.0:1616513085.274638:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513085.274641:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:15.0:1616513085.274643:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:26.0:1616513085.274647:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:15.0:1616513085.274649:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:26.0:1616513085.274655:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:4.0:1616513085.274656:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926976, portal 4 00000100:00000200:4.0:1616513085.274662:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926976, offset 0 00000400:00000200:26.0:1616513085.274663:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.22@tcp1 : PUT - for me 00000400:00000200:4.0:1616513085.274668:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:26.0:1616513085.274673:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.22@tcp1 of length 224 into portal 26 MB=0x5ffdd04632a00 00000400:00000200:4.0:1616513085.274674:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:26.0:1616513085.274679:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.22@tcp1 of length 224/224 into md 0xf97cb169 [1] + 0 00000800:00000200:1.2:1616513085.274680:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:26.0:1616513085.274685:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3650 00000400:00000200:26.0:1616513085.274687:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:4.0:1616513085.274687:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513085.274690:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:4.0:1616513085.274693:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:26.0:1616513085.274694:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3650 00000800:00000200:26.0:1616513085.274697:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.274700:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:5.2:1616513085.274700:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513085.274700:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513085.274701:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:1.0:1616513085.274701:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:4.0:1616513085.274702:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:26.0:1616513085.274704:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513085.274705:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (24)++ 00000800:00000200:4.0:1616513085.274710:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (25)-- 00000800:00000200:1.0:1616513085.274712:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (24)-- 00000400:00000200:1.0:1616513085.274715:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513085.274719:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000400:00000200:1.0:1616513085.274723:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:5.0:1616513085.274726:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (24)-- 00000400:00000200:5.0:1616513085.274729:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:1.0:1616513085.274731:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b27b180 x1693067725926960/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513085.274735:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:5.0:1616513085.274741:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d576f0e8000 x1693067725926976/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513085.274743:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef4b8 00000800:00000200:1.0:1616513085.274748:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513085.274752:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:5.0:1616513085.274753:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb298 00000800:00000200:5.0:1616513085.274757:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:16.0:1616513085.274774:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699610016256 00010000:00000200:16.0:1616513085.274785:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688699610016256/t0(0) o400->e6ed4e1a-0ab4-414f-e502-7376095bbac6@10.31.138.22@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513085.274786:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513085.274794:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:24.0:1616513085.274798:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513085.274799:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:16.0:1616513085.274800:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699610016256, offset 192 00000400:00000200:24.0:1616513085.274804:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:16.0:1616513085.274806:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.22@tcp1 00000100:00000200:24.0:1616513085.274813:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd604380 x1693067725926944/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513085.274814:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.22@tcp1 routed destination 00000400:00000200:24.0:1616513085.274825:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0bf58 00000400:00000200:16.0:1616513085.274830:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.22@tcp1(10.31.138.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513085.274835:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:16.0:1616513085.274836:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:1.2:1616513085.274836:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:16.0:1616513085.274842:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513085.274844:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:16.0:1616513085.274847:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.274849:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:1.0:1616513085.274851:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.232@o2ib1 00000800:00000200:16.0:1616513085.274854:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513085.274861:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:5.0:1616513085.274863:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:1.0:1616513085.274864:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:5.2:1616513085.274870:0:6467:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:9.0:1616513085.274873:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513085.274873:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627230 00000800:00000200:5.0:1616513085.274877:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513085.274879:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffd55 [1] + 192 00000400:00000200:5.0:1616513085.274884:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513085.274885:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513085.274890:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b27b180 x1693067725926960/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513085.274892:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627240 00000800:00000200:11.2:1616513085.274897:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:1.0:1616513085.274904:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:5.0:1616513085.274906:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffd65 [1] + 192 00000800:00000200:1.0:1616513085.274907:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513085.274909:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:5.0:1616513085.274911:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513085.274914:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b27b180 x1693067725926960/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513085.274924:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:30.0:1616513085.274926:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b27b180 x1693067725926960/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513085.274934:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:28.0:1616513085.274935:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:30.0:1616513085.274936:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eecc0 00000800:00000200:28.0:1616513085.274940:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:5.0:1616513085.274941:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d576f0e8000 x1693067725926976/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513085.274943:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.274944:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:24.0:1616513085.274945:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513085.274948:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513085.274949:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513085.274952:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513085.274952:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627220 00000400:00000200:28.0:1616513085.274954:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2000 00000800:00000200:12.0:1616513085.274954:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.274955:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:5.0:1616513085.274955:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:24.0:1616513085.274957:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513085.274958:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:5.0:1616513085.274958:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:5.0:1616513085.274959:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000400:00000200:20.0:1616513085.274960:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d83d1 [1] + 192 00000800:00000200:24.0:1616513085.274961:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513085.274962:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d576f0e8000 x1693067725926976/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513085.274965:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.274971:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd604380 x1693067725926944/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513085.274974:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d576f0e8000 x1693067725926976/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513085.274983:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00000800:00000200:20.0:1616513085.274986:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513085.274989:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513085.274990:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:15.0:1616513085.274996:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd604380 x1693067725926944/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513085.275009:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd604380 x1693067725926944/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513085.275030:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000800:00000200:7.2:1616513085.276032:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.276083:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.276092:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.276100:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.22@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.276107:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.22@tcp1 of length 224 into portal 12 MB=0x5ffdd04632b00 00000400:00000200:2.0:1616513085.276113:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.22@tcp1 of length 224/224 into md 0xf964d519 [1] + 42664 00000400:00000200:2.0:1616513085.276117:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.276120:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513085.276127:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.276129:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.276131:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.276133:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.276209:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699610016512 00010000:00000200:22.0:1616513085.276222:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebec00 x1688699610016512/t0(0) o400->b9889840-c9e2-8852-49a8-d5b7a28bf806@10.31.138.22@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.276236:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699610016512, offset 192 00000400:00000200:22.0:1616513085.276242:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.22@tcp1 00000400:00000200:22.0:1616513085.276250:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.22@tcp1 routed destination 00000400:00000200:22.0:1616513085.276266:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.22@tcp1(10.31.138.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.276273:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.276279:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.276282:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.276284:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.276289:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.276293:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.276345:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.276352:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.276357:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.276362:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513085.276368:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.276372:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b650 00000800:00000200:14.0:1616513085.276376:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.292317:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513085.292373:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.292382:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.292391:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.6@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.292400:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.6@tcp1 of length 224 into portal 26 MB=0x601827c71fb40 00000400:00000200:12.0:1616513085.292407:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.6@tcp1 of length 224/224 into md 0xf97cb189 [1] + 0 00000400:00000200:12.0:1616513085.292412:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3760 00000400:00000200:12.0:1616513085.292414:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.292417:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513085.292422:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3760 00000800:00000200:12.0:1616513085.292425:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.292427:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.292428:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.292431:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.292485:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509805484864 00010000:00000200:16.0:1616513085.292496:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1690509805484864/t0(0) o400->0c54cd7c-e4a9-1e06-39ef-200d419630b6@10.31.141.6@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.292511:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690509805484864, offset 192 00000400:00000200:16.0:1616513085.292517:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.6@tcp1 00000400:00000200:16.0:1616513085.292525:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.6@tcp1 routed destination 00000400:00000200:16.0:1616513085.292540:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.6@tcp1(10.31.141.6@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.292546:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.292552:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.292554:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.292557:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.292561:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.292566:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.292568:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513085.292620:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.292627:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513085.292630:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.292636:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.292642:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3b18 00000800:00000200:24.0:1616513085.292646:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.292649:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.293234:0:50022:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.293290:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.293299:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.293309:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.6@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.293317:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.6@tcp1 of length 224 into portal 12 MB=0x601827c720500 00000400:00000200:2.0:1616513085.293324:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.6@tcp1 of length 224/224 into md 0xf964d519 [1] + 42888 00000400:00000200:2.0:1616513085.293331:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.293334:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513085.293340:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.293343:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.293344:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.293349:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.293411:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509805487360 00010000:00000200:22.0:1616513085.293424:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eba880 x1690509805487360/t0(0) o400->2e398e95-6fa0-ff84-d280-e276e33fd2ba@10.31.141.6@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.293451:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690509805487360, offset 192 00000400:00000200:22.0:1616513085.293458:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.6@tcp1 00000400:00000200:22.0:1616513085.293465:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.6@tcp1 routed destination 00000400:00000200:22.0:1616513085.293480:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.6@tcp1(10.31.141.6@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.293487:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.293492:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.293495:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.293497:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.293502:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.293506:0:50022:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.293518:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.293525:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.293527:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.293534:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.293538:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ac38 00000800:00000200:26.0:1616513085.293542:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.293545:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.298924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513085.298941:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.298950:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513085.298959:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.173@tcp1 : PUT - for me 00000400:00000200:28.0:1616513085.298968:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.173@tcp1 of length 224 into portal 12 MB=0x5ffd41834af80 00000400:00000200:28.0:1616513085.298974:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.173@tcp1 of length 224/224 into md 0x865bbec1 [1] + 77280 00000400:00000200:28.0:1616513085.298979:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513085.298982:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513085.298989:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.298992:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.298994:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.298998:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.299054:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661287808896 00010000:00000200:0.0:1616513085.299067:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ed100 x1688661287808896/t0(0) o400->e974275d-8918-1cca-6bdc-e2a39ff84813@10.31.138.173@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513085.299072:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513085.299081:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661287808896, offset 192 00000400:00000200:0.0:1616513085.299087:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.173@tcp1 00000400:00000200:0.0:1616513085.299095:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.173@tcp1 routed destination 00000400:00000200:0.0:1616513085.299109:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.173@tcp1(10.31.138.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.299118:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.299124:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513085.299126:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.299127:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.299129:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.299134:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513085.299135:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513085.299138:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513085.299144:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.173@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.299153:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.173@tcp1 of length 224 into portal 26 MB=0x5ffd41834af00 00000400:00000200:18.0:1616513085.299160:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.173@tcp1 of length 224/224 into md 0xf97cb1a9 [1] + 0 00000400:00000200:18.0:1616513085.299165:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf26e8 00000400:00000200:18.0:1616513085.299167:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.299170:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513085.299174:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf26e8 00000800:00000200:18.0:1616513085.299177:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.299179:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.299180:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.299183:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513085.299200:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.299206:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513085.299209:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.299216:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.299220:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000800:00000200:24.0:1616513085.299224:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.299227:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.299240:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661287808768 00010000:00000200:16.0:1616513085.299252:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688661287808768/t0(0) o400->deb7bd25-8c75-3d66-943c-d16847c194df@10.31.138.173@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.299266:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661287808768, offset 192 00000400:00000200:16.0:1616513085.299272:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.173@tcp1 00000400:00000200:16.0:1616513085.299280:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.173@tcp1 routed destination 00000400:00000200:16.0:1616513085.299295:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.173@tcp1(10.31.138.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.299302:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.299308:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.299310:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.299312:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.299317:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.299322:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.299324:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513085.299375:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.299381:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.299384:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.299390:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.299396:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3ed0 00000800:00000200:26.0:1616513085.299399:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.299402:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.319242:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.319298:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.319308:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.319317:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.58@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.319326:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.58@tcp1 of length 224 into portal 26 MB=0x60198f0b1ca40 00000400:00000200:2.0:1616513085.319332:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.58@tcp1 of length 224/224 into md 0xf97cb1b9 [1] + 0 00000400:00000200:2.0:1616513085.319337:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3cb0 00000400:00000200:2.0:1616513085.319340:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.319342:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.319347:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3cb0 00000800:00000200:2.0:1616513085.319350:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.319352:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.319353:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.319356:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.319411:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513085.319414:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690606245104192 00010000:00000200:16.0:1616513085.319425:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1690606245104192/t0(0) o400->c88d15dd-8764-3f14-ae32-421d8b243681@10.31.142.58@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.319453:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690606245104192, offset 192 00000400:00000200:16.0:1616513085.319459:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.58@tcp1 00000800:00000200:26.0:1616513085.319462:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513085.319467:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.58@tcp1 routed destination 00000800:00000200:26.0:1616513085.319471:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.319480:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.58@tcp1 : PUT - for me 00000400:00000200:16.0:1616513085.319482:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.58@tcp1(10.31.142.58@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513085.319488:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.58@tcp1 of length 224 into portal 12 MB=0x60198f0b1cd40 00000800:00000200:16.0:1616513085.319488:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513085.319494:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.58@tcp1 of length 224/224 into md 0xf964d519 [1] + 43112 00000800:00000200:16.0:1616513085.319494:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.319497:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513085.319499:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513085.319499:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513085.319501:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513085.319504:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513085.319508:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.319511:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:16.0:1616513085.319511:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513085.319512:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.319515:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.319551:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513085.319572:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690606245104960 00010000:00000200:22.0:1616513085.319584:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b278480 x1690606245104960/t0(0) o400->b066ca69-619c-5187-6a47-b96f9b412d92@10.31.142.58@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.319598:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690606245104960, offset 192 00000400:00000200:22.0:1616513085.319605:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.58@tcp1 00000800:00000200:12.0:1616513085.319606:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513085.319612:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.58@tcp1 routed destination 00000800:00000200:12.0:1616513085.319613:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.319618:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513085.319625:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513085.319627:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.58@tcp1(10.31.142.58@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513085.319630:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2d48 00000800:00000200:22.0:1616513085.319633:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513085.319634:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.319637:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513085.319639:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.319642:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.319644:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.319650:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.319655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.319666:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.319672:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.319676:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513085.319676:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.319683:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.319687:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bed0 00000800:00000200:18.0:1616513085.319690:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.321979:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513085.322036:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.322046:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.322055:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.13@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.322063:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.13@tcp1 of length 224 into portal 26 MB=0x5f911a91580c0 00000400:00000200:24.0:1616513085.322070:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.13@tcp1 of length 224/224 into md 0xf97cb1d9 [1] + 0 00000400:00000200:24.0:1616513085.322075:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2440 00000400:00000200:24.0:1616513085.322077:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.322080:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513085.322085:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2440 00000800:00000200:24.0:1616513085.322088:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.322090:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.322092:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.322096:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.322099:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.322100:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513085.322111:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.13@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.322120:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.13@tcp1 of length 224 into portal 12 MB=0x5f911a9158140 00000400:00000200:20.0:1616513085.322126:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.13@tcp1 of length 224/224 into md 0x865bbec1 [1] + 77504 00000400:00000200:20.0:1616513085.322131:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.322134:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513085.322141:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.322144:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.322146:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.322149:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681229130072256 00010000:00000200:16.0:1616513085.322160:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1681229130072256/t0(0) o400->af54140d-94a6-355c-e651-b3d8177ae700@10.31.132.13@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.322174:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681229130072256, offset 192 00000400:00000200:16.0:1616513085.322180:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.13@tcp1 00000400:00000200:16.0:1616513085.322187:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.13@tcp1 routed destination 00000400:00000200:16.0:1616513085.322202:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.13@tcp1(10.31.132.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513085.322205:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681229130072384 00000800:00000200:16.0:1616513085.322209:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.322214:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.322217:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:0.0:1616513085.322217:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ced580 x1681229130072384/t0(0) o400->25c4011b-34b8-76d9-9b80-02e5d9cc1417@10.31.132.13@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.322219:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.322224:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.322229:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.322231:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513085.322231:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681229130072384, offset 192 00000400:00000200:0.0:1616513085.322237:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.13@tcp1 00000400:00000200:0.0:1616513085.322245:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.13@tcp1 routed destination 00000400:00000200:0.0:1616513085.322260:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.13@tcp1(10.31.132.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.322266:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.322272:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.322275:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.322277:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513085.322282:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513085.322282:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.322287:0:50025:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513085.322289:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.322292:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513085.322293:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.322298:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.322304:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2cc0 00000800:00000200:28.0:1616513085.322307:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513085.322342:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.322349:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.322352:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.322359:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.322363:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05e48 00000800:00000200:26.0:1616513085.322367:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.322370:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513085.322474:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725926992, portal 4 00000100:00000200:4.0:1616513085.322479:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725926992, offset 0 00000400:00000200:4.0:1616513085.322486:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:4.0:1616513085.322491:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:4.0:1616513085.322503:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513085.322510:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:4.0:1616513085.322515:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513085.322518:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:4.0:1616513085.322520:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:4.0:1616513085.322525:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513085.322573:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.322592:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513085.322598:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513085.322601:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513085.322608:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513085.322616:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4557cebf00 x1693067725926992/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513085.322628:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bacc0 00000800:00000200:1.0:1616513085.322635:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513085.322638:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513085.322699:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513085.322707:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.322713:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513085.322721:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513085.322730:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627250 00000400:00000200:1.0:1616513085.322737:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffd75 [1] + 192 00000400:00000200:1.0:1616513085.322742:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513085.322747:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4557cebf00 x1693067725926992/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513085.322758:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513085.322761:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.322764:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513085.322766:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:4.0:1616513085.322769:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4557cebf00 x1693067725926992/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513085.322782:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4557cebf00 x1693067725926992/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513085.322803:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00000800:00000200:11.2:1616513085.333813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513085.333870:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.333880:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513085.333889:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.61@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.333899:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.61@tcp1 of length 224 into portal 26 MB=0x5ffd415e8cc80 00000400:00000200:20.0:1616513085.333906:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.61@tcp1 of length 224/224 into md 0xf97cb219 [1] + 0 00000400:00000200:20.0:1616513085.333911:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2330 00000400:00000200:20.0:1616513085.333914:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.333916:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513085.333925:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2330 00000800:00000200:20.0:1616513085.333928:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.333928:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.333931:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.333935:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.333938:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.333938:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.333947:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.61@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.333956:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.61@tcp1 of length 224 into portal 12 MB=0x5ffd415e8cfc0 00000400:00000200:12.0:1616513085.333963:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.61@tcp1 of length 224/224 into md 0x865bbec1 [1] + 77728 00000400:00000200:12.0:1616513085.333968:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.333970:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513085.333977:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.333980:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.333982:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.333990:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661249281152 00010000:00000200:16.0:1616513085.334001:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688661249281152/t0(0) o400->bc523f91-0b87-a4a9-a50f-fdb69b2be79c@10.31.129.61@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.334016:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661249281152, offset 192 00000400:00000200:16.0:1616513085.334025:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.61@tcp1 00000400:00000200:16.0:1616513085.334033:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.61@tcp1 routed destination 00000400:00000200:16.0:1616513085.334048:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.61@tcp1(10.31.129.61@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.334054:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:0.0:1616513085.334058:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661249281984 00000800:00000200:16.0:1616513085.334060:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.334063:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.334065:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.334070:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:0.0:1616513085.334070:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d203f00 x1688661249281984/t0(0) o400->079bebf5-6d93-d09d-7860-8cb53bca5022@10.31.129.61@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513085.334075:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.334077:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513085.334083:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661249281984, offset 192 00000400:00000200:0.0:1616513085.334089:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.61@tcp1 00000400:00000200:0.0:1616513085.334096:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.61@tcp1 routed destination 00000400:00000200:0.0:1616513085.334109:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.61@tcp1(10.31.129.61@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.334115:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.334121:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.334123:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.334125:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.334130:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.334136:0:50026:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513085.334142:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.334151:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.334153:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513085.334154:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.334160:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.334166:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2908 00000800:00000200:28.0:1616513085.334169:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513085.334190:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.334197:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513085.334200:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513085.334207:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513085.334211:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb057e8 00000800:00000200:2.0:1616513085.334215:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.334218:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.341533:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513085.341589:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.341598:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.341607:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.194@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.341616:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.194@tcp1 of length 224 into portal 12 MB=0x5ffe9a262bf80 00000400:00000200:12.0:1616513085.341623:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.194@tcp1 of length 224/224 into md 0x865bbec1 [1] + 77952 00000400:00000200:12.0:1616513085.341628:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.341630:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513085.341637:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.341640:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.341642:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.341646:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.341708:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753800396672 00000800:00000200:11.2:1616513085.341714:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:0.0:1616513085.341719:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d200900 x1688753800396672/t0(0) o400->44b8c6a5-76fa-d602-7b5f-9a749064a6f1@10.31.128.194@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.341733:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688753800396672, offset 192 00000400:00000200:0.0:1616513085.341738:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.194@tcp1 00000400:00000200:0.0:1616513085.341746:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.194@tcp1 routed destination 00000400:00000200:0.0:1616513085.341759:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.194@tcp1(10.31.128.194@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.341765:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513085.341769:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.341770:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.341773:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513085.341775:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513085.341777:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.341779:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:20.0:1616513085.341785:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.194@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.341793:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.194@tcp1 of length 224 into portal 26 MB=0x5ffe9a262bc00 00000400:00000200:20.0:1616513085.341799:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.194@tcp1 of length 224/224 into md 0xf97cb229 [1] + 0 00000400:00000200:20.0:1616513085.341804:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf35c8 00000400:00000200:20.0:1616513085.341806:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.341809:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513085.341813:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf35c8 00000800:00000200:20.0:1616513085.341815:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513085.341820:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513085.341822:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513085.341829:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513085.341830:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513085.341832:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.341833:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513085.341840:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.341844:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04110 00000800:00000200:28.0:1616513085.341847:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.341875:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753800395776 00010000:00000200:16.0:1616513085.341885:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688753800395776/t0(0) o400->78c965de-4d05-e93d-7349-47c620d1f2b0@10.31.128.194@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.341898:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688753800395776, offset 192 00000400:00000200:16.0:1616513085.341904:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.194@tcp1 00000400:00000200:16.0:1616513085.341911:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.194@tcp1 routed destination 00000400:00000200:16.0:1616513085.341925:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.194@tcp1(10.31.128.194@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.341932:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.341937:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.341940:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.341942:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.341947:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513085.341953:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513085.341955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.342012:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.342019:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.342021:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.342028:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.342038:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b24c8 00000800:00000200:26.0:1616513085.342042:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.342044:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.342050:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513085.342052:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.342054:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.342055:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.342057:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.347415:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513085.347481:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.347490:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.347498:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.77@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.347508:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.77@tcp1 of length 224 into portal 26 MB=0x60195f1e0fb80 00000400:00000200:18.0:1616513085.347515:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.77@tcp1 of length 224/224 into md 0xf97cb239 [1] + 0 00000400:00000200:18.0:1616513085.347519:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3980 00000400:00000200:18.0:1616513085.347522:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.347525:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513085.347529:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3980 00000800:00000200:18.0:1616513085.347532:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.347535:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.347537:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.347539:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.347593:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690593380072320 00010000:00000200:16.0:1616513085.347603:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1690593380072320/t0(0) o400->f4559f53-7e52-d8b4-f762-9afa607144d0@10.31.140.77@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.347618:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690593380072320, offset 192 00000400:00000200:16.0:1616513085.347624:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.77@tcp1 00000400:00000200:16.0:1616513085.347632:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.77@tcp1 routed destination 00000800:00000200:7.2:1616513085.347640:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513085.347647:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.77@tcp1(10.31.140.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.347653:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.347661:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.347663:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.347665:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.347670:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513085.347677:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513085.347678:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513085.347696:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.347705:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.347714:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.77@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.347723:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.77@tcp1 of length 224 into portal 12 MB=0x60195f1e0fec0 00000400:00000200:14.0:1616513085.347730:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.77@tcp1 of length 224/224 into md 0xf964d519 [1] + 43336 00000800:00000200:12.0:1616513085.347731:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513085.347735:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513085.347737:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:14.0:1616513085.347738:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513085.347740:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513085.347745:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513085.347747:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513085.347749:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513085.347750:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513085.347752:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2cc0 00000800:00000200:14.0:1616513085.347753:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513085.347756:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.347759:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513085.347776:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690593380073152 00010000:00000200:22.0:1616513085.347789:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464627080 x1690593380073152/t0(0) o400->581bb045-be84-e24c-41e8-4e6ac2acf491@10.31.140.77@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.347803:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690593380073152, offset 192 00000400:00000200:22.0:1616513085.347810:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.77@tcp1 00000400:00000200:22.0:1616513085.347818:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.77@tcp1 routed destination 00000400:00000200:22.0:1616513085.347832:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.77@tcp1(10.31.140.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.347839:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.347845:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.347847:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.347850:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.347854:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.347859:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.347870:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.347874:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.347876:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513085.347878:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.347881:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.347885:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aff0 00000800:00000200:18.0:1616513085.347888:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.364826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.364885:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.364895:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513085.364901:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513085.364904:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.21@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.364913:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.21@tcp1 of length 224 into portal 26 MB=0x5ffdf04f6be80 00000400:00000200:2.0:1616513085.364919:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.21@tcp1 of length 224/224 into md 0xf97cb249 [1] + 0 00000400:00000200:2.0:1616513085.364924:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2ee0 00000400:00000200:2.0:1616513085.364926:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.364931:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.364935:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ee0 00000800:00000200:2.0:1616513085.364939:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.364942:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.364943:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.364998:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708209622656 00010000:00000200:16.0:1616513085.365008:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688708209622656/t0(0) o400->1a899004-3aca-65f0-17dc-9275a69d812e@10.31.140.21@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.365022:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688708209622656, offset 192 00000400:00000200:16.0:1616513085.365028:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.21@tcp1 00000400:00000200:16.0:1616513085.365034:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.21@tcp1 routed destination 00000400:00000200:16.0:1616513085.365048:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.21@tcp1(10.31.140.21@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.365055:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:11.2:1616513085.365058:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.365060:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.365063:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.365065:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513085.365069:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:16.0:1616513085.365076:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513085.365114:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.365124:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513085.365130:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:24.0:1616513085.365133:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.21@tcp1 : PUT - for me 00000800:00000200:28.0:1616513085.365137:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513085.365140:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.365142:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.21@tcp1 of length 224 into portal 12 MB=0x5ffdf04f6c080 00000400:00000200:28.0:1616513085.365147:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.365149:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.21@tcp1 of length 224/224 into md 0x865bbec1 [1] + 78176 00000400:00000200:24.0:1616513085.365154:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513085.365155:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3ed0 00000100:00000200:24.0:1616513085.365158:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513085.365159:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513085.365162:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.365165:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.365169:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.365170:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.365230:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708209623168 00010000:00000200:0.0:1616513085.365241:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd706780 x1688708209623168/t0(0) o400->437cbbf1-d123-ff16-65fc-dad3df3323a6@10.31.140.21@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.365255:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688708209623168, offset 192 00000400:00000200:0.0:1616513085.365261:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.21@tcp1 00000400:00000200:0.0:1616513085.365268:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.21@tcp1 routed destination 00000400:00000200:0.0:1616513085.365282:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.21@tcp1(10.31.140.21@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.365288:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.365293:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.365296:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.365298:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.365302:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.365308:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.365361:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.365369:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.365371:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.365378:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.365382:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:26.0:1616513085.365386:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.365389:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.370347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513085.370401:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.370410:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.370419:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.50@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.370428:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.50@tcp1 of length 224 into portal 26 MB=0x5ffd3a2f32680 00000400:00000200:14.0:1616513085.370448:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.50@tcp1 of length 224/224 into md 0xf97cb279 [1] + 0 00000400:00000200:14.0:1616513085.370453:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3320 00000800:00000200:2.0:1616513085.370453:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513085.370456:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513085.370458:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513085.370463:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:14.0:1616513085.370466:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.370469:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.370471:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.370528:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659320579712 00010000:00000200:16.0:1616513085.370539:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688659320579712/t0(0) o400->207ba809-d298-a8a2-f0ca-5435e5a05c46@10.31.138.50@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.370553:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688659320579712, offset 192 00000400:00000200:16.0:1616513085.370560:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.50@tcp1 00000400:00000200:16.0:1616513085.370567:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.50@tcp1 routed destination 00000400:00000200:16.0:1616513085.370582:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.50@tcp1(10.31.138.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.370589:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.370595:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.370597:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.370599:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.370604:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.370611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.370613:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513085.370653:0:50030:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513085.370666:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.370672:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.370675:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.370682:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.370687:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2000 00000800:00000200:20.0:1616513085.370691:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.370694:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513085.370708:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.370718:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.370726:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.50@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.370735:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.50@tcp1 of length 224 into portal 12 MB=0x5ffd3a2f32800 00000400:00000200:26.0:1616513085.370742:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.50@tcp1 of length 224/224 into md 0xf964d519 [1] + 43560 00000400:00000200:26.0:1616513085.370747:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.370751:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.370759:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.370761:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.370762:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.370765:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.370823:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659320580096 00010000:00000200:22.0:1616513085.370835:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b27b180 x1688659320580096/t0(0) o400->5a10f3ce-a41e-a255-b6d7-d7840353fb46@10.31.138.50@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.370848:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688659320580096, offset 192 00000400:00000200:22.0:1616513085.370855:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.50@tcp1 00000400:00000200:22.0:1616513085.370862:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.50@tcp1 routed destination 00000400:00000200:22.0:1616513085.370875:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.50@tcp1(10.31.138.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.370881:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.370887:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.370889:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.370891:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.370896:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.370902:0:50030:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.370912:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.370917:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513085.370920:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513085.370922:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513085.370926:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513085.370930:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4be48 00000800:00000200:2.0:1616513085.370934:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.407114:0:50034:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513085.407174:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.407191:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.407201:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.407212:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d0322b6580 00000400:00000200:18.0:1616513085.407218:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0xf964d519 [1] + 43784 00000400:00000200:18.0:1616513085.407224:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.407227:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513085.407235:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.407238:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.407238:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.407243:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.407301:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042590049664 00010000:00000200:22.0:1616513085.407388:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfcc80 x1693042590049664/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:711/0 lens 440/368 e 0 to 0 dl 1616513091 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513085.407403:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042590049664, offset 192 00000400:00000200:22.0:1616513085.407409:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:22.0:1616513085.407418:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:22.0:1616513085.407432:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.407452:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.407458:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.407460:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.407463:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.407467:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.407474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.407532:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.407539:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513085.407542:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.407549:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.407553:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4be48 00000800:00000200:24.0:1616513085.407557:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.407560:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.413893:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513085.413951:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.413961:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.413970:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.243@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.413979:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.243@tcp1 of length 224 into portal 12 MB=0x5ffd43d8c8880 00000400:00000200:14.0:1616513085.413987:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.243@tcp1 of length 224/224 into md 0xf964d519 [1] + 44224 00000400:00000200:14.0:1616513085.413993:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513085.413995:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513085.414002:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.414005:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.414007:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513085.414014:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513085.414017:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.414018:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.414026:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.243@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.414035:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.243@tcp1 of length 224 into portal 26 MB=0x5ffd43d8c8800 00000400:00000200:2.0:1616513085.414043:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.243@tcp1 of length 224/224 into md 0xf97cb2a9 [1] + 0 00000400:00000200:2.0:1616513085.414048:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3f58 00000400:00000200:2.0:1616513085.414050:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.414053:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.414058:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3f58 00000800:00000200:2.0:1616513085.414061:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.414064:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.414066:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.414067:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661914323072 00010000:00000200:22.0:1616513085.414079:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621b00 x1688661914323072/t0(0) o400->92f1e500-aa27-bcd2-2ea7-a909a8f99eab@10.31.132.243@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.414094:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661914323072, offset 192 00000400:00000200:22.0:1616513085.414100:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.243@tcp1 00000400:00000200:22.0:1616513085.414107:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.243@tcp1 routed destination 00000400:00000200:22.0:1616513085.414122:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.243@tcp1(10.31.132.243@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513085.414123:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661914322944 00000800:00000200:22.0:1616513085.414129:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.414135:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:16.0:1616513085.414135:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688661914322944/t0(0) o400->df157095-4082-689b-cd9e-b1652bcd52db@10.31.132.243@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513085.414138:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.414140:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.414145:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513085.414150:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661914322944, offset 192 00000800:00000200:7.2:1616513085.414150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513085.414156:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.243@tcp1 00000400:00000200:16.0:1616513085.414164:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.243@tcp1 routed destination 00000400:00000200:16.0:1616513085.414179:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.243@tcp1(10.31.132.243@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.414185:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.414191:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.414194:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.414196:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.414201:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513085.414204:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:16.0:1616513085.414207:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513085.414209:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513085.414212:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.414214:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513085.414215:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.414221:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.414225:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aff0 00000800:00000200:18.0:1616513085.414229:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513085.414264:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.414271:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.414274:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.414280:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.414286:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3210 00000800:00000200:20.0:1616513085.414290:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.414293:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.414298:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513085.414301:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.414303:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.414304:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.414307:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.415450:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.415512:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.415521:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.415530:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.415539:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 520 into portal 26 MB=0x603dfad656140 00000400:00000200:2.0:1616513085.415545:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 520/520 into md 0xf97cb2b9 [1] + 0 00000400:00000200:2.0:1616513085.415550:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2088 00000400:00000200:2.0:1616513085.415553:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.415555:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.415559:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:26.0:1616513085.415564:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.415564:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.415567:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.415569:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.415623:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109081956672 00010000:02000400:16.0:1616513085.415640:0:252364:0:(ldlm_lib.c:1150:target_handle_connect()) MGS: Received new LWP connection from 10.242.104.148@tcp1, removing former export from same NID 00010000:00080000:16.0:1616513085.415645:0:252364:0:(ldlm_lib.c:1226:target_handle_connect()) MGS: connection from d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1 t0 exp ffff8da4babad800 cur 1616513085 last 1616513085 00000400:00000200:16.0:1616513085.415661:0:252364:0:(peer.c:1109:LNetPrimaryNID()) NID 10.242.104.148@tcp1 primary NID 10.242.104.148@tcp1 rc 0 00000100:00080000:16.0:1616513085.415666:0:252364:0:(import.c:86:import_set_state_nolock()) ffff8d815a30c800 : changing import state from FULL to RECOVER 00000100:00080000:16.0:1616513085.415671:0:252364:0:(import.c:86:import_set_state_nolock()) ffff8d815a30c800 : changing import state from RECOVER to FULL 00000100:02000000:16.0:1616513085.415675:0:252364:0:(import.c:1595:ptlrpc_import_recovery_state_machine()) MGS: Connection restored to (at 10.242.104.148@tcp1) 00010000:00000200:16.0:1616513085.415683:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1693109081956672/t0(0) o250->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:0/0 lens 520/416 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.415696:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 384 bytes to portal 25, xid 1693109081956672, offset 0 00000400:00000200:16.0:1616513085.415702:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:16.0:1616513085.415708:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:16.0:1616513085.415723:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.415730:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 384 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.415736:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.415740:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.415742:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.415747:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.415753:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.415754:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513085.415805:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.415811:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.415814:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513085.415821:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.415826:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2dd0 00000800:00000200:14.0:1616513085.415830:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.415833:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.431235:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513085.431291:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.431302:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.431310:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.43@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.431320:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.43@tcp1 of length 224 into portal 26 MB=0x5ffd4e58a60c0 00000400:00000200:12.0:1616513085.431327:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.43@tcp1 of length 224/224 into md 0xf97cb2d9 [1] + 0 00000400:00000200:12.0:1616513085.431332:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2cc0 00000400:00000200:12.0:1616513085.431335:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.431337:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513085.431342:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2cc0 00000800:00000200:12.0:1616513085.431345:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.431346:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.431348:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.431350:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.431405:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664732754112 00010000:00000200:16.0:1616513085.431416:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688664732754112/t0(0) o400->8371793e-5f1f-6812-bbd4-2545c6a4d819@10.31.131.43@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.431431:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664732754112, offset 192 00000800:00000200:11.2:1616513085.431450:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:16.0:1616513085.431451:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.43@tcp1 00000400:00000200:16.0:1616513085.431459:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.43@tcp1 routed destination 00000400:00000200:16.0:1616513085.431474:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.43@tcp1(10.31.131.43@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.431481:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.431486:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.431489:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.431491:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513085.431496:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513085.431503:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:16.0:1616513085.431503:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513085.431513:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513085.431521:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.43@tcp1 : PUT - for me 00000400:00000200:28.0:1616513085.431530:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.43@tcp1 of length 224 into portal 12 MB=0x5ffd4e58a6140 00000400:00000200:28.0:1616513085.431536:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.43@tcp1 of length 224/224 into md 0x865bbec1 [1] + 78400 00000400:00000200:28.0:1616513085.431542:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513085.431544:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513085.431552:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513085.431555:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513085.431557:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:28.0:1616513085.431564:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513085.431565:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513085.431567:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513085.431568:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.431575:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.431580:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3a08 00000800:00000200:24.0:1616513085.431584:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.431623:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664732754240 00010000:00000200:0.0:1616513085.431635:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eb180 x1688664732754240/t0(0) o400->5cdd6c41-fc9e-2043-e0f7-ea6272987087@10.31.131.43@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.431650:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664732754240, offset 192 00000400:00000200:0.0:1616513085.431656:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.43@tcp1 00000400:00000200:0.0:1616513085.431663:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.43@tcp1 routed destination 00000400:00000200:0.0:1616513085.431680:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.43@tcp1(10.31.131.43@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.431687:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.431692:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.431695:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.431697:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.431702:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.431709:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.431763:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.431770:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.431773:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.431780:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.431784:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ba0 00000800:00000200:26.0:1616513085.431788:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.431791:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.450812:0:6482:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927008, portal 4 00000100:00000200:16.0:1616513085.450818:0:6482:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927008, offset 0 00000400:00000200:16.0:1616513085.450824:0:6482:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:16.0:1616513085.450829:0:6482:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:16.0:1616513085.450842:0:6482:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.450848:0:6482:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:16.0:1616513085.450854:0:6482:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.450857:0:6482:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:16.0:1616513085.450859:0:6482:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:16.0:1616513085.450863:0:6482:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513085.450910:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.450929:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513085.450935:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513085.450938:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513085.450945:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513085.450953:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb23c800 x1693067725927008/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513085.450966:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b38f8 00000800:00000200:1.0:1616513085.450971:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513085.450974:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513085.451088:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513085.451097:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.451104:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513085.451113:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513085.451121:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627260 00000400:00000200:1.0:1616513085.451128:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffd85 [1] + 192 00000400:00000200:1.0:1616513085.451134:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513085.451139:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb23c800 x1693067725927008/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513085.451148:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513085.451154:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.451157:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513085.451161:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:16.0:1616513085.451163:0:6482:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb23c800 x1693067725927008/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513085.451176:0:6482:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb23c800 x1693067725927008/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513085.451197:0:6482:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2198 00000800:00000200:11.2:1616513085.456980:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513085.457037:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.457047:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.457057:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.62@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.457065:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.62@tcp1 of length 224 into portal 26 MB=0x5fb0319c7bd00 00000400:00000200:12.0:1616513085.457072:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.62@tcp1 of length 224/224 into md 0xf97cb2e9 [1] + 0 00000400:00000200:12.0:1616513085.457077:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3c28 00000400:00000200:12.0:1616513085.457080:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.457082:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513085.457087:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3c28 00000800:00000200:12.0:1616513085.457090:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.457092:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.457093:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.457096:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.457150:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1683365619547392 00010000:00000200:16.0:1616513085.457161:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1683365619547392/t0(0) o400->4ee799b6-6e52-98d3-db52-7bb70323b11b@10.31.132.62@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.457176:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1683365619547392, offset 192 00000400:00000200:16.0:1616513085.457183:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.62@tcp1 00000400:00000200:16.0:1616513085.457190:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.62@tcp1 routed destination 00000400:00000200:16.0:1616513085.457205:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.62@tcp1(10.31.132.62@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.457212:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.457217:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.457220:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.457222:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.457227:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.457232:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.457234:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513085.457287:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.457294:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.457297:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.457303:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.457309:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2198 00000800:00000200:20.0:1616513085.457312:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.457315:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.457319:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513085.457330:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.457338:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513085.457340:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.457348:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.62@tcp1 : PUT - for me 00000400:00000200:28.0:1616513085.457356:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.62@tcp1 of length 224 into portal 12 MB=0x5fb0319c7bdc0 00000400:00000200:28.0:1616513085.457365:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.62@tcp1 of length 224/224 into md 0x865bbec1 [1] + 78624 00000400:00000200:28.0:1616513085.457370:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513085.457373:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513085.457380:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.457383:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.457385:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.457455:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1683365619547584 00010000:00000200:0.0:1616513085.457468:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ef980 x1683365619547584/t0(0) o400->eb549c9c-af4c-0622-666f-8b3e907633b0@10.31.132.62@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.457482:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1683365619547584, offset 192 00000400:00000200:0.0:1616513085.457489:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.62@tcp1 00000400:00000200:0.0:1616513085.457496:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.62@tcp1 routed destination 00000400:00000200:0.0:1616513085.457511:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.62@tcp1(10.31.132.62@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.457518:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.457524:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.457527:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.457529:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.457534:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.457542:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.457598:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.457606:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.457609:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513085.457616:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.457620:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05320 00000800:00000200:14.0:1616513085.457624:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.457627:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.471967:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513085.472023:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.472032:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513085.472041:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.6@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.472051:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.6@tcp1 of length 224 into portal 26 MB=0x5ffdd09fdbac0 00000400:00000200:20.0:1616513085.472057:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.6@tcp1 of length 224/224 into md 0xf97cb2f9 [1] + 0 00000400:00000200:20.0:1616513085.472064:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2c38 00000400:00000200:20.0:1616513085.472067:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.472070:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513085.472075:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:12.0:1616513085.472076:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.472078:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.472081:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.472083:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.472139:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699704031936 00010000:00000200:16.0:1616513085.472149:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688699704031936/t0(0) o400->7a59ddb5-e3b6-84e7-0cd9-257e96094e46@10.31.136.6@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.472164:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699704031936, offset 192 00000400:00000200:16.0:1616513085.472170:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.6@tcp1 00000400:00000200:16.0:1616513085.472178:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.6@tcp1 routed destination 00000400:00000200:16.0:1616513085.472193:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.6@tcp1(10.31.136.6@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.472199:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.472205:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.472208:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.472210:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.472215:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.472220:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.472221:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513085.472275:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.472282:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513085.472284:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.472291:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.472296:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2330 00000800:00000200:24.0:1616513085.472300:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.472304:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.473443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513085.473521:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.473531:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.473539:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.6@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.473548:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.6@tcp1 of length 224 into portal 12 MB=0x5ffdd09fdbc80 00000400:00000200:12.0:1616513085.473554:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.6@tcp1 of length 224/224 into md 0x865bbec1 [1] + 78848 00000400:00000200:12.0:1616513085.473559:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.473562:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513085.473575:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.473577:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.473578:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.473582:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.473640:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699704032384 00010000:00000200:0.0:1616513085.473652:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d206300 x1688699704032384/t0(0) o400->774c4b76-03aa-f6a8-3cc8-a4876f34bab3@10.31.136.6@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.473666:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699704032384, offset 192 00000400:00000200:0.0:1616513085.473673:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.6@tcp1 00000400:00000200:0.0:1616513085.473680:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.6@tcp1 routed destination 00000400:00000200:0.0:1616513085.473695:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.6@tcp1(10.31.136.6@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.473702:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.473708:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.473710:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.473712:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.473719:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.473725:0:50041:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.473780:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.473787:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.473790:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.473796:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.473800:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05dc0 00000800:00000200:26.0:1616513085.473804:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.473807:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.473813:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513085.473816:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.473818:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.473819:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.473822:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513085.482478:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927024, portal 4 00000100:00000200:6.0:1616513085.482484:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927024, offset 0 00000400:00000200:6.0:1616513085.482490:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:6.0:1616513085.482496:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:6.0:1616513085.482509:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513085.482516:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:6.0:1616513085.482522:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513085.482525:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:15.2:1616513085.482526:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:6.0:1616513085.482527:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:6.0:1616513085.482532:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:1.0:1616513085.482583:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:1.0:1616513085.482593:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.151@o2ib1 00000800:00000200:5.2:1616513085.482595:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:1.0:1616513085.482603:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.151@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513085.482612:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.151@o2ib1 of length 648 into portal 12 MB=0x5f9586dcfb540 00000400:00000200:1.0:1616513085.482619:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.151@o2ib1 of length 648/648 into md 0x42efdd85 [1] + 38512 00000400:00000200:1.0:1616513085.482625:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513085.482628:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:1.0:1616513085.482637:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:9.0:1616513085.482638:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:1.0:1616513085.482640:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (23)-- 00000800:00000200:1.0:1616513085.482642:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000800:00000200:24.0:1616513085.482651:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513085.482657:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:24.0:1616513085.482660:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.482667:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:24.0:1616513085.482673:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf81680 x1693067725927024/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513085.482685:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00770 00000800:00000200:24.0:1616513085.482689:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513085.482692:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:29.0:1616513085.482709:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681533078320448 00000800:00000200:5.2:1616513085.482751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513085.482804:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513085.482813:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513085.482822:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00010000:00000200:29.0:1616513085.482828:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d853e8cd580 x1681533078320448/t77331494630(0) o101->67eb3afe-7574-8f15-013a-b1df12d1e90c@10.242.46.151@o2ib1:711/0 lens 648/592 e 0 to 0 dl 1616513091 ref 1 fl Interpret:/0/0 rc 301/0 00000400:00000200:20.0:1616513085.482830:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627270 00000400:00000200:20.0:1616513085.482837:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d8581 [1] + 192 00000400:00000200:20.0:1616513085.482842:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:29.0:1616513085.482845:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 560 bytes to portal 10, xid 1681533078320448, offset 192 00000100:00000200:20.0:1616513085.482847:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf81680 x1693067725927024/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:29.0:1616513085.482852:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.151@o2ib1 00000800:00000200:12.0:1616513085.482858:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:29.0:1616513085.482860:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.151@o2ib1 local destination 00000800:00000200:20.0:1616513085.482862:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513085.482865:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513085.482867:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:6.0:1616513085.482869:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf81680 x1693067725927024/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:29.0:1616513085.482874:0:170085:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.151@o2ib1(10.242.46.151@o2ib1:10.242.46.151@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513085.482881:0:170085:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 560 bytes in 1 frags to 12345-10.242.46.151@o2ib1 00000800:00000200:29.0:1616513085.482889:0:170085:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4319f80] -> 10.242.46.151@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513085.482892:0:170085:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (20)++ 00000100:00000200:6.0:1616513085.482892:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf81680 x1693067725927024/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:29.0:1616513085.482894:0:170085:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87e8600] (21)++ 00000400:00000200:6.0:1616513085.482899:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:29.0:1616513085.482900:0:170085:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:15.2:1616513085.482905:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:5.0:1616513085.482918:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:5.0:1616513085.482923:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87e8600] (23)-- 00000400:00000200:5.0:1616513085.482925:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513085.482927:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000400:00000200:5.0:1616513085.482931:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.151@o2ib1: PUT: OK 00000400:00000200:5.0:1616513085.482935:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2bb0 00000800:00000200:5.0:1616513085.482940:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000800:00000200:15.2:1616513085.485306:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:9.0:1616513085.485360:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:9.0:1616513085.485369:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.151@o2ib1 00000400:00000200:9.0:1616513085.485378:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.151@o2ib1 : PUT - for me 00000400:00000200:9.0:1616513085.485385:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.151@o2ib1 of length 392 into portal 23 MB=0x5f9586dcfb580 00000400:00000200:9.0:1616513085.485392:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 17 from 12345-10.242.46.151@o2ib1 of length 392/392 into md 0x431afa15 [1] + 1960 00000400:00000200:9.0:1616513085.485397:0:6466:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d84cfa51cb0 00000400:00000200:9.0:1616513085.485400:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:9.0:1616513085.485403:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt_readpage 00000400:00000200:9.0:1616513085.485408:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84cfa51cb0 00000800:00000200:9.0:1616513085.485412:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:1.0:1616513085.485412:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:9.0:1616513085.485414:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (23)-- 00000800:00000200:9.0:1616513085.485416:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000100:00000200:5.0:1616513085.485476:0:183225:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681533078320512 00010000:00000200:5.0:1616513085.485544:0:183225:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cc0639050 x1681533078320512/t77331494631(0) o35->67eb3afe-7574-8f15-013a-b1df12d1e90c@10.242.46.151@o2ib1:711/0 lens 392/456 e 0 to 0 dl 1616513091 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:5.0:1616513085.485559:0:183225:0:(niobuf.c:85:ptl_send_buf()) Sending 424 bytes to portal 10, xid 1681533078320512, offset 192 00000400:00000200:5.0:1616513085.485566:0:183225:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.151@o2ib1 00000400:00000200:5.0:1616513085.485576:0:183225:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.151@o2ib1 local destination 00000400:00000200:5.0:1616513085.485590:0:183225:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.151@o2ib1(10.242.46.151@o2ib1:10.242.46.151@o2ib1) : PUT try# 0 00000800:00000200:5.0:1616513085.485596:0:183225:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 424 bytes in 1 frags to 12345-10.242.46.151@o2ib1 00000800:00000200:5.0:1616513085.485602:0:183225:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4319f80] -> 10.242.46.151@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513085.485605:0:183225:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:5.0:1616513085.485607:0:183225:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:5.0:1616513085.485612:0:183225:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:15.2:1616513085.485617:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:1.0:1616513085.485626:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)++ 00000100:00000200:5.0:1616513085.485627:0:183225:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 23 00000800:00000200:1.0:1616513085.485632:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87e8600] (23)-- 00000400:00000200:1.0:1616513085.485635:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:9.0:1616513085.485636:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000400:00000200:1.0:1616513085.485642:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.151@o2ib1: PUT: OK 00000400:00000200:1.0:1616513085.485649:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55e7f8 00000800:00000200:1.0:1616513085.485652:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000800:00000200:7.2:1616513085.489345:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513085.489415:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.489424:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.489450:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.204@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.489459:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.204@tcp1 of length 224 into portal 26 MB=0x604a2d498a480 00000400:00000200:18.0:1616513085.489466:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.204@tcp1 of length 224/224 into md 0xf97cb309 [1] + 0 00000800:00000200:26.0:1616513085.489468:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513085.489471:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf38f8 00000400:00000200:18.0:1616513085.489474:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.489476:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513085.489479:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.489481:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000800:00000200:18.0:1616513085.489484:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513085.489487:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513085.489488:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.204@tcp1 : PUT - for me 00000800:00000200:18.0:1616513085.489489:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.489492:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513085.489496:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.204@tcp1 of length 224 into portal 12 MB=0x604a2d498a500 00000400:00000200:26.0:1616513085.489503:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.204@tcp1 of length 224/224 into md 0xf964d519 [1] + 44448 00000400:00000200:26.0:1616513085.489508:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.489511:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.489520:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.489523:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.489527:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.489556:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947258250368 00010000:00000200:16.0:1616513085.489566:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1693947258250368/t0(0) o400->6ced7f92-25f6-8a32-814e-07ded8d8c0af@10.31.140.204@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.489581:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693947258250368, offset 192 00000400:00000200:16.0:1616513085.489587:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.204@tcp1 00000100:00000200:22.0:1616513085.489588:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947258250496 00000400:00000200:16.0:1616513085.489594:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.204@tcp1 routed destination 00010000:00000200:22.0:1616513085.489600:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8480 x1693947258250496/t0(0) o400->9e11c1a6-af6a-b896-3152-7ddd5c0e21d6@10.31.140.204@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513085.489608:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.204@tcp1(10.31.140.204@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513085.489614:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693947258250496, offset 192 00000800:00000200:16.0:1616513085.489614:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513085.489620:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.204@tcp1 00000800:00000200:16.0:1616513085.489620:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.489622:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.489624:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513085.489628:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.204@tcp1 routed destination 00000800:00000200:16.0:1616513085.489629:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513085.489636:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513085.489643:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.204@tcp1(10.31.140.204@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.489649:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.489655:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.489658:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.489662:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.489667:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.489672:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513085.489677:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513085.489725:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.489732:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.489735:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513085.489736:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513085.489741:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513085.489743:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513085.489745:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bed0 00000800:00000200:24.0:1616513085.489750:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513085.489750:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513085.489752:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.489759:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.489765:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2220 00000800:00000200:24.0:1616513085.489768:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.489771:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.498167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513085.498223:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.498232:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.498242:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.150@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.498251:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.150@tcp1 of length 224 into portal 26 MB=0x603e0f1963500 00000400:00000200:12.0:1616513085.498258:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.150@tcp1 of length 224/224 into md 0xf97cb319 [1] + 0 00000400:00000200:12.0:1616513085.498263:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3b18 00000400:00000200:12.0:1616513085.498265:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.498268:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513085.498272:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:12.0:1616513085.498275:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.498276:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.498278:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.498280:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.498340:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114520974592 00000800:00000200:11.2:1616513085.498345:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:16.0:1616513085.498351:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1693114520974592/t0(0) o400->56f5e351-9127-2a57-6c6a-a5d50f2b2e60@10.31.136.150@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.498366:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114520974592, offset 192 00000400:00000200:16.0:1616513085.498372:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.150@tcp1 00000400:00000200:16.0:1616513085.498380:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.150@tcp1 routed destination 00000400:00000200:16.0:1616513085.498395:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.150@tcp1(10.31.136.150@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513085.498399:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.498401:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.498407:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513085.498408:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.498410:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513085.498412:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:24.0:1616513085.498417:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.150@tcp1 : PUT - for me 00000800:00000200:16.0:1616513085.498417:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:16.0:1616513085.498423:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513085.498426:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.150@tcp1 of length 224 into portal 12 MB=0x603e0f1963640 00000400:00000200:24.0:1616513085.498432:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.150@tcp1 of length 224/224 into md 0x865bbec1 [1] + 79072 00000400:00000200:24.0:1616513085.498449:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.498452:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513085.498453:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513085.498458:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:28.0:1616513085.498461:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:24.0:1616513085.498462:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000400:00000200:28.0:1616513085.498464:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513085.498464:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.498466:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.498470:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.498476:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3cb0 00000800:00000200:28.0:1616513085.498479:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513085.498511:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927040, portal 4 00000100:00000200:26.0:1616513085.498516:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927040, offset 0 00000400:00000200:26.0:1616513085.498523:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:26.0:1616513085.498528:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:26.0:1616513085.498541:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513085.498541:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114520974912 00000800:00000200:26.0:1616513085.498547:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:26.0:1616513085.498553:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513085.498553:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd700d80 x1693114520974912/t0(0) o400->98ebac48-e948-eae7-f77d-4a362e9dc30f@10.31.136.150@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:26.0:1616513085.498556:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513085.498559:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513085.498564:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:0.0:1616513085.498578:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114520974912, offset 192 00000400:00000200:0.0:1616513085.498584:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.150@tcp1 00000400:00000200:0.0:1616513085.498591:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.150@tcp1 routed destination 00000400:00000200:0.0:1616513085.498605:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.150@tcp1(10.31.136.150@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:25.2:1616513085.498611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:0.0:1616513085.498611:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.498617:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.498619:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.498621:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.498625:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.498626:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:7.2:1616513085.498633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.498633:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513085.498636:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513085.498641:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513085.498642:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:26.0:1616513085.498644:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:2.0:1616513085.498649:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf81200 x1693067725927040/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513085.498650:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.498652:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.498657:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.498660:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04ee0 00000400:00000200:2.0:1616513085.498660:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afdc0 00000800:00000200:26.0:1616513085.498663:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.498665:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:26.0:1616513085.498668:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:25.2:1616513085.498782:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513085.498792:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513085.498798:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:26.0:1616513085.498799:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513085.498806:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513085.498813:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627280 00000400:00000200:2.0:1616513085.498820:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cba09 [1] + 192 00000400:00000200:2.0:1616513085.498824:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.498829:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf81200 x1693067725927040/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513085.498842:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513085.498845:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513085.498847:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:26.0:1616513085.498848:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf81200 x1693067725927040/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513085.498860:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf81200 x1693067725927040/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513085.498869:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa08 00000800:00000200:11.2:1616513085.517783:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513085.517839:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.517848:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.517857:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.148@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.517865:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.148@tcp1 of length 224 into portal 26 MB=0x603e0e9012900 00000400:00000200:12.0:1616513085.517872:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.148@tcp1 of length 224/224 into md 0xf97cb329 [1] + 0 00000400:00000200:12.0:1616513085.517876:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2198 00000400:00000200:12.0:1616513085.517879:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.517882:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513085.517886:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2198 00000800:00000200:12.0:1616513085.517889:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.517892:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.517893:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513085.517900:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.517903:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.517904:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.517912:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.148@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.517921:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.148@tcp1 of length 224 into portal 12 MB=0x603e0e90129c0 00000400:00000200:24.0:1616513085.517927:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.148@tcp1 of length 224/224 into md 0x865bbec1 [1] + 79296 00000400:00000200:24.0:1616513085.517932:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.517937:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513085.517944:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.517948:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513085.517949:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114376988928 00000800:00000200:24.0:1616513085.517950:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:16.0:1616513085.517960:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1693114376988928/t0(0) o400->47a81b2f-93e7-2971-b41e-542b87917a45@10.31.136.148@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.517975:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114376988928, offset 192 00000400:00000200:16.0:1616513085.517982:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.148@tcp1 00000400:00000200:16.0:1616513085.517989:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.148@tcp1 routed destination 00000400:00000200:16.0:1616513085.518006:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.148@tcp1(10.31.136.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513085.518006:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114376989120 00000800:00000200:16.0:1616513085.518013:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.518018:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513085.518018:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e3a80 x1693114376989120/t0(0) o400->3775cac5-168c-f88a-f735-6657f2e0f7dc@10.31.136.148@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.518021:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.518023:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.518028:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513085.518031:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114376989120, offset 192 00000800:00000200:11.2:1616513085.518033:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.518035:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513085.518037:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.148@tcp1 00000400:00000200:0.0:1616513085.518044:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.148@tcp1 routed destination 00000400:00000200:0.0:1616513085.518058:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.148@tcp1(10.31.136.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.518064:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513085.518069:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.518072:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.518074:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513085.518078:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.518085:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513085.518087:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.518094:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.518097:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513085.518098:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.518104:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.518109:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3cb0 00000800:00000200:28.0:1616513085.518113:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513085.518140:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.518147:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.518149:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513085.518156:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.518160:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04330 00000800:00000200:18.0:1616513085.518164:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.518167:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.533882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.533938:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.533947:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.533955:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.185@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.533965:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.185@tcp1 of length 224 into portal 26 MB=0x604a2942c6a40 00000400:00000200:26.0:1616513085.533971:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.185@tcp1 of length 224/224 into md 0xf97cb339 [1] + 0 00000400:00000200:26.0:1616513085.533975:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2000 00000400:00000200:26.0:1616513085.533978:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.533982:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513085.533986:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2000 00000800:00000200:26.0:1616513085.533989:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.533991:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.533992:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.533996:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.534010:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513085.534062:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946177415744 00010000:00000200:16.0:1616513085.534073:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1693946177415744/t0(0) o400->dba3a3f3-fca9-d5c1-1397-9d2c3ddc9187@10.31.140.185@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513085.534076:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.534085:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:16.0:1616513085.534088:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946177415744, offset 192 00000400:00000200:18.0:1616513085.534094:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.185@tcp1 : PUT - for me 00000400:00000200:16.0:1616513085.534094:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.185@tcp1 00000400:00000200:16.0:1616513085.534102:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.185@tcp1 routed destination 00000400:00000200:18.0:1616513085.534103:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.185@tcp1 of length 224 into portal 12 MB=0x604a2942c6d40 00000400:00000200:18.0:1616513085.534109:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.185@tcp1 of length 224/224 into md 0xf964d519 [1] + 44672 00000400:00000200:18.0:1616513085.534115:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513085.534117:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.185@tcp1(10.31.140.185@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:18.0:1616513085.534119:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513085.534124:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513085.534126:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.534128:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.534129:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513085.534129:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.534132:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513085.534133:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513085.534134:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.534139:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513085.534145:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513085.534185:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513085.534191:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946177416512 00010000:00000200:22.0:1616513085.534203:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfba80 x1693946177416512/t0(0) o400->10ea4eac-afeb-b287-94d6-f146eda321f4@10.31.140.185@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.534217:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946177416512, offset 192 00000400:00000200:22.0:1616513085.534224:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.185@tcp1 00000400:00000200:22.0:1616513085.534231:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.185@tcp1 routed destination 00000800:00000200:24.0:1616513085.534242:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513085.534246:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.185@tcp1(10.31.140.185@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513085.534249:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513085.534251:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513085.534253:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513085.534258:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513085.534258:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.534261:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.534263:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513085.534264:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2330 00000800:00000200:24.0:1616513085.534267:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513085.534268:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513085.534270:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.534273:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.534284:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.534289:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513085.534291:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513085.534294:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513085.534298:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513085.534302:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ac38 00000800:00000200:2.0:1616513085.534306:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:15.2:1616513085.542011:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:1.0:1616513085.542066:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:1.0:1616513085.542076:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.151@o2ib1 00000400:00000200:1.0:1616513085.542086:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.151@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513085.542094:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.151@o2ib1 of length 648 into portal 12 MB=0x5f9586dcfb5c0 00000400:00000200:1.0:1616513085.542101:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.151@o2ib1 of length 648/648 into md 0x42efdd85 [1] + 39160 00000400:00000200:1.0:1616513085.542106:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513085.542109:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:1.0:1616513085.542117:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:9.0:1616513085.542119:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:1.0:1616513085.542120:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (23)-- 00000800:00000200:1.0:1616513085.542123:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000100:00000200:29.0:1616513085.542182:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681533078320576 00010000:00000200:29.0:1616513085.542266:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d853e8c8900 x1681533078320576/t77331494632(0) o101->67eb3afe-7574-8f15-013a-b1df12d1e90c@10.242.46.151@o2ib1:711/0 lens 648/592 e 0 to 0 dl 1616513091 ref 1 fl Interpret:/0/0 rc 301/0 00000100:00000200:29.0:1616513085.542293:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 560 bytes to portal 10, xid 1681533078320576, offset 192 00000400:00000200:29.0:1616513085.542299:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.151@o2ib1 00000400:00000200:29.0:1616513085.542306:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.151@o2ib1 local destination 00000400:00000200:29.0:1616513085.542321:0:170085:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.151@o2ib1(10.242.46.151@o2ib1:10.242.46.151@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513085.542327:0:170085:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 560 bytes in 1 frags to 12345-10.242.46.151@o2ib1 00000800:00000200:29.0:1616513085.542333:0:170085:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4319f80] -> 10.242.46.151@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513085.542336:0:170085:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:29.0:1616513085.542338:0:170085:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:29.0:1616513085.542343:0:170085:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:15.2:1616513085.542348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:9.0:1616513085.542358:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:9.0:1616513085.542363:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87e8600] (23)-- 00000400:00000200:9.0:1616513085.542366:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513085.542367:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000400:00000200:9.0:1616513085.542372:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.151@o2ib1: PUT: OK 00000400:00000200:9.0:1616513085.542376:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2ee0 00000800:00000200:9.0:1616513085.542381:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000800:00000200:7.2:1616513085.543648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513085.543709:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.543719:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.543728:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.120@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.543737:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.120@tcp1 of length 224 into portal 12 MB=0x6046d825e6c00 00000400:00000200:14.0:1616513085.543744:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.120@tcp1 of length 224/224 into md 0xf964d519 [1] + 44896 00000400:00000200:14.0:1616513085.543751:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513085.543754:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513085.543760:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.543763:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.543764:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513085.543768:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.543824:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693718245436416 00010000:00000200:22.0:1616513085.543836:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464624380 x1693718245436416/t0(0) o400->bc879bad-376c-876c-8756-2e1a396437e4@10.31.138.120@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.543850:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693718245436416, offset 192 00000400:00000200:22.0:1616513085.543857:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.120@tcp1 00000400:00000200:22.0:1616513085.543864:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.120@tcp1 routed destination 00000400:00000200:22.0:1616513085.543880:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.120@tcp1(10.31.138.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.543886:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.543892:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.543895:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.543897:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.543901:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.543909:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.543964:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.543971:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.543974:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513085.543983:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.543987:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b650 00000800:00000200:12.0:1616513085.543990:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.543993:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:15.2:1616513085.544093:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:9.0:1616513085.544147:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:9.0:1616513085.544157:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.151@o2ib1 00000400:00000200:9.0:1616513085.544166:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.151@o2ib1 : PUT - for me 00000400:00000200:9.0:1616513085.544173:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.151@o2ib1 of length 392 into portal 23 MB=0x5f9586dcfb600 00000400:00000200:9.0:1616513085.544180:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 17 from 12345-10.242.46.151@o2ib1 of length 392/392 into md 0x431cbbe5 [1] + 0 00000400:00000200:9.0:1616513085.544185:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:9.0:1616513085.544188:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt_readpage 00000800:00000200:9.0:1616513085.544196:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:9.0:1616513085.544199:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (23)-- 00000800:00000200:1.0:1616513085.544199:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (23)-- 00000800:00000200:9.0:1616513085.544204:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000100:00000200:5.0:1616513085.544260:0:183225:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681533078320640 00010000:00000200:5.0:1616513085.544306:0:183225:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4558ad3600 x1681533078320640/t77331494633(0) o35->67eb3afe-7574-8f15-013a-b1df12d1e90c@10.242.46.151@o2ib1:711/0 lens 392/456 e 0 to 0 dl 1616513091 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:5.0:1616513085.544322:0:183225:0:(niobuf.c:85:ptl_send_buf()) Sending 424 bytes to portal 10, xid 1681533078320640, offset 192 00000400:00000200:5.0:1616513085.544328:0:183225:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.151@o2ib1 00000400:00000200:5.0:1616513085.544336:0:183225:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.151@o2ib1 local destination 00000400:00000200:5.0:1616513085.544350:0:183225:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.151@o2ib1(10.242.46.151@o2ib1:10.242.46.151@o2ib1) : PUT try# 0 00000800:00000200:5.0:1616513085.544356:0:183225:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 424 bytes in 1 frags to 12345-10.242.46.151@o2ib1 00000800:00000200:5.0:1616513085.544362:0:183225:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4319f80] -> 10.242.46.151@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513085.544365:0:183225:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:5.0:1616513085.544367:0:183225:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:5.0:1616513085.544374:0:183225:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:15.2:1616513085.544379:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:1.0:1616513085.544388:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:1.0:1616513085.544394:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87e8600] (23)-- 00000800:00000200:9.0:1616513085.544397:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (23)-- 00000400:00000200:1.0:1616513085.544398:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513085.544405:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.151@o2ib1: PUT: OK 00000400:00000200:1.0:1616513085.544410:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55ed48 00000800:00000200:1.0:1616513085.544414:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000100:00000200:21.0:1616513085.562586:0:6490:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927056, portal 4 00000100:00000200:21.0:1616513085.562592:0:6490:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927056, offset 0 00000400:00000200:21.0:1616513085.562598:0:6490:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:21.0:1616513085.562604:0:6490:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:21.0:1616513085.562617:0:6490:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:21.0:1616513085.562623:0:6490:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:21.0:1616513085.562629:0:6490:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513085.562634:0:6490:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:21.0:1616513085.562636:0:6490:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:21.0:1616513085.562641:0:6490:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513085.562691:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.562708:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513085.562713:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513085.562715:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513085.562722:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513085.562729:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843f7bde80 x1693067725927056/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513085.562741:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50550 00000800:00000200:1.0:1616513085.562745:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513085.562748:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513085.562879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513085.562887:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.562893:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513085.562897:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513085.562902:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513085.562909:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627290 00000400:00000200:1.0:1616513085.562916:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffde5 [1] + 192 00000400:00000200:1.0:1616513085.562921:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513085.562925:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843f7bde80 x1693067725927056/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513085.562939:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513085.562942:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513085.562944:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:21.0:1616513085.562946:0:6490:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843f7bde80 x1693067725927056/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:21.0:1616513085.562959:0:6490:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843f7bde80 x1693067725927056/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:21.0:1616513085.562969:0:6490:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51650 00000800:00000200:11.2:1616513085.592275:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513085.592333:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.592343:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.592352:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.41@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.592362:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.41@tcp1 of length 224 into portal 26 MB=0x5ffd3d609cfc0 00000400:00000200:24.0:1616513085.592369:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.41@tcp1 of length 224/224 into md 0xf97cb359 [1] + 0 00000400:00000200:24.0:1616513085.592374:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf23b8 00000400:00000200:24.0:1616513085.592376:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.592379:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513085.592384:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:20.0:1616513085.592386:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.592387:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.592390:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.592392:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.592462:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660177702848 00010000:00000200:16.0:1616513085.592473:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688660177702848/t0(0) o400->885cf53d-d762-a9e8-b0da-71e03c6cc5dc@10.31.129.41@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.592488:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660177702848, offset 192 00000400:00000200:16.0:1616513085.592494:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.41@tcp1 00000400:00000200:16.0:1616513085.592502:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.41@tcp1 routed destination 00000400:00000200:16.0:1616513085.592518:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.41@tcp1(10.31.129.41@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.592525:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.592531:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.592535:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.592538:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.592543:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.592549:0:50054:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.592550:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513085.592605:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.592612:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.592614:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.592621:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.592627:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b38f8 00000800:00000200:26.0:1616513085.592630:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.592633:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.592693:0:50054:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513085.592749:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.592758:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.592767:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.41@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.592776:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.41@tcp1 of length 224 into portal 12 MB=0x5ffd3d609d340 00000400:00000200:18.0:1616513085.592782:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.41@tcp1 of length 224/224 into md 0xf964d519 [1] + 45120 00000400:00000200:18.0:1616513085.592787:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.592790:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513085.592797:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.592799:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513085.592801:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.592805:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.592862:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660177703744 00010000:00000200:22.0:1616513085.592874:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf9200 x1688660177703744/t0(0) o400->371246f8-5643-a2cf-9d52-b4eb6f2fe35d@10.31.129.41@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.592888:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660177703744, offset 192 00000400:00000200:22.0:1616513085.592897:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.41@tcp1 00000400:00000200:22.0:1616513085.592904:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.41@tcp1 routed destination 00000400:00000200:22.0:1616513085.592919:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.41@tcp1(10.31.129.41@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.592926:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.592932:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.592934:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.592937:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.592942:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.592947:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.593001:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.593008:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.593011:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513085.593017:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.593021:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aaa0 00000800:00000200:12.0:1616513085.593025:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.593028:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.599945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.600000:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.600009:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.600018:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.26@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.600027:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.26@tcp1 of length 224 into portal 26 MB=0x5ffd34e0d9680 00000400:00000200:26.0:1616513085.600034:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.26@tcp1 of length 224/224 into md 0xf97cb369 [1] + 0 00000400:00000200:26.0:1616513085.600039:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2770 00000400:00000200:26.0:1616513085.600041:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.600044:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513085.600048:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:26.0:1616513085.600051:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.600053:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.600054:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.600057:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.600112:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657896248960 00010000:00000200:16.0:1616513085.600123:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688657896248960/t0(0) o400->2b720994-ce0a-fcbc-a6fc-27cb27733999@10.31.129.26@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.600137:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657896248960, offset 192 00000400:00000200:16.0:1616513085.600143:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.26@tcp1 00000400:00000200:16.0:1616513085.600151:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.26@tcp1 routed destination 00000400:00000200:16.0:1616513085.600166:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.26@tcp1(10.31.129.26@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.600172:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.600178:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.600181:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.600183:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.600188:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.600193:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.600194:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513085.600246:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.600253:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.600255:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513085.600262:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.600267:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2dd0 00000800:00000200:14.0:1616513085.600271:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.600274:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.601712:0:50055:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.601767:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.601776:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.601785:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.105@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.601794:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.105@tcp1 of length 224 into portal 26 MB=0x6002560bd3e00 00000400:00000200:2.0:1616513085.601802:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.105@tcp1 of length 224/224 into md 0xf97cb379 [1] + 0 00000400:00000200:2.0:1616513085.601807:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2bb0 00000400:00000200:2.0:1616513085.601810:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.601812:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.601817:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:2.0:1616513085.601820:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.601831:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.601833:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.601839:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.601842:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.601843:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.601851:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.105@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.601859:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.105@tcp1 of length 224 into portal 12 MB=0x6002560bd3e80 00000400:00000200:26.0:1616513085.601865:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.105@tcp1 of length 224/224 into md 0xf964d519 [1] + 45344 00000400:00000200:26.0:1616513085.601870:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.601872:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.601878:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.601882:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.601884:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.601888:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689010397068800 00010000:00000200:16.0:1616513085.601899:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1689010397068800/t0(0) o400->8db5ddfb-81ef-fde0-0403-43fea8b47715@10.31.142.105@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.601913:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689010397068800, offset 192 00000400:00000200:16.0:1616513085.601920:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.105@tcp1 00000400:00000200:16.0:1616513085.601927:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.105@tcp1 routed destination 00000100:00000200:22.0:1616513085.601942:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689010397068928 00000400:00000200:16.0:1616513085.601942:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.105@tcp1(10.31.142.105@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.601949:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:22.0:1616513085.601954:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dd580 x1689010397068928/t0(0) o400->7fd6ab2c-aedf-272a-2fbf-b0c348296903@10.31.142.105@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.601954:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.601957:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.601959:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.601964:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513085.601968:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689010397068928, offset 192 00000100:00000200:16.0:1616513085.601970:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513085.601975:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.105@tcp1 00000400:00000200:22.0:1616513085.601982:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.105@tcp1 routed destination 00000400:00000200:22.0:1616513085.601996:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.105@tcp1(10.31.142.105@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.602003:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.602009:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:11.2:1616513085.602009:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.602012:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.602014:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.602018:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.602023:0:50055:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.602034:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.602040:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.602042:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513085.602044:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.602048:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.602052:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b320 00000800:00000200:18.0:1616513085.602056:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513085.602065:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.602072:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.602074:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.602081:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.602087:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2000 00000800:00000200:20.0:1616513085.602090:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.602093:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.603850:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.603902:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.603911:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.603919:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.26@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.603927:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.26@tcp1 of length 224 into portal 12 MB=0x5ffd34e0d9a40 00000400:00000200:26.0:1616513085.603933:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.26@tcp1 of length 224/224 into md 0xf964d519 [1] + 45568 00000400:00000200:26.0:1616513085.603937:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.603940:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.603946:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.603949:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.603950:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.603953:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.604009:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657896249920 00010000:00000200:22.0:1616513085.604022:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647de300 x1688657896249920/t0(0) o400->25d275ac-ce18-05e0-394b-226a070094bf@10.31.129.26@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.604036:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657896249920, offset 192 00000400:00000200:22.0:1616513085.604043:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.26@tcp1 00000400:00000200:22.0:1616513085.604050:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.26@tcp1 routed destination 00000400:00000200:22.0:1616513085.604065:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.26@tcp1(10.31.129.26@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.604071:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.604077:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.604081:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.604083:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.604088:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.604135:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.604191:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.604198:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.604201:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513085.604207:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.604211:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba08 00000800:00000200:12.0:1616513085.604215:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.604219:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.618458:0:50057:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.618516:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.618525:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.618534:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.45@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.618542:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.45@tcp1 of length 224 into portal 26 MB=0x6018497da71c0 00000400:00000200:2.0:1616513085.618549:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.45@tcp1 of length 224/224 into md 0xf97cb3a9 [1] + 0 00000400:00000200:2.0:1616513085.618554:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3188 00000400:00000200:2.0:1616513085.618557:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.618560:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.618564:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000800:00000200:2.0:1616513085.618567:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.618568:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.618570:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.618572:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.618627:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690518855250368 00010000:00000200:16.0:1616513085.618638:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1690518855250368/t0(0) o400->3cce4db9-bf74-438b-e392-39260f8a47aa@10.31.129.45@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.618653:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690518855250368, offset 192 00000400:00000200:16.0:1616513085.618659:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.45@tcp1 00000400:00000200:16.0:1616513085.618666:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.45@tcp1 routed destination 00000400:00000200:16.0:1616513085.618681:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.45@tcp1(10.31.129.45@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.618688:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.618694:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.618697:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.618699:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.618703:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.618709:0:50057:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.618710:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513085.618762:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.618768:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.618771:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513085.618778:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.618783:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2cc0 00000800:00000200:14.0:1616513085.618787:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.618790:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.619033:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513085.619088:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.619098:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513085.619106:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.45@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.619115:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.45@tcp1 of length 224 into portal 12 MB=0x6018497da76c0 00000400:00000200:20.0:1616513085.619123:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.45@tcp1 of length 224/224 into md 0x865bbec1 [1] + 79520 00000400:00000200:20.0:1616513085.619129:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.619131:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513085.619138:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.619141:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.619142:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.619146:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.619202:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690518855251648 00010000:00000200:0.0:1616513085.619214:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceb600 x1690518855251648/t0(0) o400->e5cb26d3-8a33-b9d8-b557-56d7c6731aa5@10.31.129.45@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.619229:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690518855251648, offset 192 00000400:00000200:0.0:1616513085.619235:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.45@tcp1 00000400:00000200:0.0:1616513085.619244:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.45@tcp1 routed destination 00000400:00000200:0.0:1616513085.619259:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.45@tcp1(10.31.129.45@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.619266:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.619271:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.619274:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.619276:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.619281:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.619286:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.619340:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.619347:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.619350:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513085.619356:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.619360:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb053a8 00000800:00000200:12.0:1616513085.619364:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.619367:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.639979:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513085.640034:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.640043:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.640051:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.71@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.640059:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.71@tcp1 of length 224 into portal 12 MB=0x5ffdd2d971600 00000400:00000200:18.0:1616513085.640065:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.71@tcp1 of length 224/224 into md 0xf964d519 [1] + 45792 00000400:00000200:18.0:1616513085.640070:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.640072:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513085.640078:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.640081:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.640083:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.640085:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.640140:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700301284864 00010000:00000200:22.0:1616513085.640152:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf8480 x1688700301284864/t0(0) o400->48c133b8-3ab4-69db-4070-a3988c6fa81c@10.31.143.71@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.640165:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700301284864, offset 192 00000400:00000200:22.0:1616513085.640172:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.71@tcp1 00000400:00000200:22.0:1616513085.640179:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.71@tcp1 routed destination 00000400:00000200:22.0:1616513085.640192:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.71@tcp1(10.31.143.71@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.640198:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.640203:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.640206:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.640208:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.640212:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.640217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.640271:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.640278:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.640281:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513085.640287:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.640291:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a110 00000800:00000200:14.0:1616513085.640296:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.640298:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.641315:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513085.641371:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.641381:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.641390:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.71@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.641398:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.71@tcp1 of length 224 into portal 26 MB=0x5ffdd2d971400 00000400:00000200:24.0:1616513085.641404:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.71@tcp1 of length 224/224 into md 0xf97cb3d9 [1] + 0 00000400:00000200:24.0:1616513085.641409:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3100 00000400:00000200:24.0:1616513085.641412:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.641415:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513085.641419:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:24.0:1616513085.641424:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.641425:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.641427:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.641429:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.641461:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513085.641481:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700301284352 00010000:00000200:16.0:1616513085.641491:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688700301284352/t0(0) o400->4e4e306d-391a-34a0-70bf-bec1c20a48f5@10.31.143.71@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.641504:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700301284352, offset 192 00000400:00000200:16.0:1616513085.641510:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.71@tcp1 00000800:00000200:7.2:1616513085.641515:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513085.641517:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.71@tcp1 routed destination 00000400:00000200:16.0:1616513085.641530:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.71@tcp1(10.31.143.71@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.641536:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513085.641538:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.641541:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.641544:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513085.641546:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513085.641548:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.641550:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:12.0:1616513085.641556:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.61@tcp1 : PUT - for me 00000100:00000200:16.0:1616513085.641557:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513085.641565:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.61@tcp1 of length 224 into portal 12 MB=0x5ffdb7cb44e80 00000800:00000200:18.0:1616513085.641569:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513085.641572:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.61@tcp1 of length 224/224 into md 0x865bbec1 [1] + 79744 00000400:00000200:12.0:1616513085.641577:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513085.641579:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:12.0:1616513085.641579:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513085.641586:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:18.0:1616513085.641588:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.61@tcp1 : PUT - for me 00000800:00000200:12.0:1616513085.641589:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513085.641592:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.641595:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513085.641597:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.61@tcp1 of length 224 into portal 26 MB=0x5ffdb7cb44d80 00000800:00000200:12.0:1616513085.641599:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.641601:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513085.641603:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513085.641604:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.61@tcp1 of length 224/224 into md 0xf97cb3f9 [1] + 0 00000400:00000200:12.0:1616513085.641607:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.641609:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3e48 00000400:00000200:12.0:1616513085.641611:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2cc0 00000400:00000200:18.0:1616513085.641612:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.641614:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513085.641614:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513085.641619:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000800:00000200:18.0:1616513085.641622:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.641625:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.641626:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.641629:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.641631:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693038665088 00010000:00000200:16.0:1616513085.641637:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688693038665088/t0(0) o400->efd3b048-2860-89e9-8a67-2a3d9e5c95e4@10.31.140.61@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.641647:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693038665088, offset 192 00000400:00000200:16.0:1616513085.641653:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.61@tcp1 00000400:00000200:16.0:1616513085.641659:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.61@tcp1 routed destination 00000100:00000200:0.0:1616513085.641659:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693038665344 00000400:00000200:16.0:1616513085.641671:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.61@tcp1(10.31.140.61@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513085.641671:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d203600 x1688693038665344/t0(0) o400->769f92b0-c367-f8f3-e6ae-814a394e4151@10.31.140.61@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.641676:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.641681:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.641683:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.641685:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513085.641685:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693038665344, offset 192 00000800:00000200:7.2:1616513085.641697:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513085.641697:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.61@tcp1 00000800:00000200:16.0:1616513085.641698:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513085.641703:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513085.641705:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.61@tcp1 routed destination 00000400:00000200:0.0:1616513085.641720:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.61@tcp1(10.31.140.61@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.641726:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.641732:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.641735:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.641737:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.641741:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.641746:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513085.641751:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513085.641757:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513085.641758:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513085.641761:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513085.641761:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513085.641763:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513085.641766:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513085.641766:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513085.641768:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.641768:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.641773:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2dd0 00000400:00000200:24.0:1616513085.641776:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:14.0:1616513085.641777:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513085.641780:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.670855:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513085.670910:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.670919:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.670928:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.52@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.670937:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.52@tcp1 of length 224 into portal 12 MB=0x5ffdce26e0ec0 00000400:00000200:18.0:1616513085.670944:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.52@tcp1 of length 224/224 into md 0xf964d519 [1] + 46016 00000400:00000200:18.0:1616513085.670949:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.670952:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513085.670959:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.670962:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.670963:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.670967:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.671023:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699040304832 00000800:00000200:11.2:1616513085.671031:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:22.0:1616513085.671035:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfec00 x1688699040304832/t0(0) o400->0c1b10cd-6641-dc3a-f2fc-dd1c0baf9299@10.31.142.52@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.671050:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699040304832, offset 192 00000400:00000200:22.0:1616513085.671056:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.52@tcp1 00000400:00000200:22.0:1616513085.671063:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.52@tcp1 routed destination 00000400:00000200:22.0:1616513085.671078:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.52@tcp1(10.31.142.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.671085:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.671090:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.671093:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513085.671093:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.671095:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.671100:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513085.671102:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513085.671105:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513085.671111:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.52@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.671119:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.52@tcp1 of length 224 into portal 26 MB=0x5ffdce26e0b40 00000400:00000200:20.0:1616513085.671126:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.52@tcp1 of length 224/224 into md 0xf97cb419 [1] + 0 00000400:00000200:20.0:1616513085.671131:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf34b8 00000400:00000200:20.0:1616513085.671134:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.671137:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513085.671141:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf34b8 00000800:00000200:20.0:1616513085.671145:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.671146:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.671148:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.671150:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513085.671167:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.671174:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.671177:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.671183:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.671187:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a4c8 00000800:00000200:26.0:1616513085.671191:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.671194:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.671209:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699040303936 00010000:00000200:16.0:1616513085.671219:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688699040303936/t0(0) o400->de9d087c-96f2-e1a1-d502-9a71c99acf74@10.31.142.52@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.671234:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699040303936, offset 192 00000400:00000200:16.0:1616513085.671241:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.52@tcp1 00000400:00000200:16.0:1616513085.671248:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.52@tcp1 routed destination 00000400:00000200:16.0:1616513085.671264:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.52@tcp1(10.31.142.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.671270:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.671276:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.671278:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.671281:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.671285:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.671291:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.671292:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513085.671343:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.671354:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.671357:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513085.671363:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.671369:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2330 00000800:00000200:28.0:1616513085.671372:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.671375:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.673100:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513085.673155:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.673164:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.673173:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.4@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.673182:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.4@tcp1 of length 224 into portal 26 MB=0x5ffd36b9e4f00 00000400:00000200:12.0:1616513085.673189:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.4@tcp1 of length 224/224 into md 0xf97cb429 [1] + 0 00000400:00000200:12.0:1616513085.673193:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2f68 00000400:00000200:12.0:1616513085.673196:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.673199:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513085.673203:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2f68 00000800:00000200:12.0:1616513085.673206:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.673207:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.673209:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.673212:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.673265:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658392272640 00010000:00000200:16.0:1616513085.673276:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688658392272640/t0(0) o400->90034473-2d5a-4397-5112-45faa1647fd9@10.31.138.4@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.673291:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658392272640, offset 192 00000400:00000200:16.0:1616513085.673297:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.4@tcp1 00000400:00000200:16.0:1616513085.673305:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.4@tcp1 routed destination 00000400:00000200:16.0:1616513085.673320:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.4@tcp1(10.31.138.4@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.673326:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.673334:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.673336:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.673339:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.673343:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.673348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.673350:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513085.673394:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513085.673401:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.673407:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513085.673410:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513085.673416:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513085.673422:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2330 00000800:00000200:2.0:1616513085.673426:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.673428:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513085.673459:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.673468:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513085.673477:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.4@tcp1 : PUT - for me 00000400:00000200:28.0:1616513085.673486:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.4@tcp1 of length 224 into portal 12 MB=0x5ffd36b9e5000 00000400:00000200:28.0:1616513085.673492:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.4@tcp1 of length 224/224 into md 0x865bbec1 [1] + 79968 00000400:00000200:28.0:1616513085.673497:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513085.673499:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513085.673508:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.673511:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.673513:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513085.673517:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.673569:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658392272896 00010000:00000200:0.0:1616513085.673581:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ebf00 x1688658392272896/t0(0) o400->a943c55d-603d-85f0-c950-27d331090fe9@10.31.138.4@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.673594:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658392272896, offset 192 00000400:00000200:0.0:1616513085.673600:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.4@tcp1 00000400:00000200:0.0:1616513085.673607:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.4@tcp1 routed destination 00000400:00000200:0.0:1616513085.673621:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.4@tcp1(10.31.138.4@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.673627:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.673632:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.673634:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.673638:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.673642:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.673648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.673658:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.673663:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.673666:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513085.673667:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513085.673673:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.673677:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000800:00000200:20.0:1616513085.673681:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.724757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513085.724826:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.724836:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.724845:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.158@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.724855:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.158@tcp1 of length 224 into portal 26 MB=0x5ffd5e54c9900 00000400:00000200:24.0:1616513085.724862:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.158@tcp1 of length 224/224 into md 0xf97cb449 [1] + 0 00000400:00000200:24.0:1616513085.724867:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3210 00000400:00000200:24.0:1616513085.724870:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.724873:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513085.724877:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3210 00000800:00000200:28.0:1616513085.724880:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.724881:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.724883:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.724887:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513085.724890:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513085.724890:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.724899:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.158@tcp1 : PUT - for me 00000400:00000200:28.0:1616513085.724909:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.158@tcp1 of length 224 into portal 12 MB=0x5ffd5e54c9980 00000400:00000200:28.0:1616513085.724915:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.158@tcp1 of length 224/224 into md 0x865bbec1 [1] + 80192 00000400:00000200:28.0:1616513085.724920:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513085.724923:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513085.724930:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.724933:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.724935:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.724953:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688669023672576 00010000:00000200:16.0:1616513085.724965:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688669023672576/t0(0) o400->23f05589-2e85-146b-5bda-8248d4a9621b@10.31.138.158@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.724979:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688669023672576, offset 192 00000400:00000200:16.0:1616513085.724985:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.158@tcp1 00000400:00000200:16.0:1616513085.724993:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.158@tcp1 routed destination 00000100:00000200:0.0:1616513085.724994:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688669023672704 00010000:00000200:0.0:1616513085.725007:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eba80 x1688669023672704/t0(0) o400->29f5f964-a987-4140-9b72-b083b15e7420@10.31.138.158@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513085.725009:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.158@tcp1(10.31.138.158@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.725016:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.725022:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513085.725022:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688669023672704, offset 192 00000800:00000200:16.0:1616513085.725025:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.725027:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513085.725028:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.158@tcp1 00000800:00000200:16.0:1616513085.725032:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513085.725036:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.158@tcp1 routed destination 00000100:00000200:16.0:1616513085.725039:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513085.725040:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513085.725051:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.158@tcp1(10.31.138.158@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.725057:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.725063:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.725065:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.725067:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.725072:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.725078:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513085.725096:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.725104:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.725106:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.725113:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.725119:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2cc0 00000800:00000200:26.0:1616513085.725123:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.725126:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513085.725130:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.725138:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.725140:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513085.725141:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513085.725147:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.725151:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000800:00000200:20.0:1616513085.725155:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.738494:0:50070:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.738551:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.738570:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.738579:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.151@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.738589:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.151@tcp1 of length 224 into portal 26 MB=0x603fcd95ab740 00000400:00000200:2.0:1616513085.738598:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.151@tcp1 of length 224/224 into md 0xf97cb479 [1] + 0 00000400:00000200:2.0:1616513085.738603:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3078 00000400:00000200:2.0:1616513085.738605:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.738608:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513085.738609:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513085.738613:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:2.0:1616513085.738617:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513085.738620:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513085.738620:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513085.738621:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.738624:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.738629:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.151@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.738638:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.151@tcp1 of length 224 into portal 12 MB=0x603fcd95ab7c0 00000400:00000200:18.0:1616513085.738645:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.151@tcp1 of length 224/224 into md 0xf964d519 [1] + 46240 00000400:00000200:18.0:1616513085.738650:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.738653:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513085.738661:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.738664:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.738668:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.738679:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234373506880 00010000:00000200:16.0:1616513085.738689:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1693234373506880/t0(0) o400->dcaac93c-1ce6-8d67-5c08-042fb93e5fdb@10.31.138.151@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.738705:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234373506880, offset 192 00000400:00000200:16.0:1616513085.738711:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.151@tcp1 00000400:00000200:16.0:1616513085.738719:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.151@tcp1 routed destination 00000100:00000200:22.0:1616513085.738728:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234373507008 00000400:00000200:16.0:1616513085.738734:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.151@tcp1(10.31.138.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:22.0:1616513085.738741:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bad00 x1693234373507008/t0(0) o400->df875a4a-f26e-64c9-b6d7-41b46161a8d6@10.31.138.151@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.738741:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.738747:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.738750:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.738752:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.738757:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.738761:0:50070:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513085.738766:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234373507008, offset 192 00000100:00000200:16.0:1616513085.738766:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513085.738773:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.151@tcp1 00000400:00000200:22.0:1616513085.738780:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.151@tcp1 routed destination 00000400:00000200:22.0:1616513085.738795:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.151@tcp1(10.31.138.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.738802:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.738807:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.738810:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.738812:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513085.738816:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513085.738817:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513085.738823:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513085.738825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513085.738827:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513085.738828:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513085.738835:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.738840:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2cc0 00000800:00000200:14.0:1616513085.738844:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513085.738882:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.738889:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.738891:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513085.738894:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.738898:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.738902:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a908 00000800:00000200:28.0:1616513085.738906:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.739206:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513085.739259:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.739268:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.739277:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.134@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.739287:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.134@tcp1 of length 224 into portal 26 MB=0x5ffdde1dbeb80 00000400:00000200:24.0:1616513085.739294:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.134@tcp1 of length 224/224 into md 0xf97cb499 [1] + 0 00000400:00000200:24.0:1616513085.739299:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3a90 00000400:00000200:24.0:1616513085.739302:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.739305:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513085.739309:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:24.0:1616513085.739314:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.739314:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.739317:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513085.739321:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.739324:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.739324:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:16.0:1616513085.739324:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703325694848 00010000:00000200:16.0:1616513085.739330:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688703325694848/t0(0) o400->92be92ea-1533-dea2-ce62-f9d035182e01@10.31.142.134@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513085.739333:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.134@tcp1 : PUT - for me 00000100:00000200:16.0:1616513085.739341:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703325694848, offset 192 00000400:00000200:20.0:1616513085.739342:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.134@tcp1 of length 224 into portal 12 MB=0x5ffdde1dbec00 00000400:00000200:16.0:1616513085.739347:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.134@tcp1 00000400:00000200:20.0:1616513085.739352:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.134@tcp1 of length 224/224 into md 0x865bbec1 [1] + 80416 00000400:00000200:16.0:1616513085.739353:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.134@tcp1 routed destination 00000400:00000200:20.0:1616513085.739357:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.739360:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513085.739366:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.134@tcp1(10.31.142.134@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513085.739367:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.739371:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.739372:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513085.739372:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.739377:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.739379:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.739381:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.739384:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513085.739388:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513085.739388:0:50070:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.739398:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.739402:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.739404:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513085.739408:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.739410:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.739415:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7893a8 00000800:00000200:18.0:1616513085.739418:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513085.739432:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703325694976 00010000:00000200:0.0:1616513085.739459:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cebf00 x1688703325694976/t0(0) o400->455ce563-336c-91cf-6380-e17508ebfefe@10.31.142.134@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.739474:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703325694976, offset 192 00000400:00000200:0.0:1616513085.739480:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.134@tcp1 00000400:00000200:0.0:1616513085.739487:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.134@tcp1 routed destination 00000400:00000200:0.0:1616513085.739502:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.134@tcp1(10.31.142.134@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.739509:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.739514:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.739517:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.739519:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.739524:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.739530:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.739539:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.739542:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.739544:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513085.739547:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.739550:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.739552:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:28.0:1616513085.739555:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.744434:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513085.744488:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.744498:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.744506:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.15@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.744515:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.15@tcp1 of length 224 into portal 26 MB=0x604a46870bc40 00000400:00000200:12.0:1616513085.744522:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.15@tcp1 of length 224/224 into md 0xf97cb4a9 [1] + 0 00000400:00000200:12.0:1616513085.744526:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2220 00000400:00000200:12.0:1616513085.744529:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.744532:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513085.744536:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:12.0:1616513085.744539:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.744541:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.744542:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.744548:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.744551:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513085.744551:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513085.744560:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.15@tcp1 : PUT - for me 00000400:00000200:20.0:1616513085.744569:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.15@tcp1 of length 224 into portal 12 MB=0x604a46870bcc0 00000400:00000200:20.0:1616513085.744575:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.15@tcp1 of length 224/224 into md 0x865bbec1 [1] + 80640 00000400:00000200:20.0:1616513085.744580:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.744582:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513085.744590:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.744593:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.744595:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513085.744610:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954033630272 00010000:00000200:16.0:1616513085.744621:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1693954033630272/t0(0) o400->fdf6fb50-03a0-0a8a-686e-2f4537be931b@10.31.141.15@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.744635:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954033630272, offset 192 00000400:00000200:16.0:1616513085.744641:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.15@tcp1 00000400:00000200:16.0:1616513085.744648:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.15@tcp1 routed destination 00000100:00000200:0.0:1616513085.744654:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954033630400 00000400:00000200:16.0:1616513085.744663:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.15@tcp1(10.31.141.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513085.744667:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38e780 x1693954033630400/t0(0) o400->760e9175-6340-1694-21b4-12f195d5ad77@10.31.141.15@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.744670:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.744676:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.744678:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.744681:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513085.744681:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954033630400, offset 192 00000800:00000200:16.0:1616513085.744685:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513085.744689:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.15@tcp1 00000100:00000200:16.0:1616513085.744692:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513085.744692:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513085.744696:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.15@tcp1 routed destination 00000400:00000200:0.0:1616513085.744711:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.15@tcp1(10.31.141.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.744717:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.744723:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.744725:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.744728:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.744732:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.744738:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513085.744748:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.744755:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513085.744758:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513085.744764:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513085.744770:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788cc0 00000800:00000200:2.0:1616513085.744773:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.744776:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513085.744792:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.744800:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.744802:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513085.744803:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.744809:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.744813:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb053a8 00000800:00000200:28.0:1616513085.744817:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.746677:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513085.746702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513085.746730:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.746739:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.746747:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.169@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.746755:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.169@tcp1 of length 224 into portal 12 MB=0x5fcd68b1ad580 00000800:00000200:20.0:1616513085.746760:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513085.746761:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.169@tcp1 of length 224/224 into md 0xf964d519 [1] + 46464 00000400:00000200:26.0:1616513085.746768:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513085.746769:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:26.0:1616513085.746771:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.746777:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513085.746777:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.169@tcp1 : PUT - for me 00000800:00000200:26.0:1616513085.746779:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.746781:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.746784:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513085.746786:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.169@tcp1 of length 224 into portal 26 MB=0x5fcd68b1ad4c0 00000400:00000200:20.0:1616513085.746793:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.169@tcp1 of length 224/224 into md 0xf97cb4c9 [1] + 0 00000400:00000200:20.0:1616513085.746798:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf25d8 00000400:00000200:20.0:1616513085.746801:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513085.746803:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513085.746808:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000800:00000200:20.0:1616513085.746811:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.746813:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.746815:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.746820:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513085.746848:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1685373270545792 00010000:00000200:22.0:1616513085.746860:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dde80 x1685373270545792/t0(0) o400->68c4be1a-a89a-4942-8c4b-43a2383b810b@10.31.138.169@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513085.746873:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1685373270545600 00000100:00000200:22.0:1616513085.746875:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1685373270545792, offset 192 00000400:00000200:22.0:1616513085.746881:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.169@tcp1 00010000:00000200:16.0:1616513085.746883:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1685373270545600/t0(0) o400->b4af3be9-4bae-107b-c927-48afa8d1a8bb@10.31.138.169@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513085.746889:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.169@tcp1 routed destination 00000100:00000200:16.0:1616513085.746897:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1685373270545600, offset 192 00000400:00000200:22.0:1616513085.746904:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.169@tcp1(10.31.138.169@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513085.746904:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.169@tcp1 00000800:00000200:22.0:1616513085.746910:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513085.746911:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.169@tcp1 routed destination 00000800:00000200:22.0:1616513085.746916:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.746919:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.746921:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513085.746925:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.169@tcp1(10.31.138.169@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.746926:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513085.746931:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513085.746931:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.746937:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.746939:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.746941:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.746946:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.746951:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.746952:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513085.746962:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.746968:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.746971:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513085.746977:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.746982:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788990 00000800:00000200:2.0:1616513085.746984:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513085.746985:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.746988:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513085.746991:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513085.746994:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513085.747001:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513085.747005:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a000 00000800:00000200:2.0:1616513085.747009:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.747012:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.756574:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.756629:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.756638:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.756646:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.3@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.756655:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.3@tcp1 of length 224 into portal 26 MB=0x601950ff72800 00000400:00000200:26.0:1616513085.756661:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.3@tcp1 of length 224/224 into md 0xf97cb4d9 [1] + 0 00000400:00000200:26.0:1616513085.756666:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2660 00000400:00000200:26.0:1616513085.756668:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.756671:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513085.756675:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:26.0:1616513085.756678:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.756680:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.756682:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.756684:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.756738:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589589874688 00010000:00000200:16.0:1616513085.756748:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1690589589874688/t0(0) o400->ba34902b-43dc-ee83-7cda-8e274e75e56d@10.31.141.3@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.756761:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690589589874688, offset 192 00000400:00000200:16.0:1616513085.756767:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.3@tcp1 00000400:00000200:16.0:1616513085.756774:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.3@tcp1 routed destination 00000400:00000200:16.0:1616513085.756788:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.3@tcp1(10.31.141.3@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.756794:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.756799:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.756802:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.756804:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.756809:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.756813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.756815:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513085.756868:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.756875:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513085.756877:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513085.756884:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513085.756890:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788dd0 00000800:00000200:2.0:1616513085.756893:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.756896:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.757284:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.757294:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.757299:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513085.757302:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513085.757306:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.3@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.757311:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.3@tcp1 of length 224 into portal 12 MB=0x601950ff72ec0 00000400:00000200:26.0:1616513085.757316:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.3@tcp1 of length 224/224 into md 0xf964d519 [1] + 46688 00000400:00000200:26.0:1616513085.757320:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.757322:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.757327:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.757329:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.757330:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.757392:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589589876416 00010000:00000200:22.0:1616513085.757404:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dd100 x1690589589876416/t0(0) o400->3f3f4de6-7908-8d14-5f66-05a438de06b9@10.31.141.3@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.757418:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690589589876416, offset 192 00000400:00000200:22.0:1616513085.757425:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.3@tcp1 00000400:00000200:22.0:1616513085.757432:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.3@tcp1 routed destination 00000400:00000200:22.0:1616513085.757465:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.3@tcp1(10.31.141.3@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.757474:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.757480:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.757492:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.757494:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.757499:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.757506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.757560:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.757567:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.757570:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.757576:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.757580:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba90 00000800:00000200:20.0:1616513085.757584:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.757587:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.787064:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.787122:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.787132:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.787141:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.154@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.787152:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.154@tcp1 of length 224 into portal 26 MB=0x603e152cb0b00 00000400:00000200:2.0:1616513085.787161:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.154@tcp1 of length 224/224 into md 0xf97cb4e9 [1] + 0 00000400:00000200:2.0:1616513085.787166:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb42a8 00000400:00000200:2.0:1616513085.787168:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.787171:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.787176:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb42a8 00000800:00000200:14.0:1616513085.787178:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.787179:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.787183:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.787185:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.787240:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116151827200 00010000:00000200:16.0:1616513085.787252:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1693116151827200/t0(0) o400->7e612509-a2bc-3437-718a-c5cf00bc3fa0@10.31.136.154@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.787265:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693116151827200, offset 192 00000400:00000200:16.0:1616513085.787271:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.154@tcp1 00000400:00000200:16.0:1616513085.787279:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.154@tcp1 routed destination 00000400:00000200:16.0:1616513085.787294:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.154@tcp1(10.31.136.154@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.787300:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.787308:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.787311:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.787313:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.787318:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513085.787325:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513085.787325:0:50075:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.787380:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.787387:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.787390:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513085.787397:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.787402:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7894b8 00000800:00000200:18.0:1616513085.787406:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.787409:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.787435:0:50075:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513085.787448:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.787455:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513085.787458:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513085.787464:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.154@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.787473:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.154@tcp1 of length 224 into portal 12 MB=0x603e152cb0c00 00000400:00000200:14.0:1616513085.787480:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.154@tcp1 of length 224/224 into md 0xf964d519 [1] + 46912 00000400:00000200:14.0:1616513085.787485:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513085.787488:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513085.787495:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.787498:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.787500:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.787585:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116151827456 00010000:00000200:22.0:1616513085.787597:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464622400 x1693116151827456/t0(0) o400->0b0634d5-aea9-c0e3-b17d-ff4c9e0319c9@10.31.136.154@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513085.787610:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693116151827456, offset 192 00000400:00000200:22.0:1616513085.787616:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.154@tcp1 00000400:00000200:22.0:1616513085.787623:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.154@tcp1 routed destination 00000400:00000200:22.0:1616513085.787637:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.154@tcp1(10.31.136.154@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.787643:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.787649:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.787651:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.787653:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.787658:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.787665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.787679:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.787686:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513085.787689:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513085.787695:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.787699:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a550 00000800:00000200:12.0:1616513085.787703:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.787706:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.789904:0:50075:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513085.789961:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513085.789970:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513085.789979:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.107@tcp1 : PUT - for me 00000400:00000200:18.0:1616513085.789989:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.107@tcp1 of length 224 into portal 26 MB=0x5fff9e65b0ec0 00000400:00000200:18.0:1616513085.789996:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.107@tcp1 of length 224/224 into md 0xf97cb4f9 [1] + 0 00000400:00000200:18.0:1616513085.790000:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5650 00000400:00000200:18.0:1616513085.790003:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.790005:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513085.790010:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5650 00000800:00000200:18.0:1616513085.790013:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.790014:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.790016:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.790021:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513085.790024:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.790025:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.790033:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.107@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.790042:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.107@tcp1 of length 224 into portal 12 MB=0x5fff9e65b0f80 00000400:00000200:26.0:1616513085.790055:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.107@tcp1 of length 224/224 into md 0xf964d519 [1] + 47136 00000400:00000200:26.0:1616513085.790060:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.790063:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.790069:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.790072:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.790074:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.790074:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688823660220096 00010000:00000200:16.0:1616513085.790085:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688823660220096/t0(0) o400->20b7999d-de16-c5fa-490a-d97e0d874a7d@10.31.138.107@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.790100:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688823660220096, offset 192 00000400:00000200:16.0:1616513085.790106:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.107@tcp1 00000400:00000200:16.0:1616513085.790114:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.107@tcp1 routed destination 00000400:00000200:16.0:1616513085.790128:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.107@tcp1(10.31.138.107@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513085.790133:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688823660220288 00000800:00000200:16.0:1616513085.790135:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.790141:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.790143:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:22.0:1616513085.790145:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9200 x1688823660220288/t0(0) o400->f2c66e98-0dd0-f82b-30a8-a17223517ee8@10.31.138.107@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.790146:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.790150:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.790155:0:50075:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.790157:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513085.790159:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688823660220288, offset 192 00000400:00000200:22.0:1616513085.790166:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.107@tcp1 00000400:00000200:22.0:1616513085.790173:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.107@tcp1 routed destination 00000400:00000200:22.0:1616513085.790188:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.107@tcp1(10.31.138.107@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.790195:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.790200:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.790203:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.790205:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.790210:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513085.790213:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513085.790216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513085.790220:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513085.790222:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513085.790224:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513085.790229:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513085.790235:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789c28 00000800:00000200:14.0:1616513085.790238:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513085.790268:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.790275:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.790277:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.790283:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.790287:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b870 00000800:00000200:20.0:1616513085.790291:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.790293:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.790299:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513085.790301:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.790303:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.790303:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.790307:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.791650:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.791704:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.791713:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.791722:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.154@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.791731:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.154@tcp1 of length 224 into portal 12 MB=0x604a2a8bbb8c0 00000400:00000200:26.0:1616513085.791738:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.154@tcp1 of length 224/224 into md 0xf964d519 [1] + 47360 00000400:00000200:26.0:1616513085.791743:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.791745:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.791751:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.791754:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.791755:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.791759:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.791778:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513085.791793:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.791802:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513085.791810:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.154@tcp1 : PUT - for me 00000100:00000200:22.0:1616513085.791813:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946522351808 00000400:00000200:28.0:1616513085.791821:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.154@tcp1 of length 224 into portal 26 MB=0x604a2a8bbb780 00010000:00000200:22.0:1616513085.791825:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647da880 x1693946522351808/t0(0) o400->16c32c23-fb68-58da-1a2f-13d58b28a860@10.31.140.154@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513085.791827:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.154@tcp1 of length 224/224 into md 0xf97cb509 [1] + 0 00000400:00000200:28.0:1616513085.791832:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5870 00000400:00000200:28.0:1616513085.791835:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513085.791838:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513085.791842:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5870 00000800:00000200:28.0:1616513085.791845:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:22.0:1616513085.791845:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946522351808, offset 192 00000800:00000200:12.0:1616513085.791846:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.791848:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513085.791850:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513085.791851:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.154@tcp1 00000400:00000200:22.0:1616513085.791858:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.154@tcp1 routed destination 00000400:00000200:22.0:1616513085.791873:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.154@tcp1(10.31.140.154@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.791880:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.791886:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.791888:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.791890:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513085.791895:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.791900:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.791906:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946522351488 00010000:00000200:16.0:1616513085.791917:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1693946522351488/t0(0) o400->496fb759-2b95-3b40-eda0-f6543b5f09b3@10.31.140.154@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.791931:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946522351488, offset 192 00000400:00000200:16.0:1616513085.791938:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.154@tcp1 00000400:00000200:16.0:1616513085.791945:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.154@tcp1 routed destination 00000800:00000200:18.0:1616513085.791953:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.791959:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:16.0:1616513085.791960:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.154@tcp1(10.31.140.154@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513085.791962:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513085.791967:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513085.791968:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513085.791973:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.791975:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.791977:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513085.791979:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b980 00000800:00000200:16.0:1616513085.791982:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513085.791983:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.791985:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.791987:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:16.0:1616513085.791995:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513085.792040:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.792046:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.792049:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.792056:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.792061:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789100 00000800:00000200:20.0:1616513085.792065:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.792068:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.792177:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513085.792178:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513085.792187:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513085.792189:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513085.792195:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513085.792196:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:28.0:1616513085.792197:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513085.792198:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513085.792204:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.53@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.792205:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.53@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.792213:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.53@tcp1 of length 224 into portal 26 MB=0x603e263c03a80 00000400:00000200:2.0:1616513085.792213:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.53@tcp1 of length 224 into portal 12 MB=0x603e263c03bc0 00000400:00000200:2.0:1616513085.792219:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.53@tcp1 of length 224/224 into md 0xf964d519 [1] + 47584 00000400:00000200:2.0:1616513085.792224:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:12.0:1616513085.792226:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.53@tcp1 of length 224/224 into md 0xf97cb519 [1] + 0 00000400:00000200:12.0:1616513085.792231:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5320 00000400:00000200:12.0:1616513085.792234:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.792234:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:12.0:1616513085.792240:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513085.792240:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.792243:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513085.792245:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5320 00000800:00000200:2.0:1616513085.792245:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513085.792248:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513085.792250:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513085.792251:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693120731298752 00000800:00000200:12.0:1616513085.792252:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:22.0:1616513085.792258:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebcc80 x1693120731298752/t0(0) o400->168579a3-9174-2fca-ee5a-8667a61c7758@10.31.136.53@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513085.792258:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693120731298432 00010000:00000200:16.0:1616513085.792264:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1693120731298432/t0(0) o400->e1569581-da92-3b83-f504-87c6d0c8f541@10.31.136.53@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513085.792270:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693120731298752, offset 192 00000400:00000200:22.0:1616513085.792275:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.53@tcp1 00000100:00000200:16.0:1616513085.792276:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693120731298432, offset 192 00000400:00000200:22.0:1616513085.792281:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.53@tcp1 routed destination 00000400:00000200:16.0:1616513085.792281:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.53@tcp1 00000400:00000200:16.0:1616513085.792287:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.53@tcp1 routed destination 00000400:00000200:22.0:1616513085.792294:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.53@tcp1(10.31.136.53@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.792300:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513085.792300:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.53@tcp1(10.31.136.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.792304:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.792306:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.792307:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.792308:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.792311:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.792312:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513085.792313:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.792315:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513085.792317:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.792318:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513085.792323:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513085.792323:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513085.792326:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.792329:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.792331:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513085.792333:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513085.792335:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.792336:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513085.792337:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513085.792338:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a880 00000400:00000200:20.0:1616513085.792339:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513085.792340:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513085.792343:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513085.792344:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.792347:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7887f8 00000800:00000200:20.0:1616513085.792350:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.806075:0:50077:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.806130:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.806140:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.806149:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.184@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.806160:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.184@tcp1 of length 224 into portal 26 MB=0x5ffd45e0f3c40 00000400:00000200:26.0:1616513085.806166:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.184@tcp1 of length 224/224 into md 0xf97cb549 [1] + 0 00000400:00000200:26.0:1616513085.806171:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4990 00000400:00000200:26.0:1616513085.806174:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.806176:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513085.806181:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4990 00000800:00000200:26.0:1616513085.806184:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.806185:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.806187:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.806192:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.806194:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.806195:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.806204:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.184@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.806213:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.184@tcp1 of length 224 into portal 12 MB=0x5ffd45e0f3cc0 00000400:00000200:2.0:1616513085.806220:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.184@tcp1 of length 224/224 into md 0xf964d519 [1] + 47808 00000400:00000200:2.0:1616513085.806225:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.806228:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513085.806235:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.806238:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.806240:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.806245:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662459759680 00010000:00000200:16.0:1616513085.806256:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688662459759680/t0(0) o400->cce72eda-d9bd-52c5-d87c-3b83109f0db5@10.31.136.184@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.806270:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662459759680, offset 192 00000400:00000200:16.0:1616513085.806277:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.184@tcp1 00000400:00000200:16.0:1616513085.806285:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.184@tcp1 routed destination 00000400:00000200:16.0:1616513085.806300:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.184@tcp1(10.31.136.184@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513085.806305:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662459759808 00000800:00000200:16.0:1616513085.806307:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.806313:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.806316:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:22.0:1616513085.806317:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebf500 x1688662459759808/t0(0) o400->83ae2705-4618-686a-f3bd-1be156d8f5d4@10.31.136.184@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.806318:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.806323:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.806327:0:50077:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.806330:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513085.806331:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662459759808, offset 192 00000400:00000200:22.0:1616513085.806338:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.184@tcp1 00000400:00000200:22.0:1616513085.806345:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.184@tcp1 routed destination 00000400:00000200:22.0:1616513085.806362:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.184@tcp1(10.31.136.184@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.806369:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.806374:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.806377:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.806379:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513085.806381:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513085.806384:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513085.806388:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.806391:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513085.806391:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513085.806392:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.806398:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.806403:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788ff0 00000800:00000200:18.0:1616513085.806407:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513085.806448:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.806455:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.806458:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513085.806465:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.806469:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4abb0 00000800:00000200:28.0:1616513085.806473:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.806476:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.817786:0:50078:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.817842:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.817863:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.817874:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.29@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.817882:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.29@tcp1 of length 224 into portal 26 MB=0x5ffd463899b00 00000400:00000200:2.0:1616513085.817888:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.29@tcp1 of length 224/224 into md 0xf97cb559 [1] + 0 00000400:00000200:2.0:1616513085.817892:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2880 00000400:00000200:2.0:1616513085.817895:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.817897:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.817902:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2880 00000800:00000200:2.0:1616513085.817905:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.817908:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513085.817909:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513085.817915:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513085.817918:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.817919:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.817927:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.29@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.817935:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.29@tcp1 of length 224 into portal 12 MB=0x5ffd463899b80 00000400:00000200:14.0:1616513085.817941:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.29@tcp1 of length 224/224 into md 0xf964d519 [1] + 48032 00000400:00000200:14.0:1616513085.817946:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513085.817948:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513085.817955:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.817958:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.817960:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.817965:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662551665408 00010000:00000200:16.0:1616513085.817975:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688662551665408/t0(0) o400->7c246a14-51f3-3643-5863-e7dc9a70fc33@10.31.133.29@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.817988:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662551665408, offset 192 00000400:00000200:16.0:1616513085.817996:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.29@tcp1 00000400:00000200:16.0:1616513085.818002:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.29@tcp1 routed destination 00000400:00000200:16.0:1616513085.818017:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.29@tcp1(10.31.133.29@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513085.818019:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662551665536 00000800:00000200:16.0:1616513085.818023:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.818028:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513085.818031:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464627980 x1688662551665536/t0(0) o400->08df0150-783c-83e4-a295-008f771c5add@10.31.133.29@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.818031:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.818033:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.818037:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.818043:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513085.818044:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662551665536, offset 192 00000100:00000200:16.0:1616513085.818044:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513085.818051:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.29@tcp1 00000400:00000200:22.0:1616513085.818057:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.29@tcp1 routed destination 00000800:00000200:18.0:1616513085.818059:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.818066:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.818068:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513085.818069:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513085.818071:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.29@tcp1(10.31.133.29@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513085.818074:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513085.818077:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513085.818079:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7885d8 00000800:00000200:22.0:1616513085.818082:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513085.818082:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:22.0:1616513085.818085:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.818087:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.818091:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.818098:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.818154:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.818161:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513085.818163:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513085.818166:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513085.818170:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513085.818174:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a2a8 00000800:00000200:24.0:1616513085.818178:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.819672:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513085.819728:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.819739:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.819748:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.82@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.819757:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.82@tcp1 of length 224 into portal 26 MB=0x60422029fcf40 00000400:00000200:14.0:1616513085.819763:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.82@tcp1 of length 224/224 into md 0xf97cb569 [1] + 0 00000400:00000200:14.0:1616513085.819768:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3ba0 00000400:00000200:14.0:1616513085.819770:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513085.819773:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513085.819777:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3ba0 00000800:00000200:14.0:1616513085.819780:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513085.819783:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.819784:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513085.819792:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513085.819795:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.819796:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.819805:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.82@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.819813:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.82@tcp1 of length 224 into portal 12 MB=0x60422029fd280 00000400:00000200:26.0:1616513085.819819:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.82@tcp1 of length 224/224 into md 0xf964d519 [1] + 48256 00000400:00000200:26.0:1616513085.819825:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.819827:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.819834:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.819837:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.819839:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.819868:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693393979690816 00010000:00000200:16.0:1616513085.819879:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1693393979690816/t0(0) o400->e8ecf353-1b46-8548-3305-9945fdf7cdb2@10.31.143.82@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.819893:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693393979690816, offset 192 00000400:00000200:16.0:1616513085.819916:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.82@tcp1 00000100:00000200:22.0:1616513085.819917:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693393979691648 00000400:00000200:16.0:1616513085.819922:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.82@tcp1 routed destination 00010000:00000200:22.0:1616513085.819928:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dda00 x1693393979691648/t0(0) o400->a92ba501-159d-28de-ff39-1ffcc83219db@10.31.143.82@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513085.819936:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.82@tcp1(10.31.143.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513085.819942:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693393979691648, offset 192 00000800:00000200:16.0:1616513085.819942:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:22.0:1616513085.819948:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.82@tcp1 00000800:00000200:16.0:1616513085.819948:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.819950:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.819952:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:22.0:1616513085.819954:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.82@tcp1 routed destination 00000800:00000200:16.0:1616513085.819957:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.819961:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.819963:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513085.819968:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.82@tcp1(10.31.143.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.819974:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513085.819980:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.819982:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513085.819986:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.819990:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.819995:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513085.820014:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.820022:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.820024:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513085.820026:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513085.820031:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.820036:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789f58 00000800:00000200:18.0:1616513085.820040:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513085.820047:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.820054:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.820056:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513085.820062:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.820066:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a5d8 00000800:00000200:28.0:1616513085.820070:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.820072:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.820078:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513085.820080:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.820082:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513085.820082:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513085.820086:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513085.855293:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513085.855357:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.855368:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.855377:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.61@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.855387:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.61@tcp1 of length 224 into portal 26 MB=0x5ffdc1467d2c0 00000400:00000200:24.0:1616513085.855394:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.61@tcp1 of length 224/224 into md 0xf97cb589 [1] + 0 00000400:00000200:24.0:1616513085.855399:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3e48 00000400:00000200:24.0:1616513085.855403:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.855406:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513085.855411:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3e48 00000800:00000200:24.0:1616513085.855414:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.855418:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.855420:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.855422:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.855427:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.855446:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.61@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.855452:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.61@tcp1 of length 224 into portal 12 MB=0x5ffdc1467d580 00000400:00000200:24.0:1616513085.855457:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.61@tcp1 of length 224/224 into md 0x865bbec1 [1] + 80864 00000400:00000200:24.0:1616513085.855461:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.855463:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513085.855469:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.855471:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.855475:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513085.855477:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695583789760 00000800:00000200:28.0:1616513085.855478:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:16.0:1616513085.855489:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688695583789760/t0(0) o400->5068c2b4-6ced-6e09-ecdb-d3682041d82e@10.31.142.61@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.855504:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695583789760, offset 192 00000400:00000200:16.0:1616513085.855510:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.61@tcp1 00000400:00000200:16.0:1616513085.855518:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.61@tcp1 routed destination 00000400:00000200:16.0:1616513085.855533:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.61@tcp1(10.31.142.61@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513085.855538:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695583790464 00000800:00000200:16.0:1616513085.855539:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.855547:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.855550:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:0.0:1616513085.855551:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e1f80 x1688695583790464/t0(0) o400->51c42a63-4857-9f26-c8e8-4b2b00418b62@10.31.142.61@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513085.855552:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.855558:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.855564:0:50082:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.855565:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513085.855565:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695583790464, offset 192 00000400:00000200:0.0:1616513085.855571:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.61@tcp1 00000400:00000200:0.0:1616513085.855578:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.61@tcp1 routed destination 00000400:00000200:0.0:1616513085.855592:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.61@tcp1(10.31.142.61@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.855599:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.855604:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.855607:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.855609:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.855614:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.855619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513085.855620:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.855627:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.855629:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513085.855631:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513085.855636:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513085.855638:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513085.855641:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789188 00000400:00000200:12.0:1616513085.855641:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513085.855643:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513085.855645:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.855648:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513085.855648:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513085.855654:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04330 00000800:00000200:12.0:1616513085.855658:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.862244:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513085.862312:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513085.862323:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513085.862332:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.35@tcp1 : PUT - for me 00000400:00000200:14.0:1616513085.862342:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.35@tcp1 of length 224 into portal 26 MB=0x5f910baa49000 00000400:00000200:14.0:1616513085.862348:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.35@tcp1 of length 224/224 into md 0xf97cb599 [1] + 0 00000400:00000200:14.0:1616513085.862353:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3870 00000400:00000200:14.0:1616513085.862355:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513085.862358:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513085.862363:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3870 00000800:00000200:14.0:1616513085.862366:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.862367:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513085.862369:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513085.862371:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.862426:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681225129693184 00010000:00000200:16.0:1616513085.862451:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1681225129693184/t0(0) o400->6ea87112-e6d8-4309-e8e7-21fcb26b3f3d@10.31.132.35@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.862466:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681225129693184, offset 192 00000400:00000200:16.0:1616513085.862472:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.35@tcp1 00000400:00000200:16.0:1616513085.862480:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.35@tcp1 routed destination 00000400:00000200:16.0:1616513085.862496:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.35@tcp1(10.31.132.35@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.862502:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513085.862508:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.862511:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.862513:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.862518:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.862523:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.862525:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513085.862578:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.862584:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513085.862587:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513085.862594:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513085.862599:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789dc0 00000800:00000200:26.0:1616513085.862603:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.862606:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513085.862625:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513085.862680:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.862689:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513085.862698:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.35@tcp1 : PUT - for me 00000400:00000200:24.0:1616513085.862708:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.35@tcp1 of length 224 into portal 12 MB=0x5f910baa49080 00000400:00000200:24.0:1616513085.862714:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.35@tcp1 of length 224/224 into md 0x865bbec1 [1] + 81088 00000400:00000200:24.0:1616513085.862719:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.862722:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513085.862729:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.862732:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513085.862734:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.862738:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.862792:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681225129693312 00010000:00000200:0.0:1616513085.862805:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e2880 x1681225129693312/t0(0) o400->9594ff82-9722-6869-5252-c815342d9466@10.31.132.35@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.862820:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681225129693312, offset 192 00000400:00000200:0.0:1616513085.862826:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.35@tcp1 00000400:00000200:0.0:1616513085.862834:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.35@tcp1 routed destination 00000400:00000200:0.0:1616513085.862849:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.35@tcp1(10.31.132.35@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.862855:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.862861:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.862865:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.862868:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.862873:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.862878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.862889:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.862896:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.862899:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513085.862902:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513085.862906:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.862910:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04ee0 00000800:00000200:20.0:1616513085.862914:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:31.0:1616513085.872001:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513085.872009:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000100:00000200:26.0:1616513085.898486:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927072, portal 4 00000100:00000200:26.0:1616513085.898491:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 24, xid 1693067725927072, offset 0 00000400:00000200:26.0:1616513085.898498:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:26.0:1616513085.898504:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:26.0:1616513085.898518:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513085.898524:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:26.0:1616513085.898531:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513085.898534:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:26.0:1616513085.898536:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:26.0:1616513085.898543:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513085.898592:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:18.0:1616513085.898651:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:18.0:1616513085.898658:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:18.0:1616513085.898661:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513085.898668:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000100:00000200:18.0:1616513085.898675:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9641680 x1693067725927072/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513085.898688:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af870 00000800:00000200:18.0:1616513085.898694:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:18.0:1616513085.898697:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:31.2:1616513085.898762:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:26.0:1616513085.898775:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:26.0:1616513085.898781:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.234@o2ib1 00000800:00000200:2.0:1616513085.898785:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:26.0:1616513085.898788:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513085.898794:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 336 into portal 4 MB=0x603d60c6272a0 00000400:00000200:26.0:1616513085.898799:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.234@o2ib1 of length 336/336 into md 0xf97cbcc9 [1] + 192 00000400:00000200:26.0:1616513085.898804:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.898809:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9641680 x1693067725927072/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513085.898823:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:26.0:1616513085.898825:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:26.0:1616513085.898827:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:30.0:1616513085.898832:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9641680 x1693067725927072/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513085.898844:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9641680 x1693067725927072/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513085.898853:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa08 00000800:00000200:11.2:1616513085.913345:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513085.913401:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.913410:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513085.913419:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.127@tcp1 : PUT - for me 00000400:00000200:28.0:1616513085.913429:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.127@tcp1 of length 224 into portal 26 MB=0x5ffe4eef6a3c0 00000800:00000200:24.0:1616513085.913455:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513085.913456:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.127@tcp1 of length 224/224 into md 0xf97cb5b9 [1] + 0 00000400:00000200:28.0:1616513085.913461:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2b28 00000400:00000200:28.0:1616513085.913464:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513085.913466:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:28.0:1616513085.913467:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513085.913473:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2b28 00000400:00000200:24.0:1616513085.913475:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.127@tcp1 : PUT - for me 00000800:00000200:28.0:1616513085.913476:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513085.913479:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513085.913481:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513085.913484:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513085.913484:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.127@tcp1 of length 224 into portal 12 MB=0x5ffe4eef6a480 00000400:00000200:24.0:1616513085.913491:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.127@tcp1 of length 224/224 into md 0x865bbec1 [1] + 81312 00000400:00000200:24.0:1616513085.913496:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.913499:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513085.913507:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513085.913510:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.913512:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513085.913541:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688733610321024 00000100:00000200:16.0:1616513085.913553:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688733610320832 00010000:00000200:0.0:1616513085.913554:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e0480 x1688733610321024/t0(0) o400->2ceaf474-da19-8825-47e5-e88f638d63b2@10.31.141.127@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:16.0:1616513085.913565:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688733610320832/t0(0) o400->4062d17f-30b1-8e65-71ab-bae3d7724e51@10.31.141.127@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513085.913569:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688733610321024, offset 192 00000400:00000200:0.0:1616513085.913575:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.127@tcp1 00000100:00000200:16.0:1616513085.913580:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688733610320832, offset 192 00000400:00000200:0.0:1616513085.913583:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.127@tcp1 routed destination 00000400:00000200:16.0:1616513085.913586:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.127@tcp1 00000400:00000200:16.0:1616513085.913594:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.127@tcp1 routed destination 00000400:00000200:0.0:1616513085.913599:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.127@tcp1(10.31.141.127@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.913605:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513085.913610:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.127@tcp1(10.31.141.127@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.913617:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.913617:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.913620:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513085.913622:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.913623:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.913625:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.913627:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513085.913628:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.913632:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513085.913634:0:50089:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513085.913638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.913642:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513085.913650:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.913657:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.913659:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513085.913661:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513085.913666:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.913672:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789298 00000800:00000200:20.0:1616513085.913675:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513085.913689:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.913696:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.913698:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513085.913705:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.913709:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05dc0 00000800:00000200:18.0:1616513085.913713:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.913716:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513085.957293:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513085.957350:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513085.957360:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513085.957369:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.177@tcp1 : PUT - for me 00000400:00000200:2.0:1616513085.957380:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.177@tcp1 of length 224 into portal 26 MB=0x5ffd4ae30f380 00000400:00000200:2.0:1616513085.957387:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.177@tcp1 of length 224/224 into md 0xf97cb5c9 [1] + 0 00000400:00000200:2.0:1616513085.957392:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3650 00000400:00000200:2.0:1616513085.957394:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513085.957397:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513085.957402:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3650 00000800:00000200:26.0:1616513085.957404:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.957405:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513085.957408:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.957410:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.957475:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663804146560 00010000:00000200:16.0:1616513085.957487:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688663804146560/t0(0) o400->6b39dacc-8702-8459-5ded-6eeec7eebc13@10.31.136.177@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.957501:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663804146560, offset 192 00000400:00000200:16.0:1616513085.957508:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.177@tcp1 00000800:00000200:11.2:1616513085.957511:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:16.0:1616513085.957516:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.177@tcp1 routed destination 00000800:00000200:24.0:1616513085.957530:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513085.957531:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.177@tcp1(10.31.136.177@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.957538:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513085.957540:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.957544:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.957547:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513085.957549:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.177@tcp1 : PUT - for me 00000800:00000200:16.0:1616513085.957549:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513085.957554:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513085.957558:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.177@tcp1 of length 224 into portal 12 MB=0x5ffd4ae30f400 00000800:00000200:7.2:1616513085.957560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.957561:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513085.957564:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.177@tcp1 of length 224/224 into md 0x865bbec1 [1] + 81536 00000400:00000200:24.0:1616513085.957570:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.957573:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513085.957580:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.957583:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.957584:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513085.957588:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513085.957616:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.957622:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.957625:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513085.957632:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.957638:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788aa0 00000800:00000200:18.0:1616513085.957641:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.957644:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513085.957648:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663804146688 00010000:00000200:0.0:1616513085.957661:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e4800 x1688663804146688/t0(0) o400->39eae165-1b38-ee87-7837-656a409634be@10.31.136.177@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.957675:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663804146688, offset 192 00000400:00000200:0.0:1616513085.957681:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.177@tcp1 00000400:00000200:0.0:1616513085.957689:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.177@tcp1 routed destination 00000400:00000200:0.0:1616513085.957704:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.177@tcp1(10.31.136.177@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.957711:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.957716:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.957719:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.957721:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.957726:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.957732:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513085.957788:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513085.957795:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513085.957798:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513085.957800:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513085.957804:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513085.957808:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05320 00000800:00000200:28.0:1616513085.957812:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.979644:0:50095:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.979707:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.979716:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.979725:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.7@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.979735:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.7@tcp1 of length 224 into portal 12 MB=0x60393a84ab500 00000400:00000200:26.0:1616513085.979741:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.7@tcp1 of length 224/224 into md 0xf964d519 [1] + 48480 00000400:00000200:26.0:1616513085.979747:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.979750:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513085.979757:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.979761:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513085.979762:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513085.979765:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513085.979824:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692782578808064 00010000:00000200:22.0:1616513085.979837:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dc380 x1692782578808064/t0(0) o400->288905ee-29ae-82d1-3af1-1514309cc6cc@10.31.140.7@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513085.979847:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513085.979852:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692782578808064, offset 192 00000400:00000200:22.0:1616513085.979860:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.7@tcp1 00000400:00000200:22.0:1616513085.979867:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.7@tcp1 routed destination 00000400:00000200:22.0:1616513085.979882:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.7@tcp1(10.31.140.7@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513085.979888:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513085.979894:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513085.979897:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513085.979899:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513085.979903:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513085.979904:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.979909:0:50095:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513085.979913:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513085.979922:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.7@tcp1 : PUT - for me 00000400:00000200:12.0:1616513085.979931:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.7@tcp1 of length 224 into portal 26 MB=0x60393a84ab380 00000400:00000200:12.0:1616513085.979938:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.7@tcp1 of length 224/224 into md 0xf97cb5e9 [1] + 0 00000400:00000200:12.0:1616513085.979942:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3980 00000400:00000200:12.0:1616513085.979945:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513085.979948:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513085.979953:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3980 00000800:00000200:24.0:1616513085.979956:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513085.979956:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513085.979959:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.979962:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513085.979980:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.979987:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.979990:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513085.979996:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.980000:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b078 00000800:00000200:18.0:1616513085.980004:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.980007:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.980031:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692782578807680 00010000:00000200:16.0:1616513085.980043:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1692782578807680/t0(0) o400->f15e7ac1-b4c5-334c-80e7-db958140ff08@10.31.140.7@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.980057:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692782578807680, offset 192 00000400:00000200:16.0:1616513085.980063:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.7@tcp1 00000400:00000200:16.0:1616513085.980071:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.7@tcp1 routed destination 00000400:00000200:16.0:1616513085.980085:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.7@tcp1(10.31.140.7@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513085.980092:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.980097:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.980100:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513085.980102:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.980107:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.980113:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513085.980114:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513085.980168:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.980175:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.980177:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.980184:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.980190:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7882a8 00000800:00000200:20.0:1616513085.980193:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.980196:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513085.990574:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513085.990630:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513085.990640:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513085.990649:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.70@tcp1 : PUT - for me 00000400:00000200:26.0:1616513085.990658:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.70@tcp1 of length 224 into portal 26 MB=0x603ff72f81c40 00000400:00000200:26.0:1616513085.990665:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.70@tcp1 of length 224/224 into md 0xf97cb609 [1] + 0 00000400:00000200:26.0:1616513085.990669:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b25d8 00000400:00000200:26.0:1616513085.990672:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513085.990675:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513085.990682:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b25d8 00000800:00000200:2.0:1616513085.990684:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.990685:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513085.990688:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513085.990690:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513085.990745:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245540670528 00010000:00000200:16.0:1616513085.990756:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1693245540670528/t0(0) o400->7af90df8-ad87-2408-a628-47bcf08c33fb@10.31.132.70@tcp1:42/0 lens 224/224 e 0 to 0 dl 1616513177 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513085.990770:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245540670528, offset 192 00000400:00000200:16.0:1616513085.990779:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.70@tcp1 00000400:00000200:16.0:1616513085.990786:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.70@tcp1 routed destination 00000800:00000200:11.2:1616513085.990790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:16.0:1616513085.990801:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.70@tcp1(10.31.132.70@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513085.990804:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513085.990808:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513085.990814:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513085.990814:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513085.990817:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513085.990819:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513085.990823:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.70@tcp1 : PUT - for me 00000800:00000200:16.0:1616513085.990824:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513085.990828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513085.990830:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513085.990832:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.70@tcp1 of length 224 into portal 12 MB=0x603ff72f81cc0 00000400:00000200:24.0:1616513085.990838:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.70@tcp1 of length 224/224 into md 0x865bbec1 [1] + 81760 00000400:00000200:24.0:1616513085.990843:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513085.990846:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513085.990852:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513085.990855:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513085.990858:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513085.990862:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513085.990886:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513085.990893:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513085.990896:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513085.990903:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513085.990908:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7897e8 00000800:00000200:18.0:1616513085.990912:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513085.990915:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513085.990920:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245540670656 00010000:00000200:0.0:1616513085.990933:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e5e80 x1693245540670656/t0(0) o400->6a5ac55b-7d6f-aab3-9221-9c69a9ac9422@10.31.132.70@tcp1:711/0 lens 224/224 e 0 to 0 dl 1616513091 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513085.990958:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245540670656, offset 192 00000400:00000200:0.0:1616513085.990964:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.70@tcp1 00000400:00000200:0.0:1616513085.990971:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.70@tcp1 routed destination 00000400:00000200:0.0:1616513085.990986:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.70@tcp1(10.31.132.70@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513085.990992:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513085.990998:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513085.991001:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513085.991003:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513085.991008:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513085.991013:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513085.991024:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513085.991034:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513085.991037:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513085.991044:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513085.991048:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ba0 00000800:00000200:20.0:1616513085.991052:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513085.991055:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:13.0:1616513085.994959:0:6491:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927088, portal 4 00000100:00000200:13.0:1616513085.994965:0:6491:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927088, offset 0 00000400:00000200:13.0:1616513085.994971:0:6491:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:13.0:1616513085.994979:0:6491:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:13.0:1616513085.994993:0:6491:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:13.0:1616513085.995000:0:6491:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:13.0:1616513085.995008:0:6491:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513085.995011:0:6491:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:13.0:1616513085.995013:0:6491:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:13.0:1616513085.995019:0:6491:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513085.995028:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513085.995084:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513085.995091:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513085.995095:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513085.995101:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513085.995108:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843a2ea880 x1693067725927088/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513085.995131:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345980 00000800:00000200:26.0:1616513085.995136:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513085.995140:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513085.995154:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513085.995208:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513085.995217:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513085.995225:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513085.995233:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6272b0 00000400:00000200:18.0:1616513085.995241:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cbd39 [1] + 192 00000400:00000200:18.0:1616513085.995248:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513085.995254:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843a2ea880 x1693067725927088/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513085.995261:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513085.995269:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513085.995272:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513085.995274:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:13.0:1616513085.995279:0:6491:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843a2ea880 x1693067725927088/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:13.0:1616513085.995293:0:6491:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843a2ea880 x1693067725927088/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513092 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:13.0:1616513085.995303:0:6491:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345078 00000800:00000200:11.2:1616513086.035805:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.035860:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.035870:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.035879:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.50@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.035888:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.50@tcp1 of length 224 into portal 26 MB=0x5f9087b718d80 00000400:00000200:12.0:1616513086.035895:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.50@tcp1 of length 224/224 into md 0xf97cb629 [1] + 0 00000400:00000200:12.0:1616513086.035900:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3dc0 00000400:00000200:12.0:1616513086.035902:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.035905:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.035910:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3dc0 00000800:00000200:12.0:1616513086.035913:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.035914:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.035916:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.035921:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.035924:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.035925:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.035934:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.50@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.035943:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.50@tcp1 of length 224 into portal 12 MB=0x5f9087b718e00 00000400:00000200:24.0:1616513086.035949:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.50@tcp1 of length 224/224 into md 0x865bbec1 [1] + 81984 00000400:00000200:24.0:1616513086.035954:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.035957:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513086.035964:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.035967:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.035970:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513086.035986:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681189709647232 00010000:00000200:16.0:1616513086.035997:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1681189709647232/t0(0) o400->a5a421b1-dccd-697c-e870-55ee2271227c@10.31.131.50@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513086.036011:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681189709647232, offset 192 00000400:00000200:16.0:1616513086.036018:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.50@tcp1 00000400:00000200:16.0:1616513086.036025:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.50@tcp1 routed destination 00000100:00000200:0.0:1616513086.036029:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681189709647360 00000400:00000200:16.0:1616513086.036041:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.50@tcp1(10.31.131.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513086.036042:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e1b00 x1681189709647360/t0(0) o400->93e5595a-a453-4b2a-a54c-73b4a3100b49@10.31.131.50@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513086.036048:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513086.036054:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513086.036056:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681189709647360, offset 192 00000800:00000200:16.0:1616513086.036057:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513086.036059:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513086.036063:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513086.036063:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.50@tcp1 00000100:00000200:16.0:1616513086.036070:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513086.036070:0:50101:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513086.036070:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.50@tcp1 routed destination 00000400:00000200:0.0:1616513086.036085:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.50@tcp1(10.31.131.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.036091:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.036097:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.036100:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.036102:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.036106:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.036111:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513086.036127:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.036134:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.036136:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.036143:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.036149:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788880 00000800:00000200:26.0:1616513086.036153:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.036156:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513086.036165:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.036173:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.036176:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513086.036177:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513086.036182:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.036186:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:28.0:1616513086.036190:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.039206:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513086.039260:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.039270:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513086.039279:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.72@tcp1 : PUT - for me 00000800:00000200:11.2:1616513086.039285:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513086.039288:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.72@tcp1 of length 224 into portal 26 MB=0x6019d68661e00 00000400:00000200:14.0:1616513086.039294:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.72@tcp1 of length 224/224 into md 0xf97cb649 [1] + 0 00000400:00000200:14.0:1616513086.039299:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3d38 00000400:00000200:14.0:1616513086.039304:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.039306:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513086.039311:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3d38 00000800:00000200:18.0:1616513086.039314:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.039314:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.039316:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.039318:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513086.039345:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.039355:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.039364:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.72@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.039373:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.72@tcp1 of length 224 into portal 12 MB=0x6019d686620c0 00000400:00000200:24.0:1616513086.039379:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.72@tcp1 of length 224/224 into md 0x865bbec1 [1] + 82208 00000100:00000200:16.0:1616513086.039379:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690625433280000 00000400:00000200:24.0:1616513086.039384:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.039387:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:16.0:1616513086.039390:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1690625433280000/t0(0) o400->1019f00a-ba49-fa69-2122-09d5d73c4922@10.31.141.72@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513086.039393:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.039396:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513086.039398:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.039401:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513086.039405:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690625433280000, offset 192 00000400:00000200:16.0:1616513086.039411:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.72@tcp1 00000400:00000200:16.0:1616513086.039418:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.72@tcp1 routed destination 00000400:00000200:16.0:1616513086.039446:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.72@tcp1(10.31.141.72@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513086.039452:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513086.039460:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513086.039463:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513086.039463:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690625433280704 00000800:00000200:16.0:1616513086.039465:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513086.039470:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.039475:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513086.039475:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e1680 x1690625433280704/t0(0) o400->a1798616-d51e-f855-ccaa-8b108114751e@10.31.141.72@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513086.039477:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513086.039490:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690625433280704, offset 192 00000400:00000200:0.0:1616513086.039496:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.72@tcp1 00000400:00000200:0.0:1616513086.039503:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.72@tcp1 routed destination 00000400:00000200:0.0:1616513086.039518:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.72@tcp1(10.31.141.72@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.039527:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513086.039528:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513086.039532:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513086.039535:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513086.039535:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513086.039537:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513086.039537:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.039542:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513086.039544:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513086.039547:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513086.039550:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789cb0 00000800:00000200:26.0:1616513086.039553:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.039556:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513086.039602:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.039609:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.039613:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513086.039614:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513086.039620:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.039624:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04110 00000800:00000200:12.0:1616513086.039627:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.041560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.041614:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.041623:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.041632:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.68@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.041642:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.68@tcp1 of length 224 into portal 12 MB=0x603ff827c79c0 00000400:00000200:18.0:1616513086.041649:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.68@tcp1 of length 224/224 into md 0xf964d519 [1] + 48704 00000400:00000200:18.0:1616513086.041654:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.041657:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.041664:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.041667:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.041667:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.041671:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.041721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513086.041728:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245801003456 00010000:00000200:22.0:1616513086.041740:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bde80 x1693245801003456/t0(0) o400->3d66ba14-5239-00cb-55df-9523e0659d70@10.31.132.68@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.041755:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245801003456, offset 192 00000400:00000200:22.0:1616513086.041761:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.68@tcp1 00000400:00000200:22.0:1616513086.041769:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.68@tcp1 routed destination 00000400:00000200:22.0:1616513086.041784:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.68@tcp1(10.31.132.68@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513086.041785:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.041790:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513086.041794:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.041809:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.041811:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513086.041812:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.68@tcp1 : PUT - for me 00000800:00000200:22.0:1616513086.041813:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.041817:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513086.041820:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.68@tcp1 of length 224 into portal 26 MB=0x603ff827c7900 00000800:00000200:7.2:1616513086.041822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513086.041827:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.68@tcp1 of length 224/224 into md 0xf97cb659 [1] + 0 00000400:00000200:24.0:1616513086.041832:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3078 00000400:00000200:24.0:1616513086.041835:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.041837:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513086.041841:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3078 00000800:00000200:24.0:1616513086.041844:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.041847:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513086.041847:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.041851:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513086.041889:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.041896:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.041898:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.041905:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:16.0:1616513086.041908:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245801003264 00000400:00000200:26.0:1616513086.041909:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bc28 00000800:00000200:26.0:1616513086.041913:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.041916:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:16.0:1616513086.041919:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1693245801003264/t0(0) o400->17c25de8-62bd-93e8-9e0b-fc8d7d32606b@10.31.132.68@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513086.041933:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245801003264, offset 192 00000400:00000200:16.0:1616513086.041940:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.68@tcp1 00000400:00000200:16.0:1616513086.041947:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.68@tcp1 routed destination 00000400:00000200:16.0:1616513086.041962:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.68@tcp1(10.31.132.68@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513086.041969:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513086.041975:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513086.041978:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513086.041980:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513086.041984:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.041989:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513086.041991:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513086.042002:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.042008:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.042011:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513086.042018:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.042024:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788550 00000800:00000200:28.0:1616513086.042027:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.042030:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.053928:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513086.053984:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.053994:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513086.054003:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.105@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.054013:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.105@tcp1 of length 224 into portal 26 MB=0x5fffb96ed0940 00000400:00000200:20.0:1616513086.054020:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.105@tcp1 of length 224/224 into md 0xf97cb669 [1] + 0 00000400:00000200:20.0:1616513086.054029:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3f58 00000400:00000200:20.0:1616513086.054032:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.054035:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513086.054038:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513086.054039:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3f58 00000800:00000200:20.0:1616513086.054043:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.054045:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.054047:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513086.054103:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688830917544256 00000800:00000200:7.2:1616513086.054105:0:50103:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:16.0:1616513086.054114:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688830917544256/t0(0) o400->9ec5ee88-33ee-cba2-dc3e-8de5f051d315@10.31.138.105@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513086.054128:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688830917544256, offset 192 00000400:00000200:16.0:1616513086.054135:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.105@tcp1 00000400:00000200:16.0:1616513086.054143:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.105@tcp1 routed destination 00000400:00000200:16.0:1616513086.054158:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.105@tcp1(10.31.138.105@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513086.054165:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513086.054167:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513086.054171:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513086.054174:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513086.054176:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513086.054177:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:16.0:1616513086.054181:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513086.054188:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.105@tcp1 : PUT - for me 00000100:00000200:16.0:1616513086.054189:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513086.054197:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.105@tcp1 of length 224 into portal 12 MB=0x5fffb96ed0a00 00000400:00000200:14.0:1616513086.054203:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.105@tcp1 of length 224/224 into md 0xf964d519 [1] + 48928 00000400:00000200:14.0:1616513086.054208:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.054211:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513086.054217:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513086.054220:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513086.054221:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:14.0:1616513086.054228:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513086.054229:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513086.054231:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.054232:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.054239:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.054256:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7884c8 00000800:00000200:18.0:1616513086.054260:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.054291:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688830917544448 00010000:00000200:22.0:1616513086.054304:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621680 x1688830917544448/t0(0) o400->0d3c3964-74cb-7bb8-ab03-368e5a695838@10.31.138.105@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.054318:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688830917544448, offset 192 00000400:00000200:22.0:1616513086.054326:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.105@tcp1 00000400:00000200:22.0:1616513086.054334:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.105@tcp1 routed destination 00000400:00000200:22.0:1616513086.054349:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.105@tcp1(10.31.138.105@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.054356:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.054361:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.054364:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.054366:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.054371:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.054376:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.054387:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.054394:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.054397:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513086.054404:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.054408:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b760 00000800:00000200:28.0:1616513086.054414:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.054417:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:29.0:1616513086.058615:0:6492:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927104, portal 4 00000100:00000200:29.0:1616513086.058621:0:6492:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927104, offset 0 00000400:00000200:29.0:1616513086.058629:0:6492:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:29.0:1616513086.058635:0:6492:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:29.0:1616513086.058649:0:6492:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513086.058656:0:6492:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:29.0:1616513086.058664:0:6492:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513086.058667:0:6492:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:29.0:1616513086.058669:0:6492:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:29.0:1616513086.058675:0:6492:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513086.058684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513086.058740:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513086.058748:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513086.058751:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513086.058758:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513086.058765:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843a2eec00 x1693067725927104/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513086.058777:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b26e8 00000800:00000200:2.0:1616513086.058781:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513086.058784:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513086.058894:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513086.058948:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513086.058957:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513086.058966:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513086.058974:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6272c0 00000400:00000200:18.0:1616513086.058981:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cbda9 [1] + 192 00000400:00000200:18.0:1616513086.058986:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.058991:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843a2eec00 x1693067725927104/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513086.059001:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513086.059006:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513086.059011:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513086.059013:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:29.0:1616513086.059016:0:6492:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843a2eec00 x1693067725927104/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:29.0:1616513086.059029:0:6492:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843a2eec00 x1693067725927104/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:29.0:1616513086.059040:0:6492:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b37e8 00000800:00000200:7.2:1616513086.066449:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.066505:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.066515:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.066524:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.133@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.066533:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.133@tcp1 of length 224 into portal 12 MB=0x5ffd5ac6e9480 00000400:00000200:2.0:1616513086.066540:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.133@tcp1 of length 224/224 into md 0xf964d519 [1] + 49152 00000400:00000200:2.0:1616513086.066545:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.066548:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513086.066555:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513086.066556:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513086.066558:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.066558:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.066563:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.066616:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668069598336 00000800:00000200:24.0:1616513086.066622:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513086.066628:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebe300 x1688668069598336/t0(0) o400->9b6ae293-1656-9987-96ea-6ff076d04159@10.31.129.133@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513086.066631:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.066639:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.133@tcp1 : PUT - for me 00000100:00000200:22.0:1616513086.066641:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668069598336, offset 192 00000400:00000200:24.0:1616513086.066648:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.133@tcp1 of length 224 into portal 26 MB=0x5ffd5ac6e9180 00000400:00000200:22.0:1616513086.066648:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.133@tcp1 00000400:00000200:24.0:1616513086.066654:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.133@tcp1 of length 224/224 into md 0xf97cb689 [1] + 0 00000400:00000200:22.0:1616513086.066655:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.133@tcp1 routed destination 00000400:00000200:24.0:1616513086.066659:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b23b8 00000400:00000200:24.0:1616513086.066661:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.066664:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513086.066668:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b23b8 00000400:00000200:22.0:1616513086.066669:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.133@tcp1(10.31.129.133@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513086.066671:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.066673:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513086.066674:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513086.066675:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.066677:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513086.066680:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.066682:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.066684:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.066689:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.066694:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513086.066739:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668069597568 00010000:00000200:16.0:1616513086.066749:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688668069597568/t0(0) o400->5399bffd-8cc7-80c7-9780-384fba140791@10.31.129.133@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513086.066758:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:16.0:1616513086.066762:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668069597568, offset 192 00000800:00000200:26.0:1616513086.066765:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.066767:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513086.066768:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.133@tcp1 00000400:00000200:26.0:1616513086.066774:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:16.0:1616513086.066775:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.133@tcp1 routed destination 00000400:00000200:26.0:1616513086.066778:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b5c8 00000800:00000200:26.0:1616513086.066782:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.066785:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513086.066789:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.133@tcp1(10.31.129.133@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513086.066796:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513086.066801:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513086.066803:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513086.066805:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513086.066810:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.066815:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513086.066816:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513086.066827:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.066835:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.066837:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513086.066844:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.066850:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789320 00000800:00000200:28.0:1616513086.066853:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.066856:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.068631:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513086.068686:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.068695:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513086.068703:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.151@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.068712:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.151@tcp1 of length 224 into portal 26 MB=0x5fff02c1ae180 00000400:00000200:14.0:1616513086.068719:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.151@tcp1 of length 224/224 into md 0xf97cb6b9 [1] + 0 00000400:00000200:14.0:1616513086.068724:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b36d8 00000400:00000200:14.0:1616513086.068727:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.068729:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513086.068733:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b36d8 00000800:00000200:14.0:1616513086.068736:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.068738:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.068739:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.068742:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513086.068799:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688781880746368 00010000:00000200:16.0:1616513086.068810:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688781880746368/t0(0) o400->d5c57260-bfd2-f112-5e18-1b7f7ff60b7b@10.31.131.151@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513086.068824:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688781880746368, offset 192 00000800:00000200:7.2:1616513086.068827:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513086.068830:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.151@tcp1 00000400:00000200:16.0:1616513086.068838:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.151@tcp1 routed destination 00000400:00000200:16.0:1616513086.068853:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.151@tcp1(10.31.131.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513086.068859:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513086.068865:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513086.068884:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513086.068886:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513086.068891:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:16.0:1616513086.068897:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513086.068899:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.068908:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.068918:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.151@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.068927:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.151@tcp1 of length 224 into portal 12 MB=0x5fff02c1ae5c0 00000400:00000200:26.0:1616513086.068933:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.151@tcp1 of length 224/224 into md 0xf964d519 [1] + 49376 00000400:00000200:26.0:1616513086.068938:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.068941:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513086.068947:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513086.068950:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513086.068952:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513086.068959:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513086.068960:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513086.068963:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.068963:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.068970:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.068976:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789e48 00000800:00000200:18.0:1616513086.068979:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.069010:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688781880747456 00010000:00000200:22.0:1616513086.069022:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8000 x1688781880747456/t0(0) o400->1e688eef-e03f-7019-ac13-a2b4aaaba910@10.31.131.151@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.069037:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688781880747456, offset 192 00000400:00000200:22.0:1616513086.069043:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.151@tcp1 00000400:00000200:22.0:1616513086.069050:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.151@tcp1 routed destination 00000400:00000200:22.0:1616513086.069065:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.151@tcp1(10.31.131.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.069072:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.069077:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.069082:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.069084:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.069089:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.069136:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.069191:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.069198:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513086.069200:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513086.069207:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513086.069211:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bba0 00000800:00000200:24.0:1616513086.069215:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.069218:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:1.0:1616513086.106586:0:6493:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927120, portal 4 00000100:00000200:1.0:1616513086.106592:0:6493:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927120, offset 0 00000400:00000200:1.0:1616513086.106599:0:6493:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:1.0:1616513086.106605:0:6493:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000100:00000200:9.0:1616513086.106619:0:6494:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927136, portal 4 00000400:00000200:1.0:1616513086.106620:0:6493:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000100:00000200:9.0:1616513086.106625:0:6494:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927136, offset 0 00000800:00000200:1.0:1616513086.106627:0:6493:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:9.0:1616513086.106631:0:6494:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:1.0:1616513086.106634:0:6493:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513086.106637:0:6493:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000400:00000200:9.0:1616513086.106638:0:6494:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:1.0:1616513086.106642:0:6493:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:1.0:1616513086.106648:0:6493:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:9.0:1616513086.106652:0:6494:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:9.0:1616513086.106659:0:6494:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000100:00000200:13.0:1616513086.106662:0:6495:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927152, portal 4 00000800:00000200:9.0:1616513086.106666:0:6494:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000100:00000200:13.0:1616513086.106668:0:6495:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927152, offset 0 00000800:00000200:9.0:1616513086.106669:0:6494:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:9.0:1616513086.106671:0:6494:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:13.0:1616513086.106674:0:6495:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:9.0:1616513086.106676:0:6494:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:13.0:1616513086.106680:0:6495:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:5.2:1616513086.106685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:13.0:1616513086.106693:0:6495:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:27.2:1616513086.106695:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:13.0:1616513086.106700:0:6495:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:13.0:1616513086.106706:0:6495:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513086.106711:0:6495:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:13.0:1616513086.106713:0:6495:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:27.0:1616513086.106717:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:13.0:1616513086.106718:0:6495:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:27.0:1616513086.106723:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513086.106726:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:25.2:1616513086.106726:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000400:00000200:27.0:1616513086.106732:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513086.106739:0:6476:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdd68f500 x1693067725927120/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513086.106742:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513086.106749:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:27.0:1616513086.106751:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe844c8 00000400:00000200:28.0:1616513086.106752:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:27.0:1616513086.106755:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513086.106758:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:28.0:1616513086.106759:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513086.106766:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843f7bba80 x1693067725927136/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513086.106778:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca34198 00000800:00000200:28.0:1616513086.106782:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:14.0:1616513086.106783:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:28.0:1616513086.106785:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:14.0:1616513086.106790:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513086.106795:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513086.106802:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513086.106808:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843a2eb180 x1693067725927152/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513086.106820:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345d38 00000800:00000200:14.0:1616513086.106825:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513086.106827:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:27.2:1616513086.106886:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513086.106896:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513086.106902:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513086.106906:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513086.106910:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513086.106927:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6272d0 00000400:00000200:27.0:1616513086.106933:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166e6d [1] + 192 00000800:00000200:5.2:1616513086.106937:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000400:00000200:27.0:1616513086.106938:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:25.2:1616513086.106938:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000100:00000200:27.0:1616513086.106943:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdd68f500 x1693067725927120/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513086.106956:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513086.106959:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513086.106961:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:1.0:1616513086.106966:0:6493:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdd68f500 x1693067725927120/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513086.106979:0:6493:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdd68f500 x1693067725927120/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513086.106989:0:6493:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84000 00000800:00000200:24.0:1616513086.106992:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513086.107002:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:24.0:1616513086.107011:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:2.0:1616513086.107011:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:7.2:1616513086.107018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513086.107021:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6272e0 00000800:00000200:2.0:1616513086.107021:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:24.0:1616513086.107028:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d88d1 [1] + 192 00000800:00000200:14.0:1616513086.107030:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513086.107030:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:24.0:1616513086.107033:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513086.107033:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:2.0:1616513086.107038:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6272f0 00000100:00000200:24.0:1616513086.107039:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843f7bba80 x1693067725927136/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513086.107039:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513086.107043:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.107045:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cbdf9 [1] + 192 00000400:00000200:14.0:1616513086.107047:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.43@tcp1 : PUT - for me 00000800:00000200:20.0:1616513086.107050:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:2.0:1616513086.107050:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513086.107054:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:2.0:1616513086.107055:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843a2eb180 x1693067725927152/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513086.107056:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.43@tcp1 of length 224 into portal 12 MB=0x5f91db22f8a00 00000800:00000200:24.0:1616513086.107057:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513086.107059:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:9.0:1616513086.107064:0:6494:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843f7bba80 x1693067725927136/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513086.107071:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000400:00000200:14.0:1616513086.107073:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.43@tcp1 of length 224/224 into md 0xf964d519 [1] + 49600 00000800:00000200:2.0:1616513086.107076:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:9.0:1616513086.107077:0:6494:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843f7bba80 x1693067725927136/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513086.107078:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513086.107078:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:13.0:1616513086.107081:0:6495:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843a2eb180 x1693067725927152/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:9.0:1616513086.107089:0:6494:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35430 00000100:00000200:13.0:1616513086.107094:0:6495:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843a2eb180 x1693067725927152/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:13.0:1616513086.107105:0:6495:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345078 00000100:00000200:14.0:1616513086.107108:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513086.107114:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.107117:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.107119:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.107180:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681280822381056 00010000:00000200:22.0:1616513086.107193:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621f80 x1681280822381056/t0(0) o400->5430b98c-91a0-ff30-ddc8-2efd19df4670@10.31.132.43@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.107208:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681280822381056, offset 192 00000400:00000200:22.0:1616513086.107215:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.43@tcp1 00000400:00000200:22.0:1616513086.107222:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.43@tcp1 routed destination 00000400:00000200:22.0:1616513086.107238:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.43@tcp1(10.31.132.43@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.107244:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.107250:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.107253:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.107255:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.107260:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.107265:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513086.107269:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513086.107276:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.107281:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513086.107283:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513086.107283:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513086.107286:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513086.107289:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513086.107289:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.107293:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bd38 00000800:00000200:12.0:1616513086.107293:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513086.107297:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.43@tcp1 : PUT - for me 00000800:00000200:26.0:1616513086.107297:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513086.107305:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.43@tcp1 of length 224 into portal 26 MB=0x5f91db22f8980 00000400:00000200:28.0:1616513086.107312:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.43@tcp1 of length 224/224 into md 0xf97cb6d9 [1] + 0 00000400:00000200:28.0:1616513086.107316:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b35c8 00000400:00000200:28.0:1616513086.107318:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.107321:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513086.107325:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b35c8 00000800:00000200:28.0:1616513086.107328:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.107330:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.107332:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513086.107388:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681280822380928 00010000:00000200:16.0:1616513086.107398:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1681280822380928/t0(0) o400->cf1932b3-bfe5-f19b-bbbd-0df840e579f0@10.31.132.43@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513086.107412:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681280822380928, offset 192 00000400:00000200:16.0:1616513086.107420:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.43@tcp1 00000400:00000200:16.0:1616513086.107426:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.43@tcp1 routed destination 00000400:00000200:16.0:1616513086.107452:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.43@tcp1(10.31.132.43@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513086.107459:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513086.107464:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513086.107467:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513086.107469:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:9.0:1616513086.107470:0:6488:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927168, portal 4 00000800:00000200:16.0:1616513086.107473:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:9.0:1616513086.107473:0:6488:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927168, offset 0 00000800:00000200:11.2:1616513086.107478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:9.0:1616513086.107479:0:6488:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000100:00000200:16.0:1616513086.107480:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:9.0:1616513086.107483:0:6488:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:20.0:1616513086.107489:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.107495:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:9.0:1616513086.107495:0:6488:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513086.107497:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513086.107500:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:9.0:1616513086.107501:0:6488:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:20.0:1616513086.107504:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:9.0:1616513086.107506:0:6488:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:9.0:1616513086.107509:0:6488:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:20.0:1616513086.107510:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788660 00000800:00000200:9.0:1616513086.107510:0:6488:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:20.0:1616513086.107514:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:9.0:1616513086.107514:0:6488:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513086.107523:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513086.107536:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513086.107541:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513086.107544:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:9.0:1616513086.107546:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513086.107550:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513086.107555:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1393a80 x1693067725927168/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513086.107565:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca34f68 00000800:00000200:1.0:1616513086.107569:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513086.107729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513086.107739:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513086.107746:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513086.107750:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513086.107754:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513086.107762:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627300 00000400:00000200:1.0:1616513086.107768:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffe55 [1] + 192 00000400:00000200:1.0:1616513086.107773:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513086.107777:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1393a80 x1693067725927168/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:11.2:1616513086.107785:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:1.0:1616513086.107789:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513086.107792:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:9.0:1616513086.107794:0:6488:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1393a80 x1693067725927168/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513086.107794:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:12.0:1616513086.107796:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.107804:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513086.107806:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:9.0:1616513086.107806:0:6488:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1393a80 x1693067725927168/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513086.107814:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.150@tcp1 : PUT - for me 00000400:00000200:9.0:1616513086.107814:0:6488:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35430 00000400:00000200:12.0:1616513086.107823:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.150@tcp1 of length 224 into portal 26 MB=0x60595334ecfc0 00000400:00000200:12.0:1616513086.107830:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.150@tcp1 of length 224/224 into md 0xf97cb6e9 [1] + 0 00000400:00000200:12.0:1616513086.107835:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b27f8 00000400:00000200:12.0:1616513086.107838:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.107840:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.107845:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b27f8 00000800:00000200:12.0:1616513086.107848:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.107851:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.107853:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513086.107857:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694988229332928 00000800:00000200:11.2:1616513086.107861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:16.0:1616513086.107863:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1694988229332928/t0(0) o400->2d622619-009a-a600-6a3e-8f52e8c3213d@10.31.143.150@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513086.107870:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513086.107873:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694988229332928, offset 192 00000800:00000200:24.0:1616513086.107876:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513086.107878:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513086.107879:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.150@tcp1 00000400:00000200:24.0:1616513086.107883:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.150@tcp1 : PUT - for me 00000400:00000200:16.0:1616513086.107885:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.150@tcp1 routed destination 00000400:00000200:24.0:1616513086.107892:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.150@tcp1 of length 224 into portal 12 MB=0x60595334ed380 00000400:00000200:24.0:1616513086.107898:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.150@tcp1 of length 224/224 into md 0x865bbec1 [1] + 82432 00000400:00000200:16.0:1616513086.107898:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.150@tcp1(10.31.143.150@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513086.107902:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513086.107903:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:24.0:1616513086.107904:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513086.107908:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513086.107910:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513086.107910:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513086.107912:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513086.107912:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513086.107914:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513086.107926:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513086.107930:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513086.107930:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.107941:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.107945:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.107947:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513086.107949:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.107953:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.107958:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7895c8 00000800:00000200:2.0:1616513086.107961:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.107983:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694988229333888 00010000:00000200:0.0:1616513086.107996:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e7500 x1694988229333888/t0(0) o400->4cc95aae-5632-43b3-1783-b17f7d1b694c@10.31.143.150@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513086.108010:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694988229333888, offset 192 00000400:00000200:0.0:1616513086.108016:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.150@tcp1 00000400:00000200:0.0:1616513086.108024:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.150@tcp1 routed destination 00000400:00000200:0.0:1616513086.108038:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.150@tcp1(10.31.143.150@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.108045:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.108051:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.108053:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.108056:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.108060:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.108065:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.108073:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.108077:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.108079:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513086.108081:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513086.108086:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.108089:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb057e8 00000800:00000200:28.0:1616513086.108092:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:31.2:1616513086.113643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da655e00] (20)++ 00000800:00000200:18.0:1616513086.113700:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da655e00] (21)++ 00000800:00000200:18.0:1616513086.113710:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.226@o2ib1 00000400:00000200:18.0:1616513086.113719:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.226@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513086.113727:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.226@o2ib1 of length 224 into portal 26 MB=0x5ff36411d8b80 00000400:00000200:18.0:1616513086.113733:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.226@o2ib1 of length 224/224 into md 0xf97cb709 [1] + 0 00000400:00000200:18.0:1616513086.113740:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2770 00000400:00000200:18.0:1616513086.113743:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.113746:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513086.113750:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2770 00000800:00000200:18.0:1616513086.113753:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da655e00] (22)++ 00000800:00000200:7.2:1616513086.113753:0:50109:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.113756:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da655e00] (23)-- 00000800:00000200:18.0:1616513086.113759:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655e00] (22)-- 00000800:00000200:18.0:1616513086.113762:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da655e00] (21)++ 00000800:00000200:26.0:1616513086.113766:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.113768:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.226@o2ib1 00000800:00000200:26.0:1616513086.113774:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.113775:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.226@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513086.113780:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.226@o2ib1 of length 224 into portal 12 MB=0x5ff36411d8c40 00000400:00000200:26.0:1616513086.113784:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.9@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.113785:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.226@o2ib1 of length 224/224 into md 0xf964d519 [1] + 49824 00000400:00000200:18.0:1616513086.113789:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.113791:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:26.0:1616513086.113793:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.9@tcp1 of length 224 into portal 26 MB=0x5fffe46daa900 00000800:00000200:18.0:1616513086.113797:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da655e00] (22)++ 00000800:00000200:18.0:1616513086.113799:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da655e00] (23)-- 00000400:00000200:26.0:1616513086.113800:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.9@tcp1 of length 224/224 into md 0xf97cb729 [1] + 0 00000800:00000200:18.0:1616513086.113800:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655e00] (22)-- 00000800:00000200:18.0:1616513086.113803:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655e00] (21)-- 00000400:00000200:26.0:1616513086.113804:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3320 00000800:00000200:18.0:1616513086.113805:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513086.113807:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.113810:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513086.113816:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3320 00000800:00000200:26.0:1616513086.113819:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.113821:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.113823:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513086.113824:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1687983369325440 00010000:00000200:16.0:1616513086.113835:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1687983369325440/t0(0) o400->81904103-3036-4e29-aeea-8a3cee9eca9c@10.242.46.226@o2ib1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513086.113849:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1687983369325440, offset 192 00000400:00000200:16.0:1616513086.113856:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.226@o2ib1 00000400:00000200:16.0:1616513086.113863:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.226@o2ib1 local destination 00000100:00000200:28.0:1616513086.113870:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688842459064576 00000400:00000200:16.0:1616513086.113876:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.226@o2ib1(10.242.46.226@o2ib1:10.242.46.226@o2ib1) : PUT try# 0 00010000:00000200:28.0:1616513086.113882:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688842459064576/t0(0) o400->03e73905-0415-9f7b-aff9-bbbfc5e56638@10.31.142.9@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513086.113883:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1687983369325632 00000800:00000200:16.0:1616513086.113888:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.226@o2ib1 00000800:00000200:16.0:1616513086.113893:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9e00] -> 10.242.46.226@o2ib1 (2) version: 12 00000100:00000200:28.0:1616513086.113896:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688842459064576, offset 192 00010000:00000200:22.0:1616513086.113896:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bf500 x1687983369325632/t0(0) o400->c54746e4-ad7f-5fb1-a40b-8fcf7980c884@10.242.46.226@o2ib1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513086.113896:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da655e00] (20)++ 00000800:00000200:16.0:1616513086.113899:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da655e00] (21)++ 00000400:00000200:28.0:1616513086.113902:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.9@tcp1 00000800:00000200:16.0:1616513086.113903:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da655e00] (22)-- 00000800:00000200:31.2:1616513086.113909:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da655e00] (21)++ 00000400:00000200:28.0:1616513086.113910:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.9@tcp1 routed destination 00000100:00000200:22.0:1616513086.113911:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1687983369325632, offset 192 00000100:00000200:16.0:1616513086.113911:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513086.113917:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.226@o2ib1 00000800:00000200:18.0:1616513086.113919:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da655e00] (22)++ 00000800:00000200:18.0:1616513086.113923:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da655e00] (23)-- 00000400:00000200:28.0:1616513086.113925:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.9@tcp1(10.31.142.9@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513086.113925:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513086.113929:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655e00] (22)-- 00000400:00000200:18.0:1616513086.113930:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.226@o2ib1: PUT: OK 00000800:00000200:28.0:1616513086.113932:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:22.0:1616513086.113934:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.226@o2ib1 local destination 00000400:00000200:18.0:1616513086.113935:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788e58 00000800:00000200:28.0:1616513086.113938:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513086.113941:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513086.113943:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513086.113947:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513086.113947:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.226@o2ib1(10.242.46.226@o2ib1:10.242.46.226@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513086.113952:0:50109:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.113954:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.226@o2ib1 00000100:00000200:28.0:1616513086.113956:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513086.113956:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655e00] (21)-- 00000800:00000200:18.0:1616513086.113958:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513086.113960:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9e00] -> 10.242.46.226@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.113962:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da655e00] (20)++ 00000800:00000200:18.0:1616513086.113962:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513086.113964:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da655e00] (21)++ 00000400:00000200:18.0:1616513086.113964:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513086.113965:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513086.113969:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da655e00] (22)-- 00000400:00000200:18.0:1616513086.113969:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.113972:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccc38 00000800:00000200:18.0:1616513086.113974:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:31.2:1616513086.113977:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da655e00] (21)++ 00000800:00000200:2.0:1616513086.113987:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da655e00] (22)++ 00000800:00000200:2.0:1616513086.113993:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da655e00] (23)-- 00000400:00000200:2.0:1616513086.113995:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513086.113996:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655e00] (22)-- 00000400:00000200:2.0:1616513086.114002:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.226@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.114006:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4acc0 00000800:00000200:2.0:1616513086.114010:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da655e00] (21)-- 00000800:00000200:7.2:1616513086.114767:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.114779:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.114785:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513086.114788:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513086.114792:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.9@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.114798:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.9@tcp1 of length 224 into portal 12 MB=0x5fffe46dab680 00000400:00000200:26.0:1616513086.114804:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.9@tcp1 of length 224/224 into md 0xf964d519 [1] + 50048 00000400:00000200:26.0:1616513086.114809:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.114812:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513086.114817:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.114819:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.114821:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.114831:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688842459068032 00010000:00000200:22.0:1616513086.114837:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647db180 x1688842459068032/t0(0) o400->c0b08507-2b73-2379-ab39-bb32229766e0@10.31.142.9@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.114859:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688842459068032, offset 192 00000400:00000200:22.0:1616513086.114865:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.9@tcp1 00000400:00000200:22.0:1616513086.114871:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.9@tcp1 routed destination 00000400:00000200:22.0:1616513086.114885:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.9@tcp1(10.31.142.9@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.114890:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.114895:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.114897:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.114899:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.114902:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.114910:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.114966:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.114973:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.114976:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.114982:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.114987:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4add0 00000800:00000200:20.0:1616513086.114991:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.114994:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.117346:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513086.117395:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513086.117402:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513086.117407:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513086.117411:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513086.117416:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513086.117420:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.33@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.117425:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.33@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.117429:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.33@tcp1 of length 224 into portal 26 MB=0x6050c8ce87c00 00000400:00000200:14.0:1616513086.117448:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.33@tcp1 of length 224 into portal 12 MB=0x6050c8ce87d00 00000400:00000200:12.0:1616513086.117450:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.33@tcp1 of length 224/224 into md 0xf97cb779 [1] + 0 00000800:00000200:28.0:1616513086.117455:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513086.117455:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2a18 00000400:00000200:12.0:1616513086.117458:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513086.117460:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.33@tcp1 of length 224/224 into md 0xf964d519 [1] + 50272 00000100:00000200:12.0:1616513086.117465:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513086.117466:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.117471:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2a18 00000100:00000200:14.0:1616513086.117473:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513086.117474:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513086.117475:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513086.117477:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.117478:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513086.117480:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.117483:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.117485:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513086.117491:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694401322056704 00010000:00000200:28.0:1616513086.117501:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1694401322056704/t0(0) o400->ab2cabf0-599b-fe49-fdf6-236942762ab9@10.31.141.33@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513086.117515:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694401322056704, offset 192 00000400:00000200:28.0:1616513086.117521:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.33@tcp1 00000400:00000200:28.0:1616513086.117528:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.33@tcp1 routed destination 00000400:00000200:28.0:1616513086.117543:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.33@tcp1(10.31.141.33@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513086.117543:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694401322056960 00000800:00000200:28.0:1616513086.117550:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513086.117555:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513086.117555:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623600 x1694401322056960/t0(0) o400->25b5ba8b-94df-3b0c-d286-9124b299c4eb@10.31.141.33@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513086.117558:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513086.117560:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513086.117565:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513086.117569:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694401322056960, offset 192 00000800:00000200:7.2:1616513086.117570:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513086.117572:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513086.117584:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.33@tcp1 00000400:00000200:22.0:1616513086.117591:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.33@tcp1 routed destination 00000400:00000200:22.0:1616513086.117606:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.33@tcp1(10.31.141.33@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.117612:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.117618:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.117621:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.117623:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.117627:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513086.117627:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513086.117632:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513086.117633:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.117636:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.117643:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.117648:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcca18 00000800:00000200:18.0:1616513086.117651:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.117654:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513086.117702:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.117709:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.117712:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.117718:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.117722:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a3b8 00000800:00000200:20.0:1616513086.117726:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.117729:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.131707:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.131763:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.131772:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.131781:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.249@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.131790:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.249@tcp1 of length 224 into portal 26 MB=0x603ff51087100 00000400:00000200:28.0:1616513086.131796:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.249@tcp1 of length 224/224 into md 0xf97cb789 [1] + 0 00000400:00000200:28.0:1616513086.131801:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3298 00000400:00000200:28.0:1616513086.131803:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.131806:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513086.131811:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3298 00000800:00000200:28.0:1616513086.131814:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.131817:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.131817:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.131821:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.131887:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693244971315456 00010000:00000200:0.0:1616513086.131898:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1693244971315456/t0(0) o400->d0887eae-294d-310f-1d35-ec3c669ea045@10.31.132.249@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.131912:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693244971315456, offset 192 00000400:00000200:0.0:1616513086.131919:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.249@tcp1 00000400:00000200:0.0:1616513086.131927:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.249@tcp1 routed destination 00000400:00000200:0.0:1616513086.131942:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.249@tcp1(10.31.132.249@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.131949:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513086.131955:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.131958:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.131960:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.131965:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.131972:0:50111:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513086.131974:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513086.132027:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.132034:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.132037:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513086.132043:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.132048:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05e48 00000800:00000200:2.0:1616513086.132051:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.132054:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.132484:0:50111:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.132539:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.132549:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513086.132552:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.132558:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.249@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.132566:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.249@tcp1 of length 224 into portal 12 MB=0x603ff51087240 00000400:00000200:18.0:1616513086.132573:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.249@tcp1 of length 224/224 into md 0xf964d519 [1] + 50496 00000400:00000200:18.0:1616513086.132578:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.132580:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.132588:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.132591:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.132593:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.132654:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693244971315776 00010000:00000200:22.0:1616513086.132666:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bd580 x1693244971315776/t0(0) o400->d45fe868-e003-b903-a83d-1de32a2eedbd@10.31.132.249@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.132680:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693244971315776, offset 192 00000400:00000200:22.0:1616513086.132687:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.249@tcp1 00000400:00000200:22.0:1616513086.132694:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.249@tcp1 routed destination 00000400:00000200:22.0:1616513086.132709:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.249@tcp1(10.31.132.249@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.132717:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.132723:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.132726:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.132728:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.132733:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.132738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.132794:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.132801:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.132803:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.132810:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.132814:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a990 00000800:00000200:20.0:1616513086.132820:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.132823:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.143607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.143663:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.143672:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.143683:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.138@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.143693:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.138@tcp1 of length 224 into portal 26 MB=0x603fce4f93780 00000400:00000200:12.0:1616513086.143700:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.138@tcp1 of length 224/224 into md 0xf97cb7a9 [1] + 0 00000400:00000200:12.0:1616513086.143705:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b34b8 00000400:00000200:12.0:1616513086.143707:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.143710:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.143714:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b34b8 00000800:00000200:28.0:1616513086.143717:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.143717:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.143720:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.143723:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.143726:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.143727:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.143736:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.138@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.143745:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.138@tcp1 of length 224 into portal 12 MB=0x603fce4f93840 00000400:00000200:28.0:1616513086.143751:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.138@tcp1 of length 224/224 into md 0x865bbec1 [1] + 82656 00000400:00000200:28.0:1616513086.143756:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.143759:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513086.143767:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.143771:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.143772:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.143780:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234568443776 00010000:00000200:0.0:1616513086.143791:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1693234568443776/t0(0) o400->696090e1-b38a-56c5-c0c6-0675d0852c24@10.31.138.138@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.143805:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234568443776, offset 192 00000400:00000200:0.0:1616513086.143811:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.138@tcp1 00000400:00000200:0.0:1616513086.143819:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.138@tcp1 routed destination 00000400:00000200:0.0:1616513086.143834:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.138@tcp1(10.31.138.138@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.143841:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:4.0:1616513086.143846:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234568443968 00000800:00000200:0.0:1616513086.143847:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.143849:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.143851:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.143857:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:4.0:1616513086.143858:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eb600 x1693234568443968/t0(0) o400->3d2062ca-aea6-6e11-64e7-c5a4f9cbe424@10.31.138.138@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513086.143863:0:50112:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513086.143863:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:4.0:1616513086.143871:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234568443968, offset 192 00000400:00000200:4.0:1616513086.143878:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.138@tcp1 00000400:00000200:4.0:1616513086.143886:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.138@tcp1 routed destination 00000400:00000200:4.0:1616513086.143900:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.138@tcp1(10.31.138.138@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.143906:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513086.143911:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.143913:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513086.143915:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513086.143918:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513086.143920:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.143925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513086.143925:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.143928:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513086.143935:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.143940:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:2.0:1616513086.143944:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.143946:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513086.143996:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.144003:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513086.144006:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513086.144007:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513086.144013:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513086.144017:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00000800:00000200:24.0:1616513086.144021:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.161247:0:50114:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.161302:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.161312:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.161322:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.76@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.161331:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.76@tcp1 of length 224 into portal 12 MB=0x60027ef100980 00000400:00000200:26.0:1616513086.161338:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.76@tcp1 of length 224/224 into md 0xf964d519 [1] + 50720 00000400:00000200:26.0:1616513086.161343:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.161345:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513086.161353:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.161356:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.161357:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513086.161361:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.161428:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689021374794112 00010000:00000200:22.0:1616513086.161457:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dba80 x1689021374794112/t0(0) o400->73679bfc-7c10-4ba8-dce1-68510ca9479a@10.31.129.76@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.161471:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689021374794112, offset 192 00000400:00000200:22.0:1616513086.161478:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.76@tcp1 00000400:00000200:22.0:1616513086.161486:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.76@tcp1 routed destination 00000400:00000200:22.0:1616513086.161501:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.76@tcp1(10.31.129.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.161508:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.161514:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.161517:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.161519:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.161524:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.161529:0:50114:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.161541:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.161548:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.161551:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513086.161558:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.161562:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b188 00000800:00000200:2.0:1616513086.161567:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.161570:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.161644:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.161702:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.161711:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.161720:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.76@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.161729:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.76@tcp1 of length 224 into portal 26 MB=0x60027ef1005c0 00000400:00000200:28.0:1616513086.161735:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.76@tcp1 of length 224/224 into md 0xf97cb7b9 [1] + 0 00000400:00000200:28.0:1616513086.161740:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b22a8 00000400:00000200:28.0:1616513086.161743:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.161746:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513086.161750:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b22a8 00000800:00000200:28.0:1616513086.161753:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.161756:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.161758:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.161760:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.161813:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689021374793152 00010000:00000200:0.0:1616513086.161823:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1689021374793152/t0(0) o400->4aafad6f-23e7-c861-e7ba-0bac714dc22c@10.31.129.76@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.161836:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689021374793152, offset 192 00000400:00000200:0.0:1616513086.161842:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.76@tcp1 00000400:00000200:0.0:1616513086.161849:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.76@tcp1 routed destination 00000400:00000200:0.0:1616513086.161863:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.76@tcp1(10.31.129.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.161869:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.161875:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.161877:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.161879:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.161884:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.161889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.161890:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513086.161942:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.161949:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.161952:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.161958:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.161971:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05f58 00000800:00000200:12.0:1616513086.161974:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.161977:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.190364:0:50117:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.190420:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.190430:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.190451:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.1@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.190465:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.1@tcp1 of length 224 into portal 26 MB=0x604a29b17f880 00000400:00000200:18.0:1616513086.190472:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.1@tcp1 of length 224/224 into md 0xf97cb7c9 [1] + 0 00000800:00000200:26.0:1616513086.190474:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513086.190476:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2aa0 00000400:00000200:18.0:1616513086.190479:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.190483:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513086.190486:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.190487:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2aa0 00000800:00000200:18.0:1616513086.190491:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513086.190494:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513086.190496:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.1@tcp1 : PUT - for me 00000800:00000200:18.0:1616513086.190496:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.190498:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513086.190506:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.1@tcp1 of length 224 into portal 12 MB=0x604a29b17fa40 00000400:00000200:26.0:1616513086.190512:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.1@tcp1 of length 224/224 into md 0xf964d519 [1] + 50944 00000400:00000200:26.0:1616513086.190517:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.190520:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513086.190528:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.190531:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.190533:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.190567:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946293516416 00010000:00000200:0.0:1616513086.190578:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1693946293516416/t0(0) o400->e2dcfa86-5564-4a4c-3d99-c5146e195439@10.31.141.1@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.190594:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946293516416, offset 192 00000100:00000200:22.0:1616513086.190597:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946293516864 00000400:00000200:0.0:1616513086.190602:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.1@tcp1 00000400:00000200:0.0:1616513086.190609:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.1@tcp1 routed destination 00010000:00000200:22.0:1616513086.190610:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9680 x1693946293516864/t0(0) o400->e5683917-c254-9194-644e-448b6cb2284c@10.31.141.1@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.190625:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946293516864, offset 192 00000400:00000200:0.0:1616513086.190625:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.1@tcp1(10.31.141.1@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513086.190631:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.1@tcp1 00000800:00000200:0.0:1616513086.190631:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:22.0:1616513086.190638:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.1@tcp1 routed destination 00000800:00000200:0.0:1616513086.190638:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.190641:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.190643:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.190648:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.190652:0:50117:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:22.0:1616513086.190653:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.1@tcp1(10.31.141.1@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513086.190655:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513086.190660:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.190666:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513086.190667:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513086.190668:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.190670:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513086.190674:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513086.190675:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513086.190678:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.190679:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513086.190683:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513086.190686:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.190691:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:14.0:1616513086.190695:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513086.190738:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.190745:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.190748:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.190754:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.190758:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a330 00000800:00000200:20.0:1616513086.190762:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.190765:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.241717:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.241776:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.241791:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.241800:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.202@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.241810:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.202@tcp1 of length 224 into portal 26 MB=0x6000f29a5acc0 00000400:00000200:2.0:1616513086.241818:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.202@tcp1 of length 224/224 into md 0xf97cb7f9 [1] + 0 00000400:00000200:2.0:1616513086.241823:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2ff0 00000400:00000200:2.0:1616513086.241826:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.241829:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513086.241833:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2ff0 00000800:00000200:26.0:1616513086.241837:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.241838:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.241841:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513086.241845:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.241848:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.241849:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.241858:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.202@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.241867:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.202@tcp1 of length 224 into portal 12 MB=0x6000f29a5ad40 00000400:00000200:26.0:1616513086.241873:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.202@tcp1 of length 224/224 into md 0xf964d519 [1] + 51168 00000400:00000200:26.0:1616513086.241879:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.241885:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513086.241893:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.241897:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.241899:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.241913:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688914983496896 00010000:00000200:0.0:1616513086.241926:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688914983496896/t0(0) o400->61030dc8-493f-5adc-9f0e-6ea8e7c20a67@10.31.141.202@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.241942:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688914983496896, offset 192 00000400:00000200:0.0:1616513086.241948:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.202@tcp1 00000400:00000200:0.0:1616513086.241956:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.202@tcp1 routed destination 00000100:00000200:22.0:1616513086.241962:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688914983497024 00000400:00000200:0.0:1616513086.241971:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.202@tcp1(10.31.141.202@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:22.0:1616513086.241977:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9f80 x1688914983497024/t0(0) o400->1b04f10e-0d23-5f7b-6a77-200e9a587b27@10.31.141.202@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513086.241978:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513086.241984:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.241987:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.241990:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513086.241992:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688914983497024, offset 192 00000800:00000200:0.0:1616513086.241995:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513086.241998:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.202@tcp1 00000800:00000200:7.2:1616513086.242000:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513086.242003:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513086.242005:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.202@tcp1 routed destination 00000400:00000200:22.0:1616513086.242021:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.202@tcp1(10.31.141.202@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.242028:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.242034:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.242037:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.242039:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.242044:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.242052:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513086.242056:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.242064:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.242067:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513086.242067:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.242073:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.242079:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04b28 00000800:00000200:14.0:1616513086.242083:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513086.242108:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.242115:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.242118:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.242125:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.242129:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b540 00000800:00000200:12.0:1616513086.242133:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.242136:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.267540:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513086.267597:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.267607:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513086.267616:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.241@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.267626:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.241@tcp1 of length 224 into portal 26 MB=0x603e0e92ab040 00000400:00000200:20.0:1616513086.267633:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.241@tcp1 of length 224/224 into md 0xf97cb819 [1] + 0 00000400:00000200:20.0:1616513086.267637:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3c28 00000400:00000200:20.0:1616513086.267640:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.267643:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513086.267648:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3c28 00000800:00000200:20.0:1616513086.267651:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.267652:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.267654:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513086.267658:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513086.267661:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.267662:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.267671:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.241@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.267680:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.241@tcp1 of length 224 into portal 12 MB=0x603e0e92ab0c0 00000400:00000200:28.0:1616513086.267686:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.241@tcp1 of length 224/224 into md 0x865bbec1 [1] + 82880 00000400:00000200:28.0:1616513086.267692:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.267694:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513086.267701:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.267704:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.267706:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.267713:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114379710528 00010000:00000200:0.0:1616513086.267725:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1693114379710528/t0(0) o400->13f8c051-288e-0e38-c341-9c0a716fa22b@10.31.136.241@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.267739:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114379710528, offset 192 00000400:00000200:0.0:1616513086.267746:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.241@tcp1 00000400:00000200:0.0:1616513086.267753:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.241@tcp1 routed destination 00000100:00000200:4.0:1616513086.267765:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114379710656 00000400:00000200:0.0:1616513086.267769:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.241@tcp1(10.31.136.241@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.267775:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:4.0:1616513086.267779:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eda00 x1693114379710656/t0(0) o400->8ffce364-3790-cdc4-b915-297610b55491@10.31.136.241@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513086.267783:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.267786:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.267788:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513086.267793:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114379710656, offset 192 00000800:00000200:0.0:1616513086.267793:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513086.267800:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.241@tcp1 00000100:00000200:0.0:1616513086.267800:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513086.267801:0:50125:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513086.267807:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.241@tcp1 routed destination 00000400:00000200:4.0:1616513086.267823:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.241@tcp1(10.31.136.241@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.267829:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513086.267835:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.267838:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513086.267840:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513086.267845:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.267850:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513086.267857:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.267864:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.267867:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.267873:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.267879:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:18.0:1616513086.267883:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.267885:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513086.267905:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.267912:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513086.267915:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513086.267916:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513086.267922:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513086.267926:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bba90 00000800:00000200:24.0:1616513086.267930:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.268682:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.268738:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.268748:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.268757:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.23@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.268766:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.23@tcp1 of length 224 into portal 26 MB=0x6001fa39a6a40 00000400:00000200:2.0:1616513086.268773:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.23@tcp1 of length 224/224 into md 0xf97cb839 [1] + 0 00000400:00000200:2.0:1616513086.268778:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2bb0 00000400:00000200:2.0:1616513086.268781:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.268784:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513086.268788:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2bb0 00000800:00000200:2.0:1616513086.268791:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.268793:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.268794:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.268798:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.268801:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688985749056064 00010000:00000200:0.0:1616513086.268806:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688985749056064/t0(0) o400->9c2306b8-8848-481c-0df5-9c29530048cf@10.31.140.23@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.268817:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688985749056064, offset 192 00000400:00000200:0.0:1616513086.268821:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.23@tcp1 00000400:00000200:0.0:1616513086.268827:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.23@tcp1 routed destination 00000400:00000200:0.0:1616513086.268839:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.23@tcp1(10.31.140.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.268845:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513086.268849:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.268851:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.268853:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.268855:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513086.268860:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513086.268861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.268871:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.268875:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.268876:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513086.268881:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.268882:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.268885:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:18.0:1616513086.268888:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.268895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513086.268906:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.268914:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513086.268916:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.268923:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.23@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.268931:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.23@tcp1 of length 224 into portal 12 MB=0x6001fa39a6ac0 00000400:00000200:14.0:1616513086.268938:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.23@tcp1 of length 224/224 into md 0xf964d519 [1] + 51392 00000400:00000200:14.0:1616513086.268943:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.268945:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513086.268953:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.268956:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.268958:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.269020:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688985749056192 00010000:00000200:22.0:1616513086.269033:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7ad00 x1688985749056192/t0(0) o400->1143f758-8ebd-20e6-32a0-4251c85d881a@10.31.140.23@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.269047:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688985749056192, offset 192 00000400:00000200:22.0:1616513086.269054:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.23@tcp1 00000400:00000200:22.0:1616513086.269061:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.23@tcp1 routed destination 00000400:00000200:22.0:1616513086.269076:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.23@tcp1(10.31.140.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.269083:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.269089:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.269091:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.269093:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.269098:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.269103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.269157:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.269164:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.269166:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513086.269173:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.269177:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a198 00000800:00000200:28.0:1616513086.269181:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.269184:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.273789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.273842:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.273851:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.273860:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.186@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.273869:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.186@tcp1 of length 224 into portal 12 MB=0x5ffdeeb4ea7c0 00000400:00000200:26.0:1616513086.273875:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.186@tcp1 of length 224/224 into md 0xf964d519 [1] + 51616 00000400:00000200:26.0:1616513086.273881:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.273883:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513086.273889:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.273892:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513086.273894:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.273897:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.273953:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707779176384 00010000:00000200:22.0:1616513086.273965:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647df080 x1688707779176384/t0(0) o400->f6842de2-1288-27bb-ecbe-0af5410ce625@10.31.138.186@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.273980:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688707779176384, offset 192 00000400:00000200:22.0:1616513086.273987:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.186@tcp1 00000800:00000200:11.2:1616513086.273990:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513086.273995:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.186@tcp1 routed destination 00000400:00000200:22.0:1616513086.274012:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.186@tcp1(10.31.138.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.274018:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.274024:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.274027:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.274029:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.274034:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.274039:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513086.274048:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513086.274051:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513086.274057:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513086.274058:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.274061:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.274066:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.186@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.274068:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.274074:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b7e8 00000400:00000200:20.0:1616513086.274075:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.186@tcp1 of length 224 into portal 26 MB=0x5ffdeeb4ea740 00000800:00000200:2.0:1616513086.274078:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.274081:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513086.274082:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.186@tcp1 of length 224/224 into md 0xf97cb869 [1] + 0 00000400:00000200:20.0:1616513086.274087:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3540 00000400:00000200:20.0:1616513086.274090:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.274093:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513086.274097:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3540 00000800:00000200:24.0:1616513086.274102:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.274106:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.274109:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.274111:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.274161:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707779176256 00010000:00000200:0.0:1616513086.274171:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688707779176256/t0(0) o400->dae3bbb4-0950-e1fd-7eae-f781381e00bc@10.31.138.186@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.274185:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688707779176256, offset 192 00000400:00000200:0.0:1616513086.274190:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.186@tcp1 00000400:00000200:0.0:1616513086.274197:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.186@tcp1 routed destination 00000400:00000200:0.0:1616513086.274211:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.186@tcp1(10.31.138.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.274217:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.274223:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.274225:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.274227:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.274232:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.274237:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.274239:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513086.274291:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.274298:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.274301:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513086.274307:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.274313:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04198 00000800:00000200:28.0:1616513086.274316:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.274319:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.277539:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513086.277596:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.277605:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.277614:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.221@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.277623:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.221@tcp1 of length 224 into portal 26 MB=0x60020c467f100 00000400:00000200:24.0:1616513086.277629:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.221@tcp1 of length 224/224 into md 0xf97cb889 [1] + 0 00000400:00000200:24.0:1616513086.277634:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2c38 00000400:00000200:24.0:1616513086.277636:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.277639:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513086.277644:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2c38 00000800:00000200:24.0:1616513086.277647:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.277650:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513086.277650:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513086.277657:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513086.277660:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.277660:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513086.277668:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688990594363648 00000400:00000200:20.0:1616513086.277669:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.221@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.277678:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.221@tcp1 of length 224 into portal 12 MB=0x60020c467f1c0 00010000:00000200:0.0:1616513086.277679:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688990594363648/t0(0) o400->3cb75dc2-5b45-2d4a-3ec9-206bab081703@10.31.136.221@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513086.277684:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.221@tcp1 of length 224/224 into md 0x865bbec1 [1] + 83104 00000400:00000200:20.0:1616513086.277689:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.277692:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513086.277698:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688990594363648, offset 192 00000800:00000200:20.0:1616513086.277699:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.277702:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.277704:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513086.277704:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.221@tcp1 00000400:00000200:0.0:1616513086.277712:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.221@tcp1 routed destination 00000400:00000200:0.0:1616513086.277727:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.221@tcp1(10.31.136.221@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.277734:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513086.277739:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.277742:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.277744:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.277749:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.277756:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513086.277756:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:4.0:1616513086.277763:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688990594363840 00010000:00000200:4.0:1616513086.277775:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38ec00 x1688990594363840/t0(0) o400->ad2cab1e-5a02-7a74-623d-25ab0f9e386e@10.31.136.221@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.277790:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688990594363840, offset 192 00000400:00000200:4.0:1616513086.277796:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.221@tcp1 00000400:00000200:4.0:1616513086.277803:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.221@tcp1 routed destination 00000800:00000200:26.0:1616513086.277810:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.277817:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:4.0:1616513086.277818:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.221@tcp1(10.31.136.221@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513086.277819:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.277826:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:4.0:1616513086.277831:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513086.277832:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05980 00000800:00000200:26.0:1616513086.277835:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513086.277837:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513086.277838:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513086.277842:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513086.277844:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513086.277849:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.277854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.277907:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.277914:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.277916:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513086.277918:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513086.277923:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.277927:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbdc0 00000800:00000200:12.0:1616513086.277931:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.281863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513086.281865:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513086.281917:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.281926:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.281935:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.4@tcp1 : PUT - for me 00000800:00000200:28.0:1616513086.281936:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513086.281944:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.4@tcp1 of length 224 into portal 12 MB=0x5ffd65a9a7100 00000800:00000200:28.0:1616513086.281945:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513086.281951:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.4@tcp1 of length 224/224 into md 0xf964d519 [1] + 51840 00000400:00000200:28.0:1616513086.281954:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.4@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.281956:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.281958:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513086.281963:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.4@tcp1 of length 224 into portal 26 MB=0x5ffd65a9a7040 00000800:00000200:2.0:1616513086.281965:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.281968:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513086.281970:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.4@tcp1 of length 224/224 into md 0xf97cb8a9 [1] + 0 00000800:00000200:14.0:1616513086.281970:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.281974:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513086.281975:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2440 00000400:00000200:28.0:1616513086.281977:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.281986:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513086.281991:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2440 00000800:00000200:28.0:1616513086.281994:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.281996:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.281998:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.282001:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513086.282043:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688670991708416 00010000:00000200:22.0:1616513086.282055:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9f80 x1688670991708416/t0(0) o400->85257c89-a419-6646-1283-43608fc10581@10.31.130.4@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513086.282057:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688670991708224 00010000:00000200:0.0:1616513086.282067:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688670991708224/t0(0) o400->7a714505-adf1-1368-f34e-26127e04e806@10.31.130.4@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513086.282070:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688670991708416, offset 192 00000400:00000200:22.0:1616513086.282077:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.4@tcp1 00000100:00000200:0.0:1616513086.282081:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688670991708224, offset 192 00000400:00000200:22.0:1616513086.282084:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.4@tcp1 routed destination 00000400:00000200:0.0:1616513086.282086:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.4@tcp1 00000400:00000200:0.0:1616513086.282093:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.4@tcp1 routed destination 00000400:00000200:22.0:1616513086.282100:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.4@tcp1(10.31.130.4@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.282106:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513086.282107:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.4@tcp1(10.31.130.4@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.282112:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.282114:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.282115:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.282117:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.282119:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.282121:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.282122:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513086.282123:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513086.282127:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.282128:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.282133:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.282134:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513086.282182:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.282189:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.282191:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.282198:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.282202:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b3a8 00000800:00000200:24.0:1616513086.282202:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513086.282207:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513086.282208:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513086.282210:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513086.282211:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513086.282218:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513086.282223:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb042a8 00000800:00000200:24.0:1616513086.282226:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.282229:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.298084:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.298140:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.298149:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.298158:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.31@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.298167:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.31@tcp1 of length 224 into portal 12 MB=0x5ffd469c78100 00000400:00000200:28.0:1616513086.298173:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.31@tcp1 of length 224/224 into md 0x865bbec1 [1] + 83328 00000400:00000200:28.0:1616513086.298178:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.298181:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513086.298187:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.298189:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.298191:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.298194:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.298254:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662656385280 00010000:00000200:4.0:1616513086.298266:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ee300 x1688662656385280/t0(0) o400->151b2d03-7371-8203-1779-c02fad14911b@10.31.138.31@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.298281:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662656385280, offset 192 00000400:00000200:4.0:1616513086.298287:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.31@tcp1 00000400:00000200:4.0:1616513086.298295:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.31@tcp1 routed destination 00000400:00000200:4.0:1616513086.298310:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.31@tcp1(10.31.138.31@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.298316:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.298323:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.298325:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.298327:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.298332:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.298339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.298396:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.298403:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.298407:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513086.298413:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.298417:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb430 00000800:00000200:14.0:1616513086.298421:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.298424:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.300763:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513086.300819:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.300828:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.300837:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.31@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.300845:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.31@tcp1 of length 224 into portal 26 MB=0x5ffd469c77fc0 00000400:00000200:24.0:1616513086.300852:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.31@tcp1 of length 224/224 into md 0xf97cb8b9 [1] + 0 00000400:00000200:24.0:1616513086.300857:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b33a8 00000400:00000200:24.0:1616513086.300860:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.300875:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513086.300879:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b33a8 00000800:00000200:12.0:1616513086.300881:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.300882:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.300885:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.300887:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.300941:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662656384960 00010000:00000200:0.0:1616513086.300952:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688662656384960/t0(0) o400->f5699596-ff0a-9eb2-73f0-d9abb748b116@10.31.138.31@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.300965:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662656384960, offset 192 00000400:00000200:0.0:1616513086.300972:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.31@tcp1 00000400:00000200:0.0:1616513086.300979:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.31@tcp1 routed destination 00000400:00000200:0.0:1616513086.300993:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.31@tcp1(10.31.138.31@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.300999:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.301004:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.301007:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.301009:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.301013:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.301018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.301019:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.301072:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.301078:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.301081:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.301088:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.301094:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb056d8 00000800:00000200:20.0:1616513086.301097:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.301100:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513086.330973:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927184, portal 4 00000100:00000200:19.0:1616513086.330979:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927184, offset 0 00000400:00000200:19.0:1616513086.330986:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:19.0:1616513086.330992:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:19.0:1616513086.331007:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513086.331014:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:19.0:1616513086.331021:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513086.331024:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:19.0:1616513086.331026:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:19.0:1616513086.331032:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513086.331079:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513086.331100:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513086.331106:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513086.331109:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513086.331116:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513086.331124:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc567080 x1693067725927184/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513086.331137:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707dc0 00000800:00000200:1.0:1616513086.331144:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513086.331147:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513086.331219:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513086.331228:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513086.331234:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513086.331238:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513086.331243:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513086.331252:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627310 00000400:00000200:1.0:1616513086.331259:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffe75 [1] + 192 00000400:00000200:1.0:1616513086.331264:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513086.331269:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc567080 x1693067725927184/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513086.331282:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513086.331285:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513086.331287:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:19.0:1616513086.331291:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc567080 x1693067725927184/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513086.331304:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc567080 x1693067725927184/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513086.331314:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000800:00000200:7.2:1616513086.356117:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.356176:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.356187:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.356196:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.122@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.356207:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.122@tcp1 of length 224 into portal 12 MB=0x605518fbe28c0 00000400:00000200:26.0:1616513086.356214:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.122@tcp1 of length 224/224 into md 0xf964d519 [1] + 52064 00000400:00000200:26.0:1616513086.356222:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.356225:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.356232:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.356235:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.356238:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.356240:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.356302:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694697722357952 00010000:00000200:22.0:1616513086.356316:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647df980 x1694697722357952/t0(0) o400->0db6746d-49eb-e518-832b-7c1ac3491868@10.31.143.122@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.356330:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694697722357952, offset 192 00000400:00000200:22.0:1616513086.356337:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.122@tcp1 00000400:00000200:22.0:1616513086.356345:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.122@tcp1 routed destination 00000400:00000200:22.0:1616513086.356359:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.122@tcp1(10.31.143.122@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.356366:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.356372:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.356375:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.356378:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.356384:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.356389:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.356457:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.356465:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.356468:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513086.356471:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.356475:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.356479:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ab28 00000800:00000200:14.0:1616513086.356485:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.357381:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.357456:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.357473:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.357482:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.122@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.357491:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.122@tcp1 of length 224 into portal 26 MB=0x605518fbe2580 00000400:00000200:12.0:1616513086.357498:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.122@tcp1 of length 224/224 into md 0xf97cb8d9 [1] + 0 00000400:00000200:12.0:1616513086.357502:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2550 00000400:00000200:12.0:1616513086.357505:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.357508:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.357514:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2550 00000800:00000200:12.0:1616513086.357517:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.357519:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.357520:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.357523:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.357541:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694697722357120 00010000:00000200:0.0:1616513086.357552:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1694697722357120/t0(0) o400->41794c26-8e44-b6e9-39fa-7bfb9c235d9f@10.31.143.122@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.357566:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694697722357120, offset 192 00000400:00000200:0.0:1616513086.357573:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.122@tcp1 00000400:00000200:0.0:1616513086.357581:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.122@tcp1 routed destination 00000400:00000200:0.0:1616513086.357596:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.122@tcp1(10.31.143.122@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.357602:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.357608:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.357611:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.357613:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.357618:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.357623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.357624:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.357685:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.357693:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.357695:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.357702:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.357708:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a08 00000800:00000200:20.0:1616513086.357712:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.357715:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.395029:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513086.395046:0:50138:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513086.395085:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.395095:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.395104:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.122@tcp1 : PUT - for me 00000800:00000200:18.0:1616513086.395107:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513086.395114:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.122@tcp1 of length 224 into portal 26 MB=0x603fc08413c80 00000800:00000200:18.0:1616513086.395117:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513086.395121:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.122@tcp1 of length 224/224 into md 0xf97cb8e9 [1] + 0 00000400:00000200:18.0:1616513086.395126:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.122@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.395127:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b26e8 00000400:00000200:24.0:1616513086.395130:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.395133:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513086.395136:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.122@tcp1 of length 224 into portal 12 MB=0x603fc08413d00 00000400:00000200:24.0:1616513086.395138:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b26e8 00000800:00000200:24.0:1616513086.395142:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513086.395144:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.122@tcp1 of length 224/224 into md 0xf964d519 [1] + 52288 00000800:00000200:24.0:1616513086.395145:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.395146:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513086.395149:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513086.395150:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513086.395152:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.395162:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.395166:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.395166:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.395170:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.395210:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230865398912 00010000:00000200:0.0:1616513086.395222:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1693230865398912/t0(0) o400->538f804f-2222-3605-0684-bb3e5afd192e@10.31.138.122@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513086.395235:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230865399040 00000100:00000200:0.0:1616513086.395237:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230865398912, offset 192 00000400:00000200:0.0:1616513086.395243:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.122@tcp1 00010000:00000200:22.0:1616513086.395250:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b8d80 x1693230865399040/t0(0) o400->ba3bc3c3-80f6-c80e-b7e7-41cdc382d48a@10.31.138.122@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513086.395252:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.122@tcp1 routed destination 00000100:00000200:22.0:1616513086.395264:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230865399040, offset 192 00000400:00000200:0.0:1616513086.395267:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.122@tcp1(10.31.138.122@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513086.395271:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.122@tcp1 00000800:00000200:0.0:1616513086.395278:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:22.0:1616513086.395281:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.122@tcp1 routed destination 00000800:00000200:0.0:1616513086.395285:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.395288:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.395290:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:22.0:1616513086.395296:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.122@tcp1(10.31.138.122@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.395296:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.395300:0:50138:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.395303:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:0.0:1616513086.395303:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513086.395309:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.395311:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.395313:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513086.395313:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513086.395319:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513086.395320:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.395323:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513086.395324:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513086.395330:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.395336:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb045d8 00000800:00000200:2.0:1616513086.395340:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.395343:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513086.395390:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.395397:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.395400:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.395407:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.395411:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a6e8 00000800:00000200:20.0:1616513086.395415:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.395418:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.399155:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.399212:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.399221:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.399230:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.89@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.399239:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.89@tcp1 of length 224 into portal 26 MB=0x5ffdd57ce0540 00000400:00000200:12.0:1616513086.399246:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.89@tcp1 of length 224/224 into md 0xf97cb8f9 [1] + 0 00000400:00000200:12.0:1616513086.399251:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2908 00000400:00000200:12.0:1616513086.399253:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.399256:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.399260:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2908 00000800:00000200:12.0:1616513086.399263:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.399266:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.399268:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.399271:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.399323:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701009528128 00010000:00000200:0.0:1616513086.399334:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688701009528128/t0(0) o400->576e7418-4413-18ed-a2ba-cd2bd8fd1f86@10.31.141.89@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.399348:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701009528128, offset 192 00000400:00000200:0.0:1616513086.399354:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.89@tcp1 00000400:00000200:0.0:1616513086.399362:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.89@tcp1 routed destination 00000800:00000200:7.2:1616513086.399368:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513086.399377:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.89@tcp1(10.31.141.89@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.399384:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513086.399389:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.399392:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.399395:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513086.399399:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:0.0:1616513086.399406:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513086.399422:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.399431:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.399451:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.89@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.399459:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.89@tcp1 of length 224 into portal 12 MB=0x5ffdd57ce0840 00000400:00000200:26.0:1616513086.399465:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.89@tcp1 of length 224/224 into md 0xf964d519 [1] + 52512 00000400:00000200:26.0:1616513086.399470:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.399472:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.399475:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513086.399478:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513086.399481:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:26.0:1616513086.399483:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513086.399483:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513086.399486:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.399486:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.399493:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.399498:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000800:00000200:18.0:1616513086.399502:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.399543:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701009528896 00010000:00000200:22.0:1616513086.399555:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647de780 x1688701009528896/t0(0) o400->20fe7d43-08a7-175d-9e77-cee08021402b@10.31.141.89@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.399570:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701009528896, offset 192 00000400:00000200:22.0:1616513086.399576:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.89@tcp1 00000400:00000200:22.0:1616513086.399583:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.89@tcp1 routed destination 00000400:00000200:22.0:1616513086.399598:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.89@tcp1(10.31.141.89@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.399605:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.399610:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.399613:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.399615:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.399620:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.399625:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.399679:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.399685:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.399688:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.399695:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.399699:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a440 00000800:00000200:20.0:1616513086.399703:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.399706:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.435240:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513086.435298:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.435308:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.435318:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.59@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.435329:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.59@tcp1 of length 224 into portal 26 MB=0x6001b512d19c0 00000400:00000200:24.0:1616513086.435336:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.59@tcp1 of length 224/224 into md 0xf97cb909 [1] + 0 00000400:00000200:24.0:1616513086.435341:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2d48 00000400:00000200:24.0:1616513086.435344:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.435347:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513086.435352:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2d48 00000800:00000200:12.0:1616513086.435352:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.435355:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.435358:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.435360:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.435418:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688967186291136 00010000:00000200:0.0:1616513086.435430:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688967186291136/t0(0) o400->0565dedf-9a9a-16a9-c10e-6a4307acdd29@10.31.133.59@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.435464:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688967186291136, offset 192 00000400:00000200:0.0:1616513086.435470:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.59@tcp1 00000400:00000200:0.0:1616513086.435479:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.59@tcp1 routed destination 00000400:00000200:0.0:1616513086.435495:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.59@tcp1(10.31.133.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.435501:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513086.435508:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.435511:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.435514:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.435520:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513086.435527:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513086.435528:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.435585:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.435592:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.435595:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513086.435602:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.435608:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb044c8 00000800:00000200:14.0:1616513086.435611:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.435614:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.435648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513086.435709:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.435719:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513086.435728:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.59@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.435737:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.59@tcp1 of length 224 into portal 12 MB=0x6001b512d1ac0 00000400:00000200:20.0:1616513086.435743:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.59@tcp1 of length 224/224 into md 0x865bbec1 [1] + 83552 00000400:00000200:20.0:1616513086.435749:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.435752:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513086.435759:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.435762:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513086.435764:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.435766:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.435824:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688967186291392 00010000:00000200:4.0:1616513086.435837:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38d100 x1688967186291392/t0(0) o400->ee79adca-3566-c371-535a-4139bebd32f1@10.31.133.59@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.435851:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688967186291392, offset 192 00000400:00000200:4.0:1616513086.435858:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.59@tcp1 00000400:00000200:4.0:1616513086.435866:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.59@tcp1 routed destination 00000400:00000200:4.0:1616513086.435881:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.59@tcp1(10.31.133.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.435888:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513086.435893:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.435896:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513086.435898:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513086.435903:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.435908:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.435918:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.435924:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.435926:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513086.435928:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513086.435933:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.435937:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb3a8 00000800:00000200:12.0:1616513086.435941:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.436035:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.436089:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.436098:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.436108:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.63@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.436117:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.63@tcp1 of length 224 into portal 26 MB=0x5ffdbf7441540 00000400:00000200:18.0:1616513086.436124:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.63@tcp1 of length 224/224 into md 0xf97cb929 [1] + 0 00000400:00000200:18.0:1616513086.436129:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3b18 00000400:00000200:18.0:1616513086.436131:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.436134:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513086.436139:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3b18 00000800:00000200:18.0:1616513086.436142:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.436143:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.436145:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.436148:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.436152:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695094908224 00010000:00000200:0.0:1616513086.436158:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688695094908224/t0(0) o400->cf67d662-1842-57b5-32ef-0bdcebdb4e10@10.31.128.63@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.436169:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695094908224, offset 192 00000400:00000200:0.0:1616513086.436174:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.63@tcp1 00000400:00000200:0.0:1616513086.436180:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.63@tcp1 routed destination 00000400:00000200:0.0:1616513086.436193:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.63@tcp1(10.31.128.63@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513086.436195:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.436199:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513086.436203:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513086.436204:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.436206:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.436208:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513086.436210:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.436211:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513086.436213:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513086.436215:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513086.436216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513086.436217:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.63@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.436223:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.63@tcp1 of length 224 into portal 12 MB=0x5ffdbf7441800 00000800:00000200:14.0:1616513086.436225:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513086.436229:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.63@tcp1 of length 224/224 into md 0x865bbec1 [1] + 83776 00000800:00000200:14.0:1616513086.436229:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.436230:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.436233:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513086.436234:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513086.436235:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513086.436236:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513086.436239:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513086.436240:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05cb0 00000800:00000200:20.0:1616513086.436241:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513086.436242:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513086.436243:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.436252:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695094908928 00010000:00000200:4.0:1616513086.436258:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38d580 x1688695094908928/t0(0) o400->641a6f7c-5663-e6df-1c08-494d5ee503b4@10.31.128.63@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.436280:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695094908928, offset 192 00000400:00000200:4.0:1616513086.436285:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.63@tcp1 00000400:00000200:4.0:1616513086.436291:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.63@tcp1 routed destination 00000400:00000200:4.0:1616513086.436304:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.63@tcp1(10.31.128.63@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.436310:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513086.436314:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.436317:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513086.436318:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513086.436322:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.436326:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.436335:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.436339:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513086.436342:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513086.436343:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513086.436347:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513086.436351:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb760 00000800:00000200:24.0:1616513086.436353:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.445030:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.445091:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.445101:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.445109:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.445118:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x60399133fde80 00000400:00000200:2.0:1616513086.445125:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0xf964d519 [1] + 52736 00000400:00000200:2.0:1616513086.445130:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.445133:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513086.445140:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.445143:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.445144:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.445149:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.445206:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805848096384 00010000:00000200:22.0:1616513086.445290:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9680 x1692805848096384/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:712/0 lens 336/336 e 0 to 0 dl 1616513092 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513086.445304:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805848096384, offset 192 00000400:00000200:22.0:1616513086.445311:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:22.0:1616513086.445319:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:22.0:1616513086.445332:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.445338:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.445344:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.445346:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.445348:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.445353:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.445358:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.445412:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.445419:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.445422:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.445428:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.445432:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a220 00000800:00000200:26.0:1616513086.445450:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.445453:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.470805:0:50146:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.470861:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.470872:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.470882:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.144@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.470891:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.144@tcp1 of length 224 into portal 26 MB=0x5fff029611f00 00000400:00000200:18.0:1616513086.470898:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.144@tcp1 of length 224/224 into md 0xf97cb939 [1] + 0 00000400:00000200:18.0:1616513086.470903:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3a90 00000400:00000200:18.0:1616513086.470906:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.470909:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513086.470913:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3a90 00000800:00000200:18.0:1616513086.470916:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.470919:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.470923:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.470925:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.470931:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513086.470931:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.470938:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.144@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.470943:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.144@tcp1 of length 224 into portal 12 MB=0x5fff029612200 00000400:00000200:18.0:1616513086.470949:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.144@tcp1 of length 224/224 into md 0xf964d519 [1] + 53072 00000400:00000200:18.0:1616513086.470953:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.470955:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.470961:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.470963:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.470964:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.470980:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688781835017984 00010000:00000200:0.0:1616513086.470992:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688781835017984/t0(0) o400->01abf4a1-8de8-bbf2-5eaa-f44614025284@10.31.141.144@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.471007:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688781835017984, offset 192 00000400:00000200:0.0:1616513086.471014:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.144@tcp1 00000400:00000200:0.0:1616513086.471022:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.144@tcp1 routed destination 00000100:00000200:22.0:1616513086.471030:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688781835018752 00000400:00000200:0.0:1616513086.471037:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.144@tcp1(10.31.141.144@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:22.0:1616513086.471044:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b9f80 x1688781835018752/t0(0) o400->1826af94-a85c-6752-1122-cb4919ac2075@10.31.141.144@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513086.471045:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.471051:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.471053:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.471056:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513086.471058:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688781835018752, offset 192 00000800:00000200:0.0:1616513086.471061:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513086.471065:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.144@tcp1 00000800:00000200:11.2:1616513086.471068:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.471068:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513086.471072:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.144@tcp1 routed destination 00000400:00000200:22.0:1616513086.471087:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.144@tcp1(10.31.141.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.471094:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.471100:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.471103:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.471105:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.471110:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.471115:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513086.471123:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513086.471131:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513086.471139:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513086.471139:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.471141:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513086.471142:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513086.471142:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.471148:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.471149:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.471152:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b210 00000400:00000200:28.0:1616513086.471154:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04aa0 00000800:00000200:14.0:1616513086.471156:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513086.471157:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.471160:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.471241:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.471295:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.471304:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.471313:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.13@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.471322:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.13@tcp1 of length 224 into portal 26 MB=0x5ffe5c50aec00 00000400:00000200:12.0:1616513086.471329:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.13@tcp1 of length 224/224 into md 0xf97cb979 [1] + 0 00000400:00000200:12.0:1616513086.471334:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2088 00000400:00000200:12.0:1616513086.471337:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.471339:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.471344:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2088 00000800:00000200:12.0:1616513086.471347:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.471350:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513086.471350:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513086.471357:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737201974272 00000800:00000200:12.0:1616513086.471358:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.471361:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.471362:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00010000:00000200:0.0:1616513086.471363:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688737201974272/t0(0) o400->770c0466-5296-1db4-40d7-275fc4909d93@10.31.142.13@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:24.0:1616513086.471371:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.13@tcp1 : PUT - for me 00000100:00000200:0.0:1616513086.471376:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688737201974272, offset 192 00000400:00000200:24.0:1616513086.471379:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.13@tcp1 of length 224 into portal 12 MB=0x5ffe5c50aec80 00000400:00000200:0.0:1616513086.471381:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.13@tcp1 00000400:00000200:24.0:1616513086.471386:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.13@tcp1 of length 224/224 into md 0x865bbec1 [1] + 84000 00000400:00000200:0.0:1616513086.471387:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.13@tcp1 routed destination 00000400:00000200:24.0:1616513086.471391:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.471394:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513086.471400:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.13@tcp1(10.31.142.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513086.471405:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.471406:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513086.471408:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.471410:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513086.471411:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.471413:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.471415:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.471418:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513086.471422:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513086.471423:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.471449:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.471456:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.471459:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513086.471460:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513086.471466:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:4.0:1616513086.471467:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737201974400 00000400:00000200:20.0:1616513086.471471:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:20.0:1616513086.471475:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:4.0:1616513086.471480:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e4380 x1688737201974400/t0(0) o400->97f01c07-951e-b693-ebc7-3aca6aa5878f@10.31.142.13@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.471494:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688737201974400, offset 192 00000400:00000200:4.0:1616513086.471501:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.13@tcp1 00000400:00000200:4.0:1616513086.471508:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.13@tcp1 routed destination 00000400:00000200:4.0:1616513086.471523:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.13@tcp1(10.31.142.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.471530:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.471536:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.471539:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.471541:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.471546:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.471552:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.471562:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.471566:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.471568:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513086.471571:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.471573:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.471577:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb320 00000800:00000200:18.0:1616513086.471580:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.473676:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.473729:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.473739:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.473748:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.153@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.473757:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.153@tcp1 of length 224 into portal 26 MB=0x603e124377ec0 00000400:00000200:2.0:1616513086.473763:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.153@tcp1 of length 224/224 into md 0xf97cb989 [1] + 0 00000400:00000200:2.0:1616513086.473768:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2f68 00000400:00000200:2.0:1616513086.473771:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.473774:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513086.473778:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2f68 00000800:00000200:2.0:1616513086.473781:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.473783:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.473784:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.473788:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.473842:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115370405568 00010000:00000200:0.0:1616513086.473853:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1693115370405568/t0(0) o400->8833addb-84bb-22fc-1b9b-27ff35109d86@10.31.136.153@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.473867:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693115370405568, offset 192 00000400:00000200:0.0:1616513086.473874:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.153@tcp1 00000400:00000200:0.0:1616513086.473882:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.153@tcp1 routed destination 00000400:00000200:0.0:1616513086.473897:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.153@tcp1(10.31.136.153@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.473903:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.473909:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.473911:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.473914:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.473918:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.473924:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.473925:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513086.473979:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.473985:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.473988:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513086.473994:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.474000:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04908 00000800:00000200:28.0:1616513086.474004:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.474007:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.475270:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.475323:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.475332:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.475340:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.153@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.475349:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.153@tcp1 of length 224 into portal 12 MB=0x603e124377f80 00000400:00000200:26.0:1616513086.475355:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.153@tcp1 of length 224/224 into md 0xf964d519 [1] + 53296 00000400:00000200:26.0:1616513086.475360:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.475365:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513086.475371:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.475375:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.475376:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513086.475381:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.475462:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115370405760 00010000:00000200:22.0:1616513086.475474:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dc800 x1693115370405760/t0(0) o400->057a3a36-c20e-90a7-50a7-1eb817f2e2f2@10.31.136.153@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.475489:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693115370405760, offset 192 00000400:00000200:22.0:1616513086.475495:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.153@tcp1 00000400:00000200:22.0:1616513086.475502:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.153@tcp1 routed destination 00000400:00000200:22.0:1616513086.475517:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.153@tcp1(10.31.136.153@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.475523:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.475529:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.475533:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.475535:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.475540:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.475545:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.475599:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.475606:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.475609:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513086.475616:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.475620:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a7f8 00000800:00000200:14.0:1616513086.475623:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.475626:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.490055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.490113:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.490123:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.490132:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.213@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.490141:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.213@tcp1 of length 224 into portal 12 MB=0x60191c8366840 00000400:00000200:18.0:1616513086.490148:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.213@tcp1 of length 224/224 into md 0xf964d519 [1] + 53520 00000400:00000200:18.0:1616513086.490153:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.490155:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.490162:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.490165:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.490167:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.490170:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.490181:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513086.490187:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513086.490189:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.213@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.490199:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.213@tcp1 of length 224 into portal 26 MB=0x60191c83663c0 00000400:00000200:26.0:1616513086.490205:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.213@tcp1 of length 224/224 into md 0xf97cb999 [1] + 0 00000400:00000200:26.0:1616513086.490210:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3188 00000400:00000200:26.0:1616513086.490213:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.490216:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513086.490220:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3188 00000800:00000200:26.0:1616513086.490224:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.490227:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513086.490227:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690575501158464 00000800:00000200:26.0:1616513086.490228:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:22.0:1616513086.490239:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bd100 x1690575501158464/t0(0) o400->8eb546fe-7f4e-73fc-160d-f620ecd11f44@10.31.128.213@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.490253:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690575501158464, offset 192 00000400:00000200:22.0:1616513086.490260:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.213@tcp1 00000400:00000200:22.0:1616513086.490268:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.213@tcp1 routed destination 00000400:00000200:22.0:1616513086.490283:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.213@tcp1(10.31.128.213@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.490289:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.490295:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513086.490296:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690575501157312 00000800:00000200:22.0:1616513086.490298:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.490300:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.490304:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:0.0:1616513086.490307:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1690575501157312/t0(0) o400->332b0ae7-19a1-96a4-2729-d97a44af9e43@10.31.128.213@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513086.490313:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.490322:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690575501157312, offset 192 00000400:00000200:0.0:1616513086.490328:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.213@tcp1 00000400:00000200:0.0:1616513086.490336:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.213@tcp1 routed destination 00000400:00000200:0.0:1616513086.490353:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.213@tcp1(10.31.128.213@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.490359:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513086.490365:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.490368:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.490370:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513086.490371:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513086.490375:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513086.490378:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513086.490380:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513086.490381:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1616513086.490382:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513086.490387:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.490391:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b8f8 00000800:00000200:12.0:1616513086.490395:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.490398:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513086.490450:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.490458:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.490461:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513086.490462:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.490467:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.490480:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:2.0:1616513086.490483:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.530271:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.530329:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.530341:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.530350:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.119@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.530361:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.119@tcp1 of length 224 into portal 26 MB=0x5ffd5003926c0 00000400:00000200:28.0:1616513086.530368:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.119@tcp1 of length 224/224 into md 0xf97cb9a9 [1] + 0 00000400:00000200:28.0:1616513086.530373:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2990 00000400:00000200:28.0:1616513086.530376:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.530379:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513086.530385:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2990 00000800:00000200:24.0:1616513086.530385:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.530389:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.530391:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.530393:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.530465:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665180415680 00010000:00000200:0.0:1616513086.530477:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688665180415680/t0(0) o400->9e7e1d61-9c4a-0e68-ff3c-e8a97bd14af0@10.31.136.119@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.530491:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665180415680, offset 192 00000400:00000200:0.0:1616513086.530497:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.119@tcp1 00000400:00000200:0.0:1616513086.530506:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.119@tcp1 routed destination 00000400:00000200:0.0:1616513086.530520:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.119@tcp1(10.31.136.119@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.530527:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.530533:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.530536:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.530538:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.530543:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.530548:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.530550:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513086.530565:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513086.530602:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.530609:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.530612:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.530619:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.530625:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb046e8 00000800:00000200:14.0:1616513086.530626:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513086.530628:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.530631:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513086.530636:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513086.530645:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.119@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.530654:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.119@tcp1 of length 224 into portal 12 MB=0x5ffd500392840 00000400:00000200:14.0:1616513086.530661:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.119@tcp1 of length 224/224 into md 0xf964d519 [1] + 53744 00000400:00000200:14.0:1616513086.530671:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.530674:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513086.530681:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.530683:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.530685:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.530686:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.530747:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665180416064 00010000:00000200:22.0:1616513086.530759:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7b180 x1688665180416064/t0(0) o400->f0150b93-7b1b-f910-194d-0ae76ff085f7@10.31.136.119@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.530773:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665180416064, offset 192 00000400:00000200:22.0:1616513086.530779:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.119@tcp1 00000400:00000200:22.0:1616513086.530786:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.119@tcp1 routed destination 00000400:00000200:22.0:1616513086.530799:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.119@tcp1(10.31.136.119@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.530805:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.530812:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.530814:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.530816:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.530821:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.530827:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.530884:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.530891:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.530894:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.530900:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.530907:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a770 00000800:00000200:18.0:1616513086.530911:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.530914:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.541182:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513086.541238:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.541247:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.541256:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.111@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.541266:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.111@tcp1 of length 224 into portal 26 MB=0x5ffdca9aa4380 00000400:00000200:24.0:1616513086.541273:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.111@tcp1 of length 224/224 into md 0xf97cb9b9 [1] + 0 00000400:00000200:24.0:1616513086.541277:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3430 00000400:00000200:24.0:1616513086.541280:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.541283:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513086.541287:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3430 00000800:00000200:24.0:1616513086.541290:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.541291:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.541293:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513086.541295:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.541361:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698087949184 00010000:00000200:0.0:1616513086.541373:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688698087949184/t0(0) o400->c718df13-8083-3884-fff4-179439612a1c@10.31.132.111@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.541387:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698087949184, offset 192 00000400:00000200:0.0:1616513086.541393:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.111@tcp1 00000400:00000200:0.0:1616513086.541402:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.111@tcp1 routed destination 00000400:00000200:0.0:1616513086.541417:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.111@tcp1(10.31.132.111@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.541424:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.541430:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.541434:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.541451:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.541456:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.541458:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.541464:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513086.541510:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.541519:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.541528:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.111@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.541537:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.111@tcp1 of length 224 into portal 12 MB=0x5ffdca9aa4680 00000400:00000200:12.0:1616513086.541544:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.111@tcp1 of length 224/224 into md 0x865bbec1 [1] + 84224 00000400:00000200:12.0:1616513086.541554:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.541557:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513086.541563:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513086.541564:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513086.541567:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513086.541571:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513086.541572:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513086.541574:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.541575:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.541581:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.541587:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000100:00000200:4.0:1616513086.541589:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698087949952 00000800:00000200:20.0:1616513086.541590:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:4.0:1616513086.541602:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d205a00 x1688698087949952/t0(0) o400->280d44b2-1dcb-3719-6bdb-3458a9a2818e@10.31.132.111@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.541617:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698087949952, offset 192 00000400:00000200:4.0:1616513086.541623:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.111@tcp1 00000400:00000200:4.0:1616513086.541631:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.111@tcp1 routed destination 00000400:00000200:4.0:1616513086.541647:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.111@tcp1(10.31.132.111@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.541655:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.541661:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.541664:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.541666:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.541671:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.541678:0:50154:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.541734:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.541741:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.541743:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.541750:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.541754:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba220 00000800:00000200:26.0:1616513086.541758:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.541761:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.583649:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.583706:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.583716:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.583726:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.214@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.583737:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.214@tcp1 of length 224 into portal 26 MB=0x5ffdb20918f40 00000400:00000200:18.0:1616513086.583743:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.214@tcp1 of length 224/224 into md 0xf97cb9e9 [1] + 0 00000400:00000200:18.0:1616513086.583748:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b37e8 00000400:00000200:18.0:1616513086.583752:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.583755:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513086.583759:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b37e8 00000800:00000200:2.0:1616513086.583761:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.583763:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.583766:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513086.583769:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.583772:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.583773:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.583782:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.214@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.583791:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.214@tcp1 of length 224 into portal 12 MB=0x5ffdb20918fc0 00000400:00000200:2.0:1616513086.583798:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.214@tcp1 of length 224/224 into md 0xf964d519 [1] + 53968 00000400:00000200:2.0:1616513086.583803:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.583806:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513086.583814:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.583817:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.583819:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.583823:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691492884288 00010000:00000200:0.0:1616513086.583834:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688691492884288/t0(0) o400->9a1aa252-e26e-f401-5898-126d6f2f20ad@10.31.140.214@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.583846:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688691492884288, offset 192 00000400:00000200:0.0:1616513086.583852:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.214@tcp1 00000400:00000200:0.0:1616513086.583859:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.214@tcp1 routed destination 00000400:00000200:0.0:1616513086.583872:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.214@tcp1(10.31.140.214@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.583878:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:22.0:1616513086.583880:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691492884416 00000800:00000200:0.0:1616513086.583883:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.583885:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.583887:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.583892:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513086.583894:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebd100 x1688691492884416/t0(0) o400->76f1d254-7770-6949-e804-e0e385b6b6b1@10.31.140.214@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513086.583898:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513086.583899:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513086.583919:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688691492884416, offset 192 00000400:00000200:22.0:1616513086.583926:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.214@tcp1 00000400:00000200:22.0:1616513086.583933:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.214@tcp1 routed destination 00000400:00000200:22.0:1616513086.583948:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.214@tcp1(10.31.140.214@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.583954:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513086.583958:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513086.583961:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.583963:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513086.583965:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513086.583965:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513086.583968:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513086.583971:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513086.583975:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513086.583975:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513086.583980:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:24.0:1616513086.583983:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.583986:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513086.584040:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.584048:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.584050:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513086.584051:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.584057:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.584061:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b430 00000800:00000200:14.0:1616513086.584065:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513086.586529:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927200, portal 4 00000100:00000200:6.0:1616513086.586533:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927200, offset 0 00000400:00000200:6.0:1616513086.586538:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:6.0:1616513086.586543:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:6.0:1616513086.586553:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513086.586557:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:6.0:1616513086.586562:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513086.586565:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:6.0:1616513086.586566:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:6.0:1616513086.586572:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513086.586619:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513086.586639:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513086.586646:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513086.586649:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513086.586655:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513086.586662:0:6476:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc4ebe780 x1693067725927200/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513086.586675:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00b28 00000800:00000200:27.0:1616513086.586679:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513086.586682:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513086.586790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513086.586799:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513086.586806:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513086.586810:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513086.586814:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513086.586822:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627320 00000400:00000200:27.0:1616513086.586829:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166e8d [1] + 192 00000400:00000200:27.0:1616513086.586835:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513086.586839:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc4ebe780 x1693067725927200/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513086.586853:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513086.586856:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513086.586858:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:6.0:1616513086.586860:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc4ebe780 x1693067725927200/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:6.0:1616513086.586869:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc4ebe780 x1693067725927200/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513093 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513086.586877:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:11.2:1616513086.596060:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.596115:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.596124:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.596134:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.119@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.596144:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.119@tcp1 of length 224 into portal 26 MB=0x6037de5c697c0 00000400:00000200:28.0:1616513086.596151:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.119@tcp1 of length 224/224 into md 0xf97cb9f9 [1] + 0 00000400:00000200:28.0:1616513086.596155:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3100 00000400:00000200:28.0:1616513086.596158:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.596161:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513086.596166:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3100 00000800:00000200:20.0:1616513086.596168:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.596169:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.596172:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513086.596175:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.596178:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.596179:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513086.596187:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.119@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.596196:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.119@tcp1 of length 224 into portal 12 MB=0x6037de5c69840 00000400:00000200:20.0:1616513086.596203:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.119@tcp1 of length 224/224 into md 0x865bbec1 [1] + 84448 00000400:00000200:20.0:1616513086.596208:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.596211:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513086.596218:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.596221:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.596223:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.596231:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692689121056704 00010000:00000200:0.0:1616513086.596242:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1692689121056704/t0(0) o400->1598ff2a-0461-c279-5e16-63a44283c88f@10.31.141.119@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.596256:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692689121056704, offset 192 00000400:00000200:0.0:1616513086.596263:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.119@tcp1 00000400:00000200:0.0:1616513086.596271:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.119@tcp1 routed destination 00000100:00000200:4.0:1616513086.596283:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692689121056832 00000400:00000200:0.0:1616513086.596286:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.119@tcp1(10.31.141.119@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.596292:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:4.0:1616513086.596296:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38ba80 x1692689121056832/t0(0) o400->8bbdf172-eea1-92ef-40cb-06658a91fc79@10.31.141.119@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513086.596298:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.596301:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.596303:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.596308:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:4.0:1616513086.596310:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692689121056832, offset 192 00000800:00000200:11.2:1616513086.596313:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.596315:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:4.0:1616513086.596317:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.119@tcp1 00000400:00000200:4.0:1616513086.596324:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.119@tcp1 routed destination 00000400:00000200:4.0:1616513086.596339:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.119@tcp1(10.31.141.119@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.596346:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.596352:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.596355:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.596357:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.596361:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513086.596365:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513086.596368:0:50160:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513086.596373:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.596375:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513086.596377:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513086.596382:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.596387:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05870 00000800:00000200:12.0:1616513086.596391:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513086.596424:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.596431:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.596446:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513086.596453:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.596462:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bab28 00000800:00000200:2.0:1616513086.596466:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.596469:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.597877:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513086.597932:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.597941:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513086.597950:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.204@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.597959:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.204@tcp1 of length 224 into portal 26 MB=0x5ffd3cf86abc0 00000400:00000200:20.0:1616513086.597966:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.204@tcp1 of length 224/224 into md 0xf97cba29 [1] + 0 00000400:00000200:20.0:1616513086.597970:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2220 00000400:00000200:20.0:1616513086.597973:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.597976:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513086.597980:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2220 00000800:00000200:20.0:1616513086.597983:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.597986:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513086.597987:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.597990:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.598042:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660068445120 00010000:00000200:0.0:1616513086.598052:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688660068445120/t0(0) o400->b23cba96-925a-81f1-45c0-f6eac2ec449f@10.31.128.204@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.598066:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660068445120, offset 192 00000400:00000200:0.0:1616513086.598072:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.204@tcp1 00000400:00000200:0.0:1616513086.598079:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.204@tcp1 routed destination 00000400:00000200:0.0:1616513086.598093:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.204@tcp1(10.31.128.204@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.598099:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.598104:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.598107:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.598108:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.598113:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.598117:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.598119:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513086.598172:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.598178:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.598181:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513086.598188:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.598193:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05870 00000800:00000200:28.0:1616513086.598197:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.598199:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.598207:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513086.598210:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513086.598215:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.204@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.598223:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.204@tcp1 of length 224 into portal 12 MB=0x5ffd3cf86af80 00000400:00000200:28.0:1616513086.598230:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.204@tcp1 of length 224/224 into md 0x865bbec1 [1] + 84672 00000400:00000200:28.0:1616513086.598235:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.598237:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513086.598243:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.598245:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.598247:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.598307:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660068446080 00010000:00000200:4.0:1616513086.598320:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0e8000 x1688660068446080/t0(0) o400->511b81bf-abc2-4daf-e936-2b8a013be7ed@10.31.128.204@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.598335:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660068446080, offset 192 00000400:00000200:4.0:1616513086.598350:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.204@tcp1 00000400:00000200:4.0:1616513086.598358:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.204@tcp1 routed destination 00000400:00000200:4.0:1616513086.598373:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.204@tcp1(10.31.128.204@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.598379:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.598385:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.598388:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.598390:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.598395:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.598401:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.598418:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.598424:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.598427:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.598446:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.598450:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba088 00000800:00000200:18.0:1616513086.598454:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.598457:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.599011:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.599067:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.599076:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.599085:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.171@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.599094:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.171@tcp1 of length 224 into portal 26 MB=0x5ffd4cee59340 00000400:00000200:2.0:1616513086.599101:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.171@tcp1 of length 224/224 into md 0xf97cba39 [1] + 0 00000400:00000200:2.0:1616513086.599106:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2198 00000400:00000200:2.0:1616513086.599108:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.599111:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513086.599116:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2198 00000800:00000200:2.0:1616513086.599119:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.599121:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.599122:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.599125:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.599128:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664352854848 00010000:00000200:0.0:1616513086.599133:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688664352854848/t0(0) o400->07148be8-4bb0-3144-a74e-d1e527d80aec@10.31.136.171@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.599144:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664352854848, offset 192 00000400:00000200:0.0:1616513086.599148:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.171@tcp1 00000400:00000200:0.0:1616513086.599154:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.171@tcp1 routed destination 00000400:00000200:0.0:1616513086.599166:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.171@tcp1(10.31.136.171@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.599171:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.599176:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.599178:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.599179:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.599182:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513086.599186:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513086.599188:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.599242:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.599249:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513086.599252:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513086.599253:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513086.599261:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513086.599266:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:24.0:1616513086.599270:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.599883:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.599937:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.599946:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.599955:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.171@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.599963:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.171@tcp1 of length 224 into portal 12 MB=0x5ffd4cee59500 00000400:00000200:12.0:1616513086.599969:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.171@tcp1 of length 224/224 into md 0x865bbec1 [1] + 84896 00000400:00000200:12.0:1616513086.599975:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.599977:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513086.599983:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.599986:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.599988:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.599990:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.600047:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664352855296 00010000:00000200:4.0:1616513086.600060:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d201680 x1688664352855296/t0(0) o400->44befa44-2536-5d3e-45ca-aa0876c8cbdf@10.31.136.171@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.600074:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664352855296, offset 192 00000400:00000200:4.0:1616513086.600080:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.171@tcp1 00000400:00000200:4.0:1616513086.600088:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.171@tcp1 routed destination 00000400:00000200:4.0:1616513086.600103:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.171@tcp1(10.31.136.171@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.600110:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.600116:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.600119:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.600121:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.600125:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.600130:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.600182:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.600189:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.600192:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.600198:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.600203:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baf68 00000800:00000200:18.0:1616513086.600207:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.600210:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.634399:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513086.634465:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.634475:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513086.634484:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.66@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.634496:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.66@tcp1 of length 224 into portal 26 MB=0x604107ea6cb00 00000400:00000200:20.0:1616513086.634503:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.66@tcp1 of length 224/224 into md 0xf97cba69 [1] + 0 00000400:00000200:20.0:1616513086.634508:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3a08 00000400:00000200:20.0:1616513086.634511:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.634514:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513086.634519:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513086.634520:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3a08 00000800:00000200:20.0:1616513086.634523:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.634527:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.634528:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.634586:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693318751111936 00010000:00000200:0.0:1616513086.634598:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1693318751111936/t0(0) o400->271abf9e-b8e8-e5bb-e6c1-166a131282b0@10.31.128.66@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.634612:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693318751111936, offset 192 00000400:00000200:0.0:1616513086.634625:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.66@tcp1 00000400:00000200:0.0:1616513086.634633:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.66@tcp1 routed destination 00000400:00000200:0.0:1616513086.634648:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.66@tcp1(10.31.128.66@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.634654:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.634659:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.634662:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.634664:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.634669:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.634674:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.634676:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513086.634728:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.634736:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.634739:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.634745:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.634751:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:12.0:1616513086.634755:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.634757:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.634768:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513086.634769:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513086.634777:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.66@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.634786:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.66@tcp1 of length 224 into portal 12 MB=0x604107ea6cf40 00000400:00000200:28.0:1616513086.634792:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.66@tcp1 of length 224/224 into md 0x865bbec1 [1] + 85120 00000400:00000200:28.0:1616513086.634797:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.634800:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513086.634808:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.634811:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.634813:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.634872:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693318751113024 00010000:00000200:4.0:1616513086.634885:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa4800 x1693318751113024/t0(0) o400->84bbf565-2eed-f4b7-9c80-4e7e7ebaea0e@10.31.128.66@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.634900:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693318751113024, offset 192 00000400:00000200:4.0:1616513086.634906:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.66@tcp1 00000400:00000200:4.0:1616513086.634914:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.66@tcp1 routed destination 00000400:00000200:4.0:1616513086.634929:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.66@tcp1(10.31.128.66@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.634936:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.634942:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.634945:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.634947:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.634952:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.634961:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.635023:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.635030:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.635033:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.635039:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.635043:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbba0 00000800:00000200:26.0:1616513086.635047:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.635050:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.641452:0:50165:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.641508:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513086.641509:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513086.641517:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.641526:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.155@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.641538:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.155@tcp1 of length 224 into portal 26 MB=0x5ffd632cf66c0 00000400:00000200:18.0:1616513086.641544:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.155@tcp1 of length 224/224 into md 0xf97cba89 [1] + 0 00000400:00000200:18.0:1616513086.641549:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3210 00000400:00000200:18.0:1616513086.641552:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.641555:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513086.641559:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3210 00000800:00000200:18.0:1616513086.641562:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.641563:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513086.641565:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513086.641565:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.641567:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513086.641574:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513086.641583:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.155@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.641592:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.155@tcp1 of length 224 into portal 12 MB=0x5ffd632cf6a40 00000400:00000200:20.0:1616513086.641598:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.155@tcp1 of length 224/224 into md 0x865bbec1 [1] + 85344 00000400:00000200:20.0:1616513086.641603:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.641606:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513086.641612:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.641615:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513086.641617:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.641620:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.641624:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688670324090560 00010000:00000200:0.0:1616513086.641635:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688670324090560/t0(0) o400->0eb35485-2c6d-6149-7997-29c874b53d54@10.31.129.155@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513086.641647:0:50165:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513086.641649:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688670324090560, offset 192 00000400:00000200:0.0:1616513086.641656:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.155@tcp1 00000400:00000200:0.0:1616513086.641663:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.155@tcp1 routed destination 00000100:00000200:4.0:1616513086.641676:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688670324091456 00000400:00000200:0.0:1616513086.641678:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.155@tcp1(10.31.129.155@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.641685:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:4.0:1616513086.641689:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38da00 x1688670324091456/t0(0) o400->3e158ece-9444-26b9-1653-b0bff1ec945d@10.31.129.155@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513086.641691:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.641694:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.641696:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.641701:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513086.641703:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513086.641704:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688670324091456, offset 192 00000800:00000200:11.2:1616513086.641706:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.641708:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:4.0:1616513086.641710:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.155@tcp1 00000800:00000200:26.0:1616513086.641713:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:4.0:1616513086.641718:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.155@tcp1 routed destination 00000400:00000200:26.0:1616513086.641723:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.110@tcp1 : PUT - for me 00000400:00000200:4.0:1616513086.641734:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.155@tcp1(10.31.129.155@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.641741:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.641744:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.110@tcp1 of length 224 into portal 26 MB=0x5ffdedac44cc0 00000800:00000200:4.0:1616513086.641747:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.641750:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513086.641751:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.110@tcp1 of length 224/224 into md 0xf97cbaa9 [1] + 0 00000800:00000200:4.0:1616513086.641752:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:26.0:1616513086.641756:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3ed0 00000800:00000200:4.0:1616513086.641757:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513086.641758:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513086.641758:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:24.0:1616513086.641760:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:26.0:1616513086.641761:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513086.641766:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3ed0 00000800:00000200:24.0:1616513086.641767:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513086.641769:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000400:00000200:24.0:1616513086.641769:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513086.641769:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513086.641770:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513086.641772:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (26)-- 00000400:00000200:24.0:1616513086.641776:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.641780:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.110@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.641782:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:26.0:1616513086.641783:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (25)-- 00000800:00000200:14.0:1616513086.641784:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:24.0:1616513086.641785:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513086.641786:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513086.641786:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1616513086.641786:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707501673664 00000800:00000200:24.0:1616513086.641788:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513086.641790:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.110@tcp1 of length 224 into portal 12 MB=0x5ffdedac44d40 00010000:00000200:0.0:1616513086.641792:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688707501673664/t0(0) o400->584e7240-efa6-0044-244d-9ef2a25d2113@10.31.142.110@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513086.641793:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.641796:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.110@tcp1 of length 224/224 into md 0xf964d519 [1] + 54192 00000400:00000200:14.0:1616513086.641797:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba000 00000800:00000200:14.0:1616513086.641801:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.641802:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513086.641803:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688707501673664, offset 192 00000100:00000200:2.0:1616513086.641804:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513086.641809:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.110@tcp1 00000800:00000200:2.0:1616513086.641812:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513086.641814:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.110@tcp1 routed destination 00000800:00000200:2.0:1616513086.641815:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.641817:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513086.641827:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.110@tcp1(10.31.142.110@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.641837:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.641842:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.641844:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.641846:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.641849:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.641853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.641853:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.641863:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.641867:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.641869:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513086.641872:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513086.641875:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:22.0:1616513086.641879:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707501673792 00000400:00000200:20.0:1616513086.641879:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05cb0 00000800:00000200:20.0:1616513086.641882:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:22.0:1616513086.641893:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebc380 x1688707501673792/t0(0) o400->d2dacd6c-7124-6025-193e-b4f0f36f51c7@10.31.142.110@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.641907:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688707501673792, offset 192 00000400:00000200:22.0:1616513086.641913:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.110@tcp1 00000400:00000200:22.0:1616513086.641921:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.110@tcp1 routed destination 00000400:00000200:22.0:1616513086.641935:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.110@tcp1(10.31.142.110@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.641942:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.641948:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.641951:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.641953:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.641957:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.641961:0:50165:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.641972:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.641976:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.641978:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513086.641980:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513086.641983:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.641987:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bdc0 00000800:00000200:26.0:1616513086.641989:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.644966:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513086.645020:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.645029:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.645038:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.70@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.645048:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.70@tcp1 of length 224 into portal 26 MB=0x5ffe062fb1a40 00000400:00000200:24.0:1616513086.645055:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.70@tcp1 of length 224/224 into md 0xf97cbac9 [1] + 0 00000400:00000200:24.0:1616513086.645060:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b24c8 00000400:00000200:24.0:1616513086.645063:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.645065:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513086.645069:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b24c8 00000800:00000200:24.0:1616513086.645073:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.645075:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513086.645077:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.645080:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.645136:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714081933888 00010000:00000200:0.0:1616513086.645147:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688714081933888/t0(0) o400->7f7b3602-2909-23b1-1d26-ee3f91ef2ca1@10.31.129.70@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.645161:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688714081933888, offset 192 00000400:00000200:0.0:1616513086.645167:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.70@tcp1 00000400:00000200:0.0:1616513086.645175:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.70@tcp1 routed destination 00000400:00000200:0.0:1616513086.645190:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.70@tcp1(10.31.129.70@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.645196:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.645202:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.645205:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.645207:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.645212:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.645217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.645218:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513086.645269:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.645276:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.645278:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.645285:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.645292:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000800:00000200:12.0:1616513086.645296:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.645299:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.645383:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513086.645393:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.645398:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513086.645402:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513086.645406:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.70@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.645412:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.70@tcp1 of length 224 into portal 12 MB=0x5ffe062fb1f00 00000400:00000200:24.0:1616513086.645417:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.70@tcp1 of length 224/224 into md 0x865bbec1 [1] + 85568 00000400:00000200:24.0:1616513086.645421:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.645423:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513086.645428:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.645432:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.645445:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.645492:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714081935104 00010000:00000200:4.0:1616513086.645504:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e5580 x1688714081935104/t0(0) o400->da806a1f-1376-002e-fbbb-fa1de73e1442@10.31.129.70@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.645519:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688714081935104, offset 192 00000400:00000200:4.0:1616513086.645525:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.70@tcp1 00000400:00000200:4.0:1616513086.645533:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.70@tcp1 routed destination 00000400:00000200:4.0:1616513086.645548:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.70@tcp1(10.31.129.70@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.645554:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.645560:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.645563:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.645565:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.645570:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.645577:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.645634:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.645641:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.645644:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513086.645650:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.645655:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6badd0 00000800:00000200:14.0:1616513086.645659:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.645662:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.654147:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.654203:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.654213:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.654222:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.28@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.654231:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.28@tcp1 of length 224 into portal 26 MB=0x5ffd36ccdb500 00000400:00000200:18.0:1616513086.654238:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.28@tcp1 of length 224/224 into md 0xf97cbae9 [1] + 0 00000400:00000200:18.0:1616513086.654242:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2000 00000400:00000200:18.0:1616513086.654245:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.654247:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513086.654251:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2000 00000800:00000200:18.0:1616513086.654255:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.654257:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.654259:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.654262:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.654266:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.654273:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.28@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.654279:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.28@tcp1 of length 224 into portal 12 MB=0x5ffd36ccdb5c0 00000400:00000200:18.0:1616513086.654284:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.28@tcp1 of length 224/224 into md 0xf964d519 [1] + 54416 00000400:00000200:18.0:1616513086.654288:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.654290:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.654295:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.654297:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.654299:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.654304:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.654314:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658412156160 00010000:00000200:0.0:1616513086.654324:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688658412156160/t0(0) o400->e912b0be-1e3e-a8ce-e37e-8fa21add00c0@10.31.133.28@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.654337:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658412156160, offset 192 00000400:00000200:0.0:1616513086.654343:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.28@tcp1 00000400:00000200:0.0:1616513086.654350:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.28@tcp1 routed destination 00000400:00000200:0.0:1616513086.654364:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.28@tcp1(10.31.133.28@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.654370:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.654375:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.654378:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513086.654381:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658412156352 00000800:00000200:0.0:1616513086.654381:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.654386:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513086.654392:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:22.0:1616513086.654393:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bcc80 x1688658412156352/t0(0) o400->35d8a688-c958-e811-e4d6-52fee32f6924@10.31.133.28@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513086.654393:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513086.654408:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658412156352, offset 192 00000400:00000200:22.0:1616513086.654414:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.28@tcp1 00000400:00000200:22.0:1616513086.654422:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.28@tcp1 routed destination 00000400:00000200:22.0:1616513086.654451:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.28@tcp1(10.31.133.28@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513086.654451:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.654458:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513086.654460:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513086.654460:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513086.654466:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513086.654466:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513086.654469:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.654471:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513086.654471:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a08 00000800:00000200:12.0:1616513086.654475:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513086.654476:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513086.654477:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.654481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.654537:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.654544:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.654547:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513086.654550:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.654553:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.654557:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4af68 00000800:00000200:2.0:1616513086.654560:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.664725:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.664781:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.664791:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.664800:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.13@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.664809:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.13@tcp1 of length 224 into portal 26 MB=0x604a4d08bfd00 00000400:00000200:28.0:1616513086.664815:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.13@tcp1 of length 224/224 into md 0xf97cbaf9 [1] + 0 00000400:00000200:28.0:1616513086.664820:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b38f8 00000400:00000200:28.0:1616513086.664823:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.664825:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513086.664830:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b38f8 00000800:00000200:28.0:1616513086.664833:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.664835:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.664836:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.664842:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513086.664845:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513086.664846:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513086.664854:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.13@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.664863:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.13@tcp1 of length 224 into portal 12 MB=0x604a4d08bfd80 00000400:00000200:24.0:1616513086.664869:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.13@tcp1 of length 224/224 into md 0x865bbec1 [1] + 85792 00000400:00000200:24.0:1616513086.664874:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.664877:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513086.664883:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.664886:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.664888:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.664894:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693955780246784 00010000:00000200:0.0:1616513086.664905:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1693955780246784/t0(0) o400->3beef1be-d51b-db55-0658-0c8df5c6e8f4@10.31.141.13@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.664920:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693955780246784, offset 192 00000400:00000200:0.0:1616513086.664926:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.13@tcp1 00000400:00000200:0.0:1616513086.664933:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.13@tcp1 routed destination 00000100:00000200:4.0:1616513086.664947:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693955780246912 00000400:00000200:0.0:1616513086.664948:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.13@tcp1(10.31.141.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.664955:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:4.0:1616513086.664960:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e6300 x1693955780246912/t0(0) o400->883003b2-7fbe-8c3e-c668-674363a3cef6@10.31.141.13@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513086.664961:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.664963:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.664965:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.664970:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.664974:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513086.664974:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693955780246912, offset 192 00000100:00000200:0.0:1616513086.664977:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:4.0:1616513086.664981:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.13@tcp1 00000400:00000200:4.0:1616513086.664988:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.13@tcp1 routed destination 00000400:00000200:4.0:1616513086.665003:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.13@tcp1(10.31.141.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.665010:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.665015:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.665018:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.665022:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.665027:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513086.665029:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513086.665034:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513086.665036:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.665039:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513086.665041:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513086.665046:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.665051:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb042a8 00000800:00000200:20.0:1616513086.665055:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513086.665101:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.665108:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.665110:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.665116:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.665120:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000800:00000200:18.0:1616513086.665124:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.665126:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.695708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513086.695765:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.695775:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.695785:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.180@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.695794:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.180@tcp1 of length 224 into portal 12 MB=0x5ffde3ec03e40 00000400:00000200:24.0:1616513086.695801:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.180@tcp1 of length 224/224 into md 0x865bbec1 [1] + 86016 00000400:00000200:24.0:1616513086.695806:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.695809:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513086.695816:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.695819:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.695819:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513086.695827:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513086.695830:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.695831:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.695840:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.180@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.695849:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.180@tcp1 of length 224 into portal 26 MB=0x5ffde3ec03dc0 00000400:00000200:12.0:1616513086.695855:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.180@tcp1 of length 224/224 into md 0xf97cbb19 [1] + 0 00000400:00000200:12.0:1616513086.695860:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b3cb0 00000400:00000200:12.0:1616513086.695863:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.695866:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.695871:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b3cb0 00000800:00000200:12.0:1616513086.695874:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.695877:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.695879:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.695882:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704884162112 00010000:00000200:4.0:1616513086.695895:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e6c00 x1688704884162112/t0(0) o400->348509cf-bf65-dcd6-cda8-d6afbe7693b7@10.31.132.180@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.695910:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704884162112, offset 192 00000400:00000200:4.0:1616513086.695916:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.180@tcp1 00000400:00000200:4.0:1616513086.695924:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.180@tcp1 routed destination 00000100:00000200:0.0:1616513086.695936:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704884161984 00000400:00000200:4.0:1616513086.695940:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.180@tcp1(10.31.132.180@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.695947:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:0.0:1616513086.695947:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688704884161984/t0(0) o400->5a7b2e10-10de-9436-7498-14fb2d019a9d@10.31.132.180@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:4.0:1616513086.695953:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.695956:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513086.695958:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.695961:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704884161984, offset 192 00000800:00000200:4.0:1616513086.695963:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.695968:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513086.695968:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.180@tcp1 00000400:00000200:0.0:1616513086.695975:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.180@tcp1 routed destination 00000800:00000200:20.0:1616513086.695987:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513086.695989:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.180@tcp1(10.31.132.180@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.695995:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513086.695996:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.695999:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513086.696000:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513086.696001:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.696004:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.696005:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513086.696006:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.696010:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba3b8 00000800:00000200:0.0:1616513086.696011:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513086.696014:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.696017:0:50171:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513086.696017:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513086.696075:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.696082:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.696085:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.696092:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.696098:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04198 00000800:00000200:26.0:1616513086.696101:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.696104:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.707664:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.707719:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.707728:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.707738:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.237@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.707747:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.237@tcp1 of length 224 into portal 26 MB=0x5ffd559fa8400 00000400:00000200:28.0:1616513086.707753:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.237@tcp1 of length 224/224 into md 0xf97cbb29 [1] + 0 00000400:00000200:28.0:1616513086.707758:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2dd0 00000400:00000200:28.0:1616513086.707761:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.707764:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513086.707768:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2dd0 00000800:00000200:28.0:1616513086.707771:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.707773:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.707775:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.707778:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.707832:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666686260224 00010000:00000200:0.0:1616513086.707844:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688666686260224/t0(0) o400->199a1d65-ce88-58fb-b955-0e744074c78c@10.31.131.237@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.707860:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666686260224, offset 192 00000800:00000200:7.2:1616513086.707863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513086.707866:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.237@tcp1 00000400:00000200:0.0:1616513086.707874:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.237@tcp1 routed destination 00000400:00000200:0.0:1616513086.707888:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.237@tcp1(10.31.131.237@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.707895:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.707901:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.707904:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.707906:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.707911:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.707916:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.707918:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513086.707926:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.707936:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513086.707943:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.707945:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.237@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.707954:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.237@tcp1 of length 224 into portal 12 MB=0x5ffd559fa8500 00000400:00000200:18.0:1616513086.707960:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.237@tcp1 of length 224/224 into md 0xf964d519 [1] + 54640 00000400:00000200:18.0:1616513086.707965:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.707968:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513086.707970:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513086.707975:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513086.707977:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513086.707978:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513086.707980:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513086.707980:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513086.707982:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513086.707986:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513086.707992:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:24.0:1616513086.707995:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513086.708041:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666686260480 00010000:00000200:22.0:1616513086.708054:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bf980 x1688666686260480/t0(0) o400->8f23dffa-e508-a684-4cc7-13470d6d11de@10.31.131.237@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.708068:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666686260480, offset 192 00000400:00000200:22.0:1616513086.708075:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.237@tcp1 00000400:00000200:22.0:1616513086.708082:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.237@tcp1 routed destination 00000400:00000200:22.0:1616513086.708097:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.237@tcp1(10.31.131.237@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.708104:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.708110:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.708113:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.708115:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.708120:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.708125:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.708179:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.708186:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.708189:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.708195:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.708199:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:26.0:1616513086.708203:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.708206:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.712708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.712762:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.712771:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.712780:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.254@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.712790:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.254@tcp1 of length 224 into portal 26 MB=0x60405a8adbf00 00000400:00000200:28.0:1616513086.712797:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.254@tcp1 of length 224/224 into md 0xf97cbb39 [1] + 0 00000400:00000200:28.0:1616513086.712802:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82043b2330 00000400:00000200:28.0:1616513086.712805:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.712807:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513086.712812:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82043b2330 00000800:00000200:28.0:1616513086.712815:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.712815:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.712817:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.712819:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.712872:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693272211570432 00010000:00000200:0.0:1616513086.712882:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1693272211570432/t0(0) o400->c3434282-a262-2550-2351-f40a0d9be1d1@10.31.132.254@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.712896:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693272211570432, offset 192 00000400:00000200:0.0:1616513086.712901:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.254@tcp1 00000400:00000200:0.0:1616513086.712909:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.254@tcp1 routed destination 00000400:00000200:0.0:1616513086.712923:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.254@tcp1(10.31.132.254@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.712929:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.712934:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.712937:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.712939:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.712943:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.712949:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.712949:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.712961:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.712967:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.712970:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.712977:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.712982:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:20.0:1616513086.712986:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.712989:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.713413:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.713424:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.713432:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.713452:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.254@tcp1 : PUT - for me 00000800:00000200:28.0:1616513086.713454:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513086.713461:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.254@tcp1 of length 224 into portal 12 MB=0x60405a8adc000 00000400:00000200:12.0:1616513086.713468:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.254@tcp1 of length 224/224 into md 0x865bbec1 [1] + 86240 00000400:00000200:12.0:1616513086.713473:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.713475:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513086.713482:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.713485:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.713486:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.713544:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693272211570688 00010000:00000200:4.0:1616513086.713557:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d200480 x1693272211570688/t0(0) o400->a024fc4e-fddb-010d-c024-69a1b475dc85@10.31.132.254@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.713571:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693272211570688, offset 192 00000400:00000200:4.0:1616513086.713578:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.254@tcp1 00000400:00000200:4.0:1616513086.713585:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.254@tcp1 routed destination 00000400:00000200:4.0:1616513086.713601:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.254@tcp1(10.31.132.254@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.713607:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.713613:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.713616:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.713618:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.713623:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.713629:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.713683:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.713690:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.713693:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513086.713699:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.713703:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bae58 00000800:00000200:14.0:1616513086.713708:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.713710:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.713853:0:50173:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.713906:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.713915:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.713925:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.131@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.713934:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.131@tcp1 of length 224 into portal 26 MB=0x5ffe2fa6ccb40 00000400:00000200:26.0:1616513086.713940:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.131@tcp1 of length 224/224 into md 0xf97cbb59 [1] + 0 00000400:00000200:26.0:1616513086.713945:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788ee0 00000400:00000200:26.0:1616513086.713947:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.713950:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513086.713954:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788ee0 00000800:00000200:26.0:1616513086.713956:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.713959:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.713960:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513086.713964:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.713965:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688725212678976 00010000:00000200:0.0:1616513086.713971:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688725212678976/t0(0) o400->1d14da7a-a73c-448b-160e-4fdced06adce@10.31.141.131@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.713981:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688725212678976, offset 192 00000400:00000200:0.0:1616513086.713986:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.131@tcp1 00000400:00000200:0.0:1616513086.713991:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.131@tcp1 routed destination 00000400:00000200:0.0:1616513086.714005:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.131@tcp1(10.31.141.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.714010:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.714014:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.714017:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.714018:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.714021:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513086.714025:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513086.714026:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.714034:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.714038:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.714040:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513086.714044:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513086.714045:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.714048:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:20.0:1616513086.714051:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.714094:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.714103:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.714109:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513086.714111:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513086.714116:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.131@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.714123:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.131@tcp1 of length 224 into portal 12 MB=0x5ffe2fa6cce80 00000400:00000200:28.0:1616513086.714128:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.131@tcp1 of length 224/224 into md 0x865bbec1 [1] + 86464 00000400:00000200:28.0:1616513086.714134:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.714137:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513086.714142:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.714144:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.714146:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.714155:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688725212679808 00010000:00000200:4.0:1616513086.714161:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5100 x1688725212679808/t0(0) o400->eec9bd53-53ab-e405-d8ea-ad33b10578d0@10.31.141.131@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.714173:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688725212679808, offset 192 00000400:00000200:4.0:1616513086.714180:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.131@tcp1 00000400:00000200:4.0:1616513086.714185:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.131@tcp1 routed destination 00000400:00000200:4.0:1616513086.714199:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.131@tcp1(10.31.141.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.714204:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.714209:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.714211:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.714213:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.714216:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.714221:0:50173:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.714230:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.714234:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.714237:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513086.714240:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.714243:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.714245:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb4b8 00000800:00000200:14.0:1616513086.714248:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.722684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513086.722739:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.722748:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.722759:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.252@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.722768:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.252@tcp1 of length 224 into portal 26 MB=0x603ffc07def40 00000400:00000200:24.0:1616513086.722775:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.252@tcp1 of length 224/224 into md 0xf97cbb69 [1] + 0 00000400:00000200:24.0:1616513086.722780:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789ed0 00000400:00000200:24.0:1616513086.722782:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.722785:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513086.722789:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789ed0 00000800:00000200:24.0:1616513086.722792:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.722792:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.722795:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513086.722797:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.722851:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693246841286464 00010000:00000200:0.0:1616513086.722861:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1693246841286464/t0(0) o400->3ecc0835-b86d-c602-d900-5f7bb3b7d048@10.31.132.252@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.722875:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693246841286464, offset 192 00000400:00000200:0.0:1616513086.722880:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.252@tcp1 00000400:00000200:0.0:1616513086.722888:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.252@tcp1 routed destination 00000400:00000200:0.0:1616513086.722901:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.252@tcp1(10.31.132.252@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.722908:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.722913:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.722915:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.722917:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.722922:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.722926:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.722928:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513086.722978:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.722985:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.722988:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.722994:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.723000:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb057e8 00000800:00000200:12.0:1616513086.723004:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.723007:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.723061:0:50174:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.723116:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.723127:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.723136:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.252@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.723145:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.252@tcp1 of length 224 into portal 12 MB=0x603ffc07df040 00000400:00000200:2.0:1616513086.723152:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.252@tcp1 of length 224/224 into md 0xf964d519 [1] + 54864 00000400:00000200:2.0:1616513086.723157:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.723159:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513086.723166:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.723170:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513086.723171:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.723175:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.723232:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693246841286720 00010000:00000200:22.0:1616513086.723245:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebb600 x1693246841286720/t0(0) o400->b0bfb757-3300-7575-f9a7-af8430c9fa0b@10.31.132.252@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.723259:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693246841286720, offset 192 00000400:00000200:22.0:1616513086.723267:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.252@tcp1 00000400:00000200:22.0:1616513086.723274:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.252@tcp1 routed destination 00000400:00000200:22.0:1616513086.723289:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.252@tcp1(10.31.132.252@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.723295:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.723301:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.723304:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.723306:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.723311:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.723315:0:50174:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.723369:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.723376:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.723378:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.723385:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.723389:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:18.0:1616513086.723393:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.723396:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.725349:0:50174:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.725405:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.725414:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.725423:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.135@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.725432:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.135@tcp1 of length 224 into portal 26 MB=0x603ce18d26600 00000400:00000200:26.0:1616513086.725452:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.135@tcp1 of length 224/224 into md 0xf97cbb79 [1] + 0 00000400:00000200:26.0:1616513086.725457:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788d48 00000400:00000200:26.0:1616513086.725459:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513086.725459:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:26.0:1616513086.725462:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513086.725466:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788d48 00000800:00000200:26.0:1616513086.725469:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513086.725469:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513086.725472:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513086.725474:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513086.725476:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.725478:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.135@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.725487:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.135@tcp1 of length 224 into portal 12 MB=0x603ce18d26740 00000400:00000200:2.0:1616513086.725493:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.135@tcp1 of length 224/224 into md 0xf964d519 [1] + 55088 00000400:00000200:2.0:1616513086.725498:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.725501:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513086.725507:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.725510:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.725512:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.725530:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693033574852096 00010000:00000200:0.0:1616513086.725541:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1693033574852096/t0(0) o400->ed17e64b-60fb-a714-141b-92be7959954b@10.31.141.135@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.725555:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693033574852096, offset 192 00000400:00000200:0.0:1616513086.725561:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.135@tcp1 00000400:00000200:0.0:1616513086.725569:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.135@tcp1 routed destination 00000100:00000200:22.0:1616513086.725570:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693033574852416 00010000:00000200:22.0:1616513086.725583:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebf080 x1693033574852416/t0(0) o400->fc88f002-f291-b25e-95f6-85111768aa8e@10.31.141.135@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513086.725583:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.135@tcp1(10.31.141.135@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.725590:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.725596:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:22.0:1616513086.725597:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693033574852416, offset 192 00000800:00000200:0.0:1616513086.725600:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.725602:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513086.725604:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.135@tcp1 00000800:00000200:0.0:1616513086.725607:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513086.725611:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.135@tcp1 routed destination 00000800:00000200:11.2:1616513086.725612:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.725613:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.725623:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513086.725626:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.135@tcp1(10.31.141.135@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513086.725630:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513086.725632:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513086.725633:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513086.725638:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513086.725640:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513086.725641:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.725643:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513086.725645:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:22.0:1616513086.725648:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513086.725648:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.725651:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.725653:0:50174:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.725709:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.725717:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.725720:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513086.725721:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.725726:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.725730:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4af68 00000800:00000200:14.0:1616513086.725734:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.733660:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.733713:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.733722:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.733743:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.24@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.733751:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.24@tcp1 of length 224 into portal 26 MB=0x6053558b8a980 00000400:00000200:12.0:1616513086.733757:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.24@tcp1 of length 224/224 into md 0xf97cbb99 [1] + 0 00000400:00000200:12.0:1616513086.733762:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d7896d8 00000400:00000200:12.0:1616513086.733764:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.733767:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.733770:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7896d8 00000800:00000200:12.0:1616513086.733773:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.733776:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.733777:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.733780:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.733792:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694576540166528 00010000:00000200:0.0:1616513086.733801:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1694576540166528/t0(0) o400->05c28725-4223-1b33-a3a5-62458be8f3a1@10.31.129.24@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.733814:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694576540166528, offset 192 00000400:00000200:0.0:1616513086.733820:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.24@tcp1 00000400:00000200:0.0:1616513086.733826:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.24@tcp1 routed destination 00000400:00000200:0.0:1616513086.733838:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.24@tcp1(10.31.129.24@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.733844:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.733849:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.733852:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.733854:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.733858:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.733862:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.733864:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.733873:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.733879:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.733882:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.733888:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.733893:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05320 00000800:00000200:20.0:1616513086.733896:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.733899:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.734145:0:50175:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.734201:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.734211:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.734220:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.24@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.734228:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.24@tcp1 of length 224 into portal 12 MB=0x6053558b8ad80 00000400:00000200:18.0:1616513086.734234:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.24@tcp1 of length 224/224 into md 0xf964d519 [1] + 55312 00000400:00000200:18.0:1616513086.734240:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.734242:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.734251:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.734254:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.734254:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.734258:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.734324:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694576540167552 00010000:00000200:22.0:1616513086.734344:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b8900 x1694576540167552/t0(0) o400->8469e58a-93fd-a702-b185-a0c68bd2df5c@10.31.129.24@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.734358:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694576540167552, offset 192 00000400:00000200:22.0:1616513086.734364:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.24@tcp1 00000400:00000200:22.0:1616513086.734371:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.24@tcp1 routed destination 00000400:00000200:22.0:1616513086.734386:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.24@tcp1(10.31.129.24@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.734392:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.734398:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.734401:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.734403:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.734408:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.734412:0:50175:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.734466:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.734472:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.734475:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.734482:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.734486:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bdc0 00000800:00000200:26.0:1616513086.734490:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.734492:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.738274:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.738336:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.738345:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.738355:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.35@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.738364:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.35@tcp1 of length 224 into portal 26 MB=0x5ffdc03bbb4c0 00000400:00000200:2.0:1616513086.738370:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.35@tcp1 of length 224/224 into md 0xf97cbba9 [1] + 0 00000400:00000200:2.0:1616513086.738375:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789078 00000400:00000200:2.0:1616513086.738378:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.738380:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513086.738385:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789078 00000800:00000200:18.0:1616513086.738388:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.738388:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.738390:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.738395:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.738398:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513086.738399:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.738406:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.35@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.738415:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.35@tcp1 of length 224 into portal 12 MB=0x5ffdc03bbb540 00000400:00000200:18.0:1616513086.738420:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.35@tcp1 of length 224/224 into md 0xf964d519 [1] + 55536 00000400:00000200:18.0:1616513086.738425:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.738428:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.738447:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.738450:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.738452:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.738459:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695304074432 00010000:00000200:0.0:1616513086.738469:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688695304074432/t0(0) o400->937ebe23-b23a-f732-2bb8-cf80c856cfed@10.31.143.35@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.738483:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695304074432, offset 192 00000400:00000200:0.0:1616513086.738489:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.35@tcp1 00000100:00000200:22.0:1616513086.738496:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695304074560 00000400:00000200:0.0:1616513086.738496:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.35@tcp1 routed destination 00010000:00000200:22.0:1616513086.738507:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645ba400 x1688695304074560/t0(0) o400->dc3efcaf-93a6-6712-b300-2f41cb0e1ace@10.31.143.35@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513086.738510:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.35@tcp1(10.31.143.35@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.738517:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:22.0:1616513086.738520:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695304074560, offset 192 00000800:00000200:0.0:1616513086.738522:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.738524:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513086.738526:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.35@tcp1 00000800:00000200:0.0:1616513086.738526:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.738531:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513086.738533:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.35@tcp1 routed destination 00000800:00000200:11.2:1616513086.738537:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.738537:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513086.738548:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.35@tcp1(10.31.143.35@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.738554:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.738559:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.738562:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.738564:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.738568:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.738573:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513086.738590:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.738597:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.738600:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.738607:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.738612:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04ee0 00000800:00000200:12.0:1616513086.738616:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.738619:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513086.738626:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.738633:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.738636:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513086.738637:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.738643:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.738647:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b430 00000800:00000200:14.0:1616513086.738651:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.782776:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513086.782833:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.782845:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513086.782855:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.24@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.782865:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.24@tcp1 of length 224 into portal 26 MB=0x60393be528780 00000400:00000200:20.0:1616513086.782872:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.24@tcp1 of length 224/224 into md 0xf97cbbc9 [1] + 0 00000400:00000200:20.0:1616513086.782877:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788c38 00000400:00000200:20.0:1616513086.782880:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.782883:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513086.782888:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788c38 00000800:00000200:24.0:1616513086.782890:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.782891:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.782894:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.782896:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.782957:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692782948419456 00010000:00000200:0.0:1616513086.782969:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1692782948419456/t0(0) o400->0b82010f-74b8-b093-cbf1-994468cce8ca@10.31.140.24@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.782984:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692782948419456, offset 192 00000400:00000200:0.0:1616513086.782991:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.24@tcp1 00000400:00000200:0.0:1616513086.782999:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.24@tcp1 routed destination 00000400:00000200:0.0:1616513086.783015:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.24@tcp1(10.31.140.24@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.783022:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.783028:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.783031:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.783033:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.783038:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.783043:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.783045:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513086.783097:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.783104:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.783106:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.783113:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.783119:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb053a8 00000800:00000200:12.0:1616513086.783123:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.783126:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.783421:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.783474:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.783484:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.783492:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.24@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.783500:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.24@tcp1 of length 224 into portal 12 MB=0x60393be528840 00000400:00000200:26.0:1616513086.783505:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.24@tcp1 of length 224/224 into md 0xf964d519 [1] + 55760 00000400:00000200:26.0:1616513086.783510:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.783512:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513086.783519:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.783522:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513086.783523:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.783526:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.783584:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692782948419648 00010000:00000200:22.0:1616513086.783596:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dbf00 x1692782948419648/t0(0) o400->505d95b5-fa16-4a41-b866-2216c6071c74@10.31.140.24@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.783610:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692782948419648, offset 192 00000400:00000200:22.0:1616513086.783615:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.24@tcp1 00000400:00000200:22.0:1616513086.783622:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.24@tcp1 routed destination 00000400:00000200:22.0:1616513086.783636:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.24@tcp1(10.31.140.24@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.783642:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.783648:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.783651:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.783653:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.783658:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.783663:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.783717:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.783724:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.783727:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513086.783732:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.783734:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.783737:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a770 00000800:00000200:2.0:1616513086.783741:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.785656:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513086.785710:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.785719:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513086.785728:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.215@tcp1 : PUT - for me 00000400:00000200:24.0:1616513086.785738:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.215@tcp1 of length 224 into portal 26 MB=0x6001454c3c2c0 00000400:00000200:24.0:1616513086.785744:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.215@tcp1 of length 224/224 into md 0xf97cbbf9 [1] + 0 00000400:00000200:24.0:1616513086.785749:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788000 00000400:00000200:24.0:1616513086.785752:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.785754:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513086.785759:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788000 00000800:00000200:24.0:1616513086.785762:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.785764:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513086.785764:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513086.785768:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.785806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513086.785824:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688937181725376 00010000:00000200:0.0:1616513086.785835:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688937181725376/t0(0) o400->409a6b55-3f07-5ac3-7b63-fd7967535cc3@10.31.136.215@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.785849:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688937181725376, offset 192 00000400:00000200:0.0:1616513086.785856:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.215@tcp1 00000800:00000200:12.0:1616513086.785858:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513086.785864:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.215@tcp1 routed destination 00000800:00000200:12.0:1616513086.785867:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513086.785879:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.215@tcp1(10.31.136.215@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.785886:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.785888:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.215@tcp1 : PUT - for me 00000800:00000200:0.0:1616513086.785891:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.785894:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513086.785896:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:12.0:1616513086.785898:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.215@tcp1 of length 224 into portal 12 MB=0x6001454c3c380 00000800:00000200:0.0:1616513086.785901:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:12.0:1616513086.785904:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.215@tcp1 of length 224/224 into md 0x865bbec1 [1] + 86688 00000100:00000200:0.0:1616513086.785907:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513086.785909:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:28.0:1616513086.785912:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:12.0:1616513086.785912:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513086.785919:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:28.0:1616513086.785920:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513086.785922:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513086.785923:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513086.785924:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.785926:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513086.785929:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.785935:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000800:00000200:28.0:1616513086.785938:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.785940:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688937181725568 00010000:00000200:4.0:1616513086.785953:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d207500 x1688937181725568/t0(0) o400->9e067d7d-eaa9-b642-d6c4-1f2e7d757692@10.31.136.215@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.785967:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688937181725568, offset 192 00000400:00000200:4.0:1616513086.785974:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.215@tcp1 00000400:00000200:4.0:1616513086.785982:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.215@tcp1 routed destination 00000400:00000200:4.0:1616513086.785997:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.215@tcp1(10.31.136.215@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.786004:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.786010:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.786012:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.786015:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.786019:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.786025:0:50181:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.786076:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.786082:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.786084:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.786090:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.786094:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb5c8 00000800:00000200:26.0:1616513086.786097:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.786100:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.786834:0:50181:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513086.786846:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.786853:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513086.786862:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.229@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.786872:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.229@tcp1 of length 224 into portal 26 MB=0x5ffd469d92900 00000400:00000200:14.0:1616513086.786879:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.229@tcp1 of length 224/224 into md 0xf97cbc19 [1] + 0 00000400:00000200:14.0:1616513086.786884:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d7886e8 00000400:00000200:14.0:1616513086.786887:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.786890:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513086.786901:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7886e8 00000800:00000200:2.0:1616513086.786904:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.786905:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.786907:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.786909:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.786910:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662657542400 00010000:00000200:0.0:1616513086.786916:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688662657542400/t0(0) o400->9060de30-d566-ca85-5078-cfb479cd8593@10.31.132.229@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.786927:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662657542400, offset 192 00000400:00000200:0.0:1616513086.786934:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.229@tcp1 00000400:00000200:0.0:1616513086.786941:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.229@tcp1 routed destination 00000400:00000200:0.0:1616513086.786953:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.229@tcp1(10.31.132.229@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.786959:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.786964:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.786966:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.786968:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.786971:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.786975:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.786975:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513086.787028:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.787035:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513086.787038:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513086.787039:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513086.787045:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513086.787050:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04ee0 00000800:00000200:24.0:1616513086.787054:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.787629:0:50181:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.787684:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.787693:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.787702:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.229@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.787711:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.229@tcp1 of length 224 into portal 12 MB=0x5ffd469d92a00 00000400:00000200:26.0:1616513086.787718:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.229@tcp1 of length 224/224 into md 0xf964d519 [1] + 55984 00000400:00000200:26.0:1616513086.787723:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.787725:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513086.787732:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.787735:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513086.787737:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.787740:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.787795:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662657542656 00010000:00000200:22.0:1616513086.787807:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8900 x1688662657542656/t0(0) o400->89c40b51-1ca8-60a2-bbde-d70e7f83e696@10.31.132.229@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.787822:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662657542656, offset 192 00000400:00000200:22.0:1616513086.787828:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.229@tcp1 00000400:00000200:22.0:1616513086.787836:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.229@tcp1 routed destination 00000400:00000200:22.0:1616513086.787851:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.229@tcp1(10.31.132.229@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.787857:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.787863:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.787866:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.787870:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.787874:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.787879:0:50181:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.787890:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.787895:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.787898:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.787898:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513086.787904:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.787908:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b8f8 00000800:00000200:2.0:1616513086.787912:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.798125:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.798180:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.798189:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513086.798195:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513086.798198:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.219@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.798207:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.219@tcp1 of length 224 into portal 26 MB=0x5ffd3d4bb3f40 00000400:00000200:12.0:1616513086.798213:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.219@tcp1 of length 224/224 into md 0xf97cbc39 [1] + 0 00000400:00000200:12.0:1616513086.798218:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789760 00000400:00000200:12.0:1616513086.798221:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.798223:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.798228:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789760 00000800:00000200:12.0:1616513086.798231:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.798233:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.798235:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.798293:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660155776832 00010000:00000200:0.0:1616513086.798304:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688660155776832/t0(0) o400->1f56c666-d29c-d061-543b-c31a2e4a2f26@10.31.128.219@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.798318:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660155776832, offset 192 00000400:00000200:0.0:1616513086.798324:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.219@tcp1 00000400:00000200:0.0:1616513086.798332:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.219@tcp1 routed destination 00000400:00000200:0.0:1616513086.798347:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.219@tcp1(10.31.128.219@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.798353:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.798359:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.798362:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:11.2:1616513086.798363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.798364:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.798369:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513086.798375:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.798376:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.798385:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513086.798393:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.219@tcp1 : PUT - for me 00000400:00000200:20.0:1616513086.798402:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.219@tcp1 of length 224 into portal 12 MB=0x5ffd3d4bb42c0 00000400:00000200:20.0:1616513086.798409:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.219@tcp1 of length 224/224 into md 0x865bbec1 [1] + 86912 00000400:00000200:20.0:1616513086.798414:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513086.798416:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513086.798423:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513086.798426:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513086.798428:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513086.798430:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.798441:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513086.798448:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513086.798450:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.798456:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.798460:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:20.0:1616513086.798463:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.798485:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660155777728 00010000:00000200:4.0:1616513086.798497:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38e300 x1688660155777728/t0(0) o400->0fc5b2d3-c693-c2c6-a87e-6b73b893e6db@10.31.128.219@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.798510:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660155777728, offset 192 00000400:00000200:4.0:1616513086.798516:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.219@tcp1 00000400:00000200:4.0:1616513086.798523:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.219@tcp1 routed destination 00000400:00000200:4.0:1616513086.798537:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.219@tcp1(10.31.128.219@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.798543:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.798548:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.798552:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.798554:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.798559:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.798567:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.798580:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.798586:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.798589:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.798594:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.798604:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba908 00000800:00000200:26.0:1616513086.798607:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.798612:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.802220:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513086.802276:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.802286:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513086.802295:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.199@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.802304:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.199@tcp1 of length 224 into portal 26 MB=0x5ffea94d93340 00000400:00000200:14.0:1616513086.802313:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.199@tcp1 of length 224/224 into md 0xf97cbc59 [1] + 0 00000400:00000200:14.0:1616513086.802318:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789210 00000400:00000200:14.0:1616513086.802320:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.802323:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513086.802327:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789210 00000800:00000200:14.0:1616513086.802330:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.802331:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.802333:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.802338:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.802341:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.802341:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.802350:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.199@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.802359:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.199@tcp1 of length 224 into portal 12 MB=0x5ffea94d93640 00000400:00000200:2.0:1616513086.802365:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.199@tcp1 of length 224/224 into md 0xf964d519 [1] + 56208 00000400:00000200:2.0:1616513086.802370:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.802373:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513086.802380:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.802383:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.802385:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.802390:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688757868245824 00010000:00000200:0.0:1616513086.802400:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688757868245824/t0(0) o400->e8d3417b-d121-9889-0fa7-9765eb0be5d5@10.31.141.199@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.802413:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688757868245824, offset 192 00000400:00000200:0.0:1616513086.802419:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.199@tcp1 00000400:00000200:0.0:1616513086.802425:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.199@tcp1 routed destination 00000400:00000200:0.0:1616513086.802452:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.199@tcp1(10.31.141.199@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513086.802460:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688757868246592 00000800:00000200:0.0:1616513086.802462:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.802468:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.802470:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.802472:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513086.802473:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebbf00 x1688757868246592/t0(0) o400->a22757f5-b14c-ab00-ee00-3e99fbc22488@10.31.141.199@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513086.802477:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.802484:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.802484:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513086.802488:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688757868246592, offset 192 00000400:00000200:22.0:1616513086.802494:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.199@tcp1 00000400:00000200:22.0:1616513086.802501:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.199@tcp1 routed destination 00000400:00000200:22.0:1616513086.802515:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.199@tcp1(10.31.141.199@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.802522:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.802527:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.802530:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.802532:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.802537:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513086.802540:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513086.802542:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513086.802554:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.802556:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.802563:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.802568:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:12.0:1616513086.802571:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.802574:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513086.802616:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.802624:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.802627:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513086.802628:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.802633:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.802637:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a7f8 00000800:00000200:18.0:1616513086.802641:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.853165:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.853221:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.853231:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.853241:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.204@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.853252:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.204@tcp1 of length 224 into portal 26 MB=0x605a0737237c0 00000400:00000200:28.0:1616513086.853259:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.204@tcp1 of length 224/224 into md 0xf97cbc79 [1] + 0 00000400:00000200:28.0:1616513086.853264:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789ba0 00000400:00000200:28.0:1616513086.853267:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.853271:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513086.853275:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513086.853276:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789ba0 00000800:00000200:28.0:1616513086.853279:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.853282:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.853284:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.853357:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036550035392 00000800:00000200:7.2:1616513086.853367:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:0.0:1616513086.853368:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1695036550035392/t0(0) o400->7a0fa776-5961-3c9e-b0ab-eeb24f3cd9cd@10.31.129.204@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.853383:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036550035392, offset 192 00000400:00000200:0.0:1616513086.853389:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.204@tcp1 00000400:00000200:0.0:1616513086.853397:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.204@tcp1 routed destination 00000400:00000200:0.0:1616513086.853412:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.204@tcp1(10.31.129.204@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.853419:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513086.853424:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.853424:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.853427:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.853429:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:11.2:1616513086.853443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513086.853447:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513086.853449:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:0.0:1616513086.853455:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513086.853458:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.204@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.853468:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.204@tcp1 of length 224 into portal 12 MB=0x605a073723900 00000400:00000200:2.0:1616513086.853475:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.204@tcp1 of length 224/224 into md 0xf964d519 [1] + 56432 00000800:00000200:26.0:1616513086.853478:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.853480:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.853483:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513086.853495:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.853500:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.853502:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513086.853505:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.853512:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.853514:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.853520:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.853527:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:12.0:1616513086.853530:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.853533:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513086.853561:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036550035712 00010000:00000200:22.0:1616513086.853574:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebb180 x1695036550035712/t0(0) o400->7bd5c375-9518-579b-1a96-4e68a8648166@10.31.129.204@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.853589:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036550035712, offset 192 00000400:00000200:22.0:1616513086.853595:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.204@tcp1 00000400:00000200:22.0:1616513086.853603:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.204@tcp1 routed destination 00000400:00000200:22.0:1616513086.853618:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.204@tcp1(10.31.129.204@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.853624:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.853631:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.853634:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.853636:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.853641:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.853646:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.853702:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.853710:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.853713:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513086.853719:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.853723:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b210 00000800:00000200:14.0:1616513086.853727:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.853730:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.853990:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.854001:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.854009:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513086.854011:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513086.854018:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.179@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.854027:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.179@tcp1 of length 224 into portal 26 MB=0x5ffdcfe0b5680 00000400:00000200:26.0:1616513086.854034:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.179@tcp1 of length 224/224 into md 0xf97cbc99 [1] + 0 00000400:00000200:26.0:1616513086.854039:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d7898f8 00000400:00000200:26.0:1616513086.854041:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.854044:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513086.854048:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7898f8 00000800:00000200:26.0:1616513086.854051:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.854054:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.854056:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.854062:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699503597184 00010000:00000200:0.0:1616513086.854068:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688699503597184/t0(0) o400->8223f229-2d9c-b331-4d80-b1e0794f67b3@10.31.131.179@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.854079:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699503597184, offset 192 00000400:00000200:0.0:1616513086.854085:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.179@tcp1 00000400:00000200:0.0:1616513086.854091:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.179@tcp1 routed destination 00000400:00000200:0.0:1616513086.854105:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.179@tcp1(10.31.131.179@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.854111:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.854115:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.854118:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.854121:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.854125:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513086.854129:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513086.854131:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513086.854151:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.854157:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.854161:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513086.854161:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.854168:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.854173:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb042a8 00000800:00000200:20.0:1616513086.854177:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.854303:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513086.854312:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.854318:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513086.854322:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.854326:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.179@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.854334:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.179@tcp1 of length 224 into portal 12 MB=0x5ffdcfe0b5740 00000400:00000200:14.0:1616513086.854341:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.179@tcp1 of length 224/224 into md 0xf964d519 [1] + 56656 00000400:00000200:14.0:1616513086.854346:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.854348:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513086.854355:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.854357:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.854359:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.854366:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699503597376 00010000:00000200:22.0:1616513086.854373:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7ba80 x1688699503597376/t0(0) o400->cf4fb38b-c555-c682-896e-1059195f3e36@10.31.131.179@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.854384:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699503597376, offset 192 00000400:00000200:22.0:1616513086.854391:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.179@tcp1 00000400:00000200:22.0:1616513086.854397:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.179@tcp1 routed destination 00000400:00000200:22.0:1616513086.854410:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.179@tcp1(10.31.131.179@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.854416:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.854420:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.854423:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.854424:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.854427:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.854432:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.854446:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.854450:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.854452:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513086.854454:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.854457:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.854461:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a220 00000800:00000200:2.0:1616513086.854464:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.856162:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513086.856215:0:50189:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513086.856216:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.856225:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.856234:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.66@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.856243:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.66@tcp1 of length 224 into portal 26 MB=0x603ff69d6c440 00000400:00000200:12.0:1616513086.856250:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.66@tcp1 of length 224/224 into md 0xf97cbca9 [1] + 0 00000400:00000200:12.0:1616513086.856255:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788198 00000400:00000200:12.0:1616513086.856257:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.856260:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.856264:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788198 00000800:00000200:12.0:1616513086.856267:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513086.856269:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513086.856270:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.856272:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.856274:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513086.856278:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.856287:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.66@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.856296:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.66@tcp1 of length 224 into portal 12 MB=0x603ff69d6c4c0 00000400:00000200:18.0:1616513086.856302:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.66@tcp1 of length 224/224 into md 0xf964d519 [1] + 56880 00000400:00000200:18.0:1616513086.856308:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.856310:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513086.856317:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.856319:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.856321:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.856324:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.856344:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245387490368 00010000:00000200:0.0:1616513086.856355:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1693245387490368/t0(0) o400->c9d37027-d047-029f-5f03-49c231593fa1@10.31.132.66@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.856369:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245387490368, offset 192 00000400:00000200:0.0:1616513086.856376:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.66@tcp1 00000100:00000200:22.0:1616513086.856380:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245387490496 00000400:00000200:0.0:1616513086.856383:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.66@tcp1 routed destination 00010000:00000200:22.0:1616513086.856393:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bba80 x1693245387490496/t0(0) o400->29b0b146-acf7-c87a-85f4-871f797667d8@10.31.132.66@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513086.856399:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.66@tcp1(10.31.132.66@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.856405:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:22.0:1616513086.856407:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245387490496, offset 192 00000800:00000200:0.0:1616513086.856411:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513086.856413:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.66@tcp1 00000800:00000200:0.0:1616513086.856414:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.856416:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513086.856420:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.66@tcp1 routed destination 00000800:00000200:0.0:1616513086.856421:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.856426:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.856428:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513086.856445:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.66@tcp1(10.31.132.66@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.856452:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.856457:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.856460:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.856462:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.856467:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.856472:0:50189:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513086.856481:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.856488:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.856490:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513086.856493:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513086.856497:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.856502:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000800:00000200:28.0:1616513086.856506:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513086.856525:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.856531:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.856534:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.856541:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.856545:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a440 00000800:00000200:26.0:1616513086.856549:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.856551:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:31.0:1616513086.871545:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513086.871553:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513086.874089:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.874144:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.874153:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.874162:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.184@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.874171:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.184@tcp1 of length 224 into portal 12 MB=0x5ffdc5841be80 00000400:00000200:12.0:1616513086.874178:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.184@tcp1 of length 224/224 into md 0x865bbec1 [1] + 87136 00000400:00000200:12.0:1616513086.874183:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.874186:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513086.874201:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.874203:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.874204:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.874207:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.874265:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696722144896 00010000:00000200:4.0:1616513086.874277:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d200d80 x1688696722144896/t0(0) o400->a7f8ee54-c705-205e-8e36-67aa2b9eae0e@10.31.141.184@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513086.874289:0:50191:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:4.0:1616513086.874290:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696722144896, offset 192 00000400:00000200:4.0:1616513086.874296:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.184@tcp1 00000400:00000200:4.0:1616513086.874303:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.184@tcp1 routed destination 00000400:00000200:4.0:1616513086.874318:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.184@tcp1(10.31.141.184@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.874324:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513086.874329:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.874332:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513086.874334:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513086.874338:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.874343:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513086.874346:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.874358:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.874367:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.184@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.874376:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.184@tcp1 of length 224 into portal 26 MB=0x5ffdc5841bb80 00000400:00000200:18.0:1616513086.874383:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.184@tcp1 of length 224/224 into md 0xf97cbcb9 [1] + 0 00000400:00000200:18.0:1616513086.874387:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788bb0 00000400:00000200:18.0:1616513086.874390:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.874393:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513086.874396:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513086.874397:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788bb0 00000800:00000200:18.0:1616513086.874400:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513086.874403:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513086.874403:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513086.874405:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513086.874406:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513086.874408:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513086.874412:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.874416:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6babb0 00000800:00000200:20.0:1616513086.874420:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.874423:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.874465:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696722144128 00010000:00000200:0.0:1616513086.874476:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688696722144128/t0(0) o400->4ed24d3c-d5f5-8513-3caf-84840f92a7c8@10.31.141.184@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.874490:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696722144128, offset 192 00000400:00000200:0.0:1616513086.874497:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.184@tcp1 00000400:00000200:0.0:1616513086.874504:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.184@tcp1 routed destination 00000400:00000200:0.0:1616513086.874520:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.184@tcp1(10.31.141.184@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.874526:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513086.874532:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.874535:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.874537:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.874542:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.874546:0:50191:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513086.874549:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513086.874602:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.874609:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.874612:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513086.874619:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.874624:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:26.0:1616513086.874628:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.874630:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.875055:0:50191:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.875066:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.875072:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513086.875076:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.875079:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.122@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.875086:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.122@tcp1 of length 224 into portal 26 MB=0x5ffe357c50840 00000400:00000200:18.0:1616513086.875092:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.122@tcp1 of length 224/224 into md 0xf97cbce9 [1] + 0 00000400:00000200:18.0:1616513086.875095:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788f68 00000400:00000200:18.0:1616513086.875097:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.875099:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513086.875103:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788f68 00000800:00000200:18.0:1616513086.875105:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.875107:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.875108:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.875116:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726778742848 00010000:00000200:0.0:1616513086.875122:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688726778742848/t0(0) o400->554fbfcd-fec4-2a46-05eb-76a06eaa159c@10.31.141.122@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.875135:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726778742848, offset 192 00000400:00000200:0.0:1616513086.875140:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.122@tcp1 00000400:00000200:0.0:1616513086.875146:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.122@tcp1 routed destination 00000400:00000200:0.0:1616513086.875159:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.122@tcp1(10.31.141.122@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.875165:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.875170:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.875172:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.875174:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.875177:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.875182:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.875182:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513086.875193:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.875199:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513086.875201:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513086.875203:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513086.875210:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513086.875215:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:24.0:1616513086.875219:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.875823:0:50191:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.875878:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.875887:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.875896:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.122@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.875905:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.122@tcp1 of length 224 into portal 12 MB=0x5ffe357c50fc0 00000400:00000200:26.0:1616513086.875911:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.122@tcp1 of length 224/224 into md 0xf964d519 [1] + 57104 00000400:00000200:26.0:1616513086.875919:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.875922:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513086.875928:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.875931:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513086.875933:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513086.875936:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.875994:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726778744768 00010000:00000200:22.0:1616513086.876006:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8d80 x1688726778744768/t0(0) o400->3a26f46b-0063-3a47-a009-b6f54a34e2ee@10.31.141.122@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.876020:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726778744768, offset 192 00000400:00000200:22.0:1616513086.876026:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.122@tcp1 00000400:00000200:22.0:1616513086.876034:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.122@tcp1 routed destination 00000400:00000200:22.0:1616513086.876049:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.122@tcp1(10.31.141.122@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.876055:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.876061:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.876066:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.876068:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.876073:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.876077:0:50191:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.876088:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.876094:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.876096:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513086.876098:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.876103:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.876107:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a6e8 00000800:00000200:14.0:1616513086.876112:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.919080:0:50196:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.919139:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.919149:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.919159:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.53@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.919170:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.53@tcp1 of length 224 into portal 26 MB=0x602fbc4cc9d40 00000400:00000200:2.0:1616513086.919177:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.53@tcp1 of length 224/224 into md 0xf97cbcf9 [1] + 0 00000400:00000200:2.0:1616513086.919182:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789980 00000400:00000200:2.0:1616513086.919185:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.919189:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513086.919193:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513086.919193:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789980 00000800:00000200:2.0:1616513086.919196:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.919199:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.919201:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.919272:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692130222054720 00010000:00000200:0.0:1616513086.919283:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1692130222054720/t0(0) o400->8efc0da2-9a08-7499-81d1-d692b2456189@10.31.142.53@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.919299:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692130222054720, offset 192 00000400:00000200:0.0:1616513086.919305:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.53@tcp1 00000400:00000200:0.0:1616513086.919313:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.53@tcp1 routed destination 00000400:00000200:0.0:1616513086.919328:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.53@tcp1(10.31.142.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513086.919334:0:50196:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.919334:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.919340:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.919343:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.919345:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.919350:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.919357:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.919357:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.919370:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.919377:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.919380:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.919387:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513086.919389:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513086.919393:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:20.0:1616513086.919396:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.919399:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513086.919399:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513086.919408:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.53@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.919417:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.53@tcp1 of length 224 into portal 12 MB=0x602fbc4cc9e80 00000400:00000200:18.0:1616513086.919423:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.53@tcp1 of length 224/224 into md 0xf964d519 [1] + 57328 00000400:00000200:18.0:1616513086.919429:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.919431:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513086.919451:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.919455:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.919459:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.919461:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.919520:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692130222055040 00010000:00000200:22.0:1616513086.919533:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bb180 x1692130222055040/t0(0) o400->debad8e8-a302-7def-3bde-b81fff2c823c@10.31.142.53@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.919546:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692130222055040, offset 192 00000400:00000200:22.0:1616513086.919552:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.53@tcp1 00000400:00000200:22.0:1616513086.919559:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.53@tcp1 routed destination 00000400:00000200:22.0:1616513086.919573:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.53@tcp1(10.31.142.53@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.919579:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.919585:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.919588:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.919590:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.919594:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.919601:0:50196:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.919613:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513086.919618:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513086.919621:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513086.919623:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513086.919627:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.919631:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a088 00000800:00000200:26.0:1616513086.919635:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.922725:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513086.922782:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.922792:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513086.922801:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.149@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.922809:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.149@tcp1 of length 224 into portal 26 MB=0x5ffdc277f6fc0 00000400:00000200:14.0:1616513086.922816:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.149@tcp1 of length 224/224 into md 0xf97cbd19 [1] + 0 00000400:00000200:14.0:1616513086.922821:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788b28 00000400:00000200:14.0:1616513086.922823:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.922826:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513086.922830:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788b28 00000800:00000200:14.0:1616513086.922833:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.922835:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.922837:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.922839:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.922846:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513086.922853:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.149@tcp1 : PUT - for me 00000800:00000200:18.0:1616513086.922854:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.922858:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.149@tcp1 of length 224 into portal 12 MB=0x5ffdc277f7340 00000400:00000200:14.0:1616513086.922864:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.149@tcp1 of length 224/224 into md 0xf964d519 [1] + 57552 00000400:00000200:14.0:1616513086.922867:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.922870:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513086.922874:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.922876:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.922878:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.922914:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695904104384 00010000:00000200:0.0:1616513086.922925:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688695904104384/t0(0) o400->0fd9a6f0-f4e0-d8a4-57cd-785e4c8cce3d@10.31.141.149@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.922939:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695904104384, offset 192 00000100:00000200:22.0:1616513086.922940:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695904105280 00000400:00000200:0.0:1616513086.922945:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.149@tcp1 00000400:00000200:0.0:1616513086.922952:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.149@tcp1 routed destination 00010000:00000200:22.0:1616513086.922953:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a79680 x1688695904105280/t0(0) o400->fbf449c8-5d5d-a878-6456-65d1c5da0ab5@10.31.141.149@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.922967:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695904105280, offset 192 00000400:00000200:0.0:1616513086.922969:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.149@tcp1(10.31.141.149@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513086.922973:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.149@tcp1 00000800:00000200:0.0:1616513086.922976:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513086.922980:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.149@tcp1 routed destination 00000800:00000200:0.0:1616513086.922981:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.922984:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.922986:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.922991:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513086.922995:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.149@tcp1(10.31.141.149@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513086.922996:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.922998:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513086.923002:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.923007:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.923010:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.923012:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.923017:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.923022:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513086.923048:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.923054:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.923057:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.923063:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.923068:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000800:00000200:12.0:1616513086.923072:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513086.923076:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513086.923076:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.923082:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513086.923083:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513086.923084:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513086.923086:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513086.923086:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513086.923086:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513086.923086:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.923090:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513086.923092:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513086.923096:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ab28 00000800:00000200:26.0:1616513086.923100:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513086.927282:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513086.927337:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.927348:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513086.927356:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.150@tcp1 : PUT - for me 00000400:00000200:28.0:1616513086.927366:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.150@tcp1 of length 224 into portal 26 MB=0x603fce3d53c40 00000400:00000200:28.0:1616513086.927372:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.150@tcp1 of length 224/224 into md 0xf97cbd29 [1] + 0 00000400:00000200:28.0:1616513086.927377:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789b18 00000400:00000200:28.0:1616513086.927380:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513086.927382:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513086.927387:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789b18 00000800:00000200:28.0:1616513086.927390:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.927391:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513086.927393:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.927395:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513086.927463:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234549308480 00010000:00000200:0.0:1616513086.927474:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1693234549308480/t0(0) o400->178fa0e3-a175-ccc0-0f7e-bcd4fefa1d41@10.31.138.150@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513086.927480:0:50197:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513086.927488:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234549308480, offset 192 00000400:00000200:0.0:1616513086.927495:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.150@tcp1 00000400:00000200:0.0:1616513086.927502:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.150@tcp1 routed destination 00000400:00000200:0.0:1616513086.927518:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.150@tcp1(10.31.138.150@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.927524:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.927530:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.927532:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.927534:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513086.927536:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.927539:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.927544:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513086.927546:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:0.0:1616513086.927546:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513086.927555:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.150@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.927564:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.150@tcp1 of length 224 into portal 12 MB=0x603fce3d53cc0 00000400:00000200:14.0:1616513086.927570:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.150@tcp1 of length 224/224 into md 0xf964d519 [1] + 57776 00000400:00000200:14.0:1616513086.927575:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.927578:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513086.927584:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.927589:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.927591:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.927595:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513086.927610:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.927617:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513086.927619:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513086.927626:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513086.927632:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:12.0:1616513086.927635:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513086.927638:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513086.927651:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234549308608 00010000:00000200:22.0:1616513086.927663:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a78d80 x1693234549308608/t0(0) o400->39a9bd8d-3583-6f16-ea61-c63d84654250@10.31.138.150@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.927688:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234549308608, offset 192 00000400:00000200:22.0:1616513086.927695:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.150@tcp1 00000400:00000200:22.0:1616513086.927702:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.150@tcp1 routed destination 00000400:00000200:22.0:1616513086.927717:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.150@tcp1(10.31.138.150@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.927723:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.927729:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.927739:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.927741:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.927746:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.927750:0:50197:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.927804:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.927810:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.927813:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.927820:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.927824:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b3a8 00000800:00000200:18.0:1616513086.927827:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.927830:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.954681:0:50200:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.954739:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.954748:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.954758:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.15@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.954768:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.15@tcp1 of length 224 into portal 26 MB=0x5ffe0bf26b6c0 00000400:00000200:2.0:1616513086.954775:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.15@tcp1 of length 224/224 into md 0xf97cbd49 [1] + 0 00000400:00000200:2.0:1616513086.954780:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d7883b8 00000400:00000200:2.0:1616513086.954783:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.954785:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513086.954790:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7883b8 00000800:00000200:2.0:1616513086.954793:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.954794:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.954798:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513086.954800:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.954856:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688715628295872 00010000:00000200:0.0:1616513086.954868:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688715628295872/t0(0) o400->6b489312-80e9-f35d-7c02-9aa4c092ce8d@10.31.133.15@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.954882:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688715628295872, offset 192 00000400:00000200:0.0:1616513086.954889:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.15@tcp1 00000400:00000200:0.0:1616513086.954897:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.15@tcp1 routed destination 00000400:00000200:0.0:1616513086.954912:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.15@tcp1(10.31.133.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.954919:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.954925:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.954930:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.954932:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.954937:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.954944:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.954944:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.954958:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.954965:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.954968:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.954975:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.954980:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:20.0:1616513086.954984:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.954987:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.955243:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.955307:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.955316:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.955326:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.15@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.955334:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.15@tcp1 of length 224 into portal 12 MB=0x5ffe0bf26b8c0 00000400:00000200:12.0:1616513086.955341:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.15@tcp1 of length 224/224 into md 0x865bbec1 [1] + 87360 00000400:00000200:12.0:1616513086.955346:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.955349:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513086.955355:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.955358:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513086.955360:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513086.955363:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513086.955419:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688715628296384 00010000:00000200:4.0:1616513086.955432:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d206780 x1688715628296384/t0(0) o400->a23bed4a-bdcd-bc5b-110b-061c8ba5c097@10.31.133.15@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.955460:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688715628296384, offset 192 00000400:00000200:4.0:1616513086.955466:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.15@tcp1 00000400:00000200:4.0:1616513086.955474:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.15@tcp1 routed destination 00000400:00000200:4.0:1616513086.955489:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.15@tcp1(10.31.133.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.955496:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.955502:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.955505:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.955507:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.955512:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.955516:0:50200:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.955575:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.955582:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.955585:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.955591:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.955595:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba770 00000800:00000200:18.0:1616513086.955599:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.955602:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.982039:0:50203:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513086.982095:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.982106:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513086.982116:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.81@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.982126:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.81@tcp1 of length 224 into portal 26 MB=0x5ffe09b225140 00000400:00000200:14.0:1616513086.982133:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.81@tcp1 of length 224/224 into md 0xf97cbd59 [1] + 0 00000400:00000200:14.0:1616513086.982138:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789650 00000400:00000200:14.0:1616513086.982141:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.982144:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513086.982148:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789650 00000800:00000200:2.0:1616513086.982150:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.982151:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.982154:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.982156:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.982214:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688715024027968 00010000:00000200:0.0:1616513086.982226:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688715024027968/t0(0) o400->8f367b6b-6b5a-bf2a-c35c-8dca6f53dc8d@10.31.132.81@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513086.982241:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688715024027968, offset 192 00000400:00000200:0.0:1616513086.982247:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.81@tcp1 00000400:00000200:0.0:1616513086.982256:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.81@tcp1 routed destination 00000400:00000200:0.0:1616513086.982271:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.81@tcp1(10.31.132.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.982277:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513086.982284:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.982286:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513086.982289:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.982294:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.982301:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.982301:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.982314:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.982321:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.982324:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.982330:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.982336:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000800:00000200:20.0:1616513086.982340:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.982343:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513086.983153:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513086.983207:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513086.983216:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.983225:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.82@tcp1 : PUT - for me 00000400:00000200:12.0:1616513086.983234:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.82@tcp1 of length 224 into portal 26 MB=0x604a4097882c0 00000400:00000200:12.0:1616513086.983241:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.82@tcp1 of length 224/224 into md 0xf97cbd79 [1] + 0 00000400:00000200:12.0:1616513086.983246:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788220 00000400:00000200:12.0:1616513086.983248:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.983251:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513086.983256:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788220 00000800:00000200:12.0:1616513086.983259:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513086.983261:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513086.983262:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513086.983268:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693952440304320 00000800:00000200:12.0:1616513086.983269:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513086.983272:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513086.983272:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:0.0:1616513086.983274:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1693952440304320/t0(0) o400->eddf8828-cd1c-61a2-e938-5e7445ad4b1b@10.31.140.82@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:24.0:1616513086.983281:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.82@tcp1 : PUT - for me 00000100:00000200:0.0:1616513086.983285:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693952440304320, offset 192 00000400:00000200:24.0:1616513086.983290:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.82@tcp1 of length 224 into portal 12 MB=0x604a409788400 00000400:00000200:0.0:1616513086.983290:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.82@tcp1 00000400:00000200:0.0:1616513086.983296:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.82@tcp1 routed destination 00000400:00000200:24.0:1616513086.983297:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.82@tcp1 of length 224/224 into md 0x865bbec1 [1] + 87584 00000400:00000200:24.0:1616513086.983302:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513086.983304:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513086.983309:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.82@tcp1(10.31.140.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513086.983312:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513086.983315:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513086.983315:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513086.983317:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513086.983320:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.983322:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513086.983324:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.983327:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513086.983331:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513086.983332:0:50203:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513086.983394:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693952440304640 00010000:00000200:4.0:1616513086.983407:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e0d80 x1693952440304640/t0(0) o400->306c6b7e-17ac-ea08-5336-489de1166210@10.31.140.82@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513086.983411:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.983418:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.983421:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.983428:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.983446:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:18.0:1616513086.983450:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513086.983450:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693952440304640, offset 192 00000800:00000200:18.0:1616513086.983453:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513086.983457:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.82@tcp1 00000400:00000200:4.0:1616513086.983464:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.82@tcp1 routed destination 00000400:00000200:4.0:1616513086.983479:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.82@tcp1(10.31.140.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.983486:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513086.983492:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.983495:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513086.983497:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513086.983502:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.983507:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513086.983567:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513086.983573:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513086.983576:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513086.983577:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513086.983583:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513086.983587:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba7f8 00000800:00000200:28.0:1616513086.983591:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.984311:0:50203:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513086.984367:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513086.984376:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513086.984385:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.81@tcp1 : PUT - for me 00000400:00000200:2.0:1616513086.984394:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.81@tcp1 of length 224 into portal 12 MB=0x5ffe09b225580 00000400:00000200:2.0:1616513086.984400:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.81@tcp1 of length 224/224 into md 0xf964d519 [1] + 58000 00000400:00000200:2.0:1616513086.984406:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513086.984409:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513086.984416:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513086.984419:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513086.984420:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513086.984424:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.984502:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688715024029056 00010000:00000200:22.0:1616513086.984514:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebd580 x1688715024029056/t0(0) o400->32c9ed31-5928-920b-8720-abae88395a69@10.31.132.81@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513086.984529:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688715024029056, offset 192 00000400:00000200:22.0:1616513086.984535:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.81@tcp1 00000400:00000200:22.0:1616513086.984542:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.81@tcp1 routed destination 00000400:00000200:22.0:1616513086.984557:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.81@tcp1(10.31.132.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513086.984564:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513086.984570:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513086.984572:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.984575:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513086.984579:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.984585:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.984596:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513086.984600:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513086.984602:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513086.984608:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513086.984610:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b7e8 00000800:00000200:18.0:1616513086.984613:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.984616:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.984950:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513086.984961:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.984969:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513086.984971:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513086.984978:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.222@tcp1 : PUT - for me 00000400:00000200:14.0:1616513086.984986:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.222@tcp1 of length 224 into portal 26 MB=0x604a71dea09c0 00000400:00000200:14.0:1616513086.984993:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.222@tcp1 of length 224/224 into md 0xf97cbd89 [1] + 0 00000400:00000200:14.0:1616513086.984998:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788770 00000400:00000200:14.0:1616513086.985000:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513086.985003:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513086.985007:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788770 00000800:00000200:14.0:1616513086.985010:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.985012:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.985014:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513086.985018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513086.985027:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.985033:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513086.985036:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513086.985041:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.222@tcp1 : PUT - for me 00000400:00000200:18.0:1616513086.985051:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.222@tcp1 of length 224 into portal 12 MB=0x604a71dea0ac0 00000400:00000200:18.0:1616513086.985057:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.222@tcp1 of length 224/224 into md 0xf964d519 [1] + 58224 00000400:00000200:18.0:1616513086.985062:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513086.985064:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513086.985070:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693965668190656 00000800:00000200:18.0:1616513086.985077:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513086.985080:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513086.985081:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1693965668190656/t0(0) o400->2d4db9f0-4722-4bc8-c46e-c560134c08e0@10.31.140.222@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513086.985082:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513086.985087:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693965668190912 00010000:00000200:22.0:1616513086.985094:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645ba880 x1693965668190912/t0(0) o400->3a5d54ca-5be6-ea27-1859-38f1474ec1fc@10.31.140.222@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513086.985096:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693965668190656, offset 192 00000400:00000200:0.0:1616513086.985102:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.222@tcp1 00000100:00000200:22.0:1616513086.985105:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693965668190912, offset 192 00000400:00000200:0.0:1616513086.985109:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.222@tcp1 routed destination 00000400:00000200:22.0:1616513086.985110:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.222@tcp1 00000400:00000200:22.0:1616513086.985116:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.222@tcp1 routed destination 00000400:00000200:0.0:1616513086.985125:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.222@tcp1(10.31.140.222@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513086.985129:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.222@tcp1(10.31.140.222@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513086.985131:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513086.985137:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513086.985137:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.985140:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513086.985142:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513086.985142:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513086.985144:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513086.985146:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513086.985147:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513086.985149:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513086.985153:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513086.985154:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513086.985154:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513086.985162:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513086.985164:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513086.985166:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513086.985168:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513086.985171:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513086.985171:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513086.985173:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513086.985174:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513086.985177:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a198 00000400:00000200:20.0:1616513086.985180:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513086.985181:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513086.985186:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:20.0:1616513086.985189:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.985192:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513086.997858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513086.997913:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513086.997922:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513086.997930:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.37@tcp1 : PUT - for me 00000400:00000200:26.0:1616513086.997939:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.37@tcp1 of length 224 into portal 26 MB=0x5ffd50648e040 00000400:00000200:26.0:1616513086.997945:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.37@tcp1 of length 224/224 into md 0xf97cbdc9 [1] + 0 00000400:00000200:26.0:1616513086.997949:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788a18 00000400:00000200:26.0:1616513086.997952:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513086.997954:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513086.997958:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788a18 00000800:00000200:26.0:1616513086.997961:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513086.997961:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513086.997964:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513086.997965:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513086.997968:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513086.998023:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665282109504 00000800:00000200:12.0:1616513086.998029:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:0.0:1616513086.998034:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688665282109504/t0(0) o400->488b8d35-bd42-6eb7-5b23-2b6c08fdfb61@10.31.136.37@tcp1:43/0 lens 224/224 e 0 to 0 dl 1616513178 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513086.998038:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513086.998047:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.37@tcp1 : PUT - for me 00000100:00000200:0.0:1616513086.998048:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665282109504, offset 192 00000400:00000200:0.0:1616513086.998055:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.37@tcp1 00000400:00000200:12.0:1616513086.998056:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.37@tcp1 of length 224 into portal 12 MB=0x5ffd50648e180 00000400:00000200:0.0:1616513086.998062:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.37@tcp1 routed destination 00000400:00000200:12.0:1616513086.998063:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.37@tcp1 of length 224/224 into md 0x865bbec1 [1] + 87808 00000400:00000200:12.0:1616513086.998068:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513086.998070:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513086.998078:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.37@tcp1(10.31.136.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513086.998083:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513086.998084:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513086.998086:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513086.998088:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513086.998090:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513086.998092:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513086.998093:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.998095:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513086.998100:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513086.998105:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513086.998106:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513086.998117:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513086.998124:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513086.998127:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513086.998133:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513086.998139:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000100:00000200:4.0:1616513086.998140:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665282109824 00000800:00000200:20.0:1616513086.998142:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513086.998145:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:4.0:1616513086.998153:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d205580 x1688665282109824/t0(0) o400->44f69b52-e74d-37d6-595f-73174a5b948c@10.31.136.37@tcp1:712/0 lens 224/224 e 0 to 0 dl 1616513092 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513086.998167:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665282109824, offset 192 00000400:00000200:4.0:1616513086.998173:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.37@tcp1 00000400:00000200:4.0:1616513086.998181:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.37@tcp1 routed destination 00000400:00000200:4.0:1616513086.998196:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.37@tcp1(10.31.136.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513086.998203:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513086.998209:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513086.998211:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513086.998214:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513086.998218:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513086.998224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513086.998280:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513086.998287:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513086.998290:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513086.998297:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513086.998303:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb188 00000800:00000200:14.0:1616513086.998307:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513086.998309:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.015917:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.015972:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.015982:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513087.015991:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.224@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.016001:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.224@tcp1 of length 224 into portal 26 MB=0x5ffd3d4b773c0 00000400:00000200:28.0:1616513087.016008:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.224@tcp1 of length 224/224 into md 0xf97cbdd9 [1] + 0 00000400:00000200:28.0:1616513087.016013:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789870 00000400:00000200:28.0:1616513087.016016:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.016019:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513087.016023:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789870 00000800:00000200:12.0:1616513087.016026:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.016027:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.016029:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.016033:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.016036:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.016037:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513087.016045:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.98@tcp1 : PUT - for me 00000400:00000200:12.0:1616513087.016055:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.98@tcp1 of length 224 into portal 26 MB=0x5ffdd124c1900 00000400:00000200:12.0:1616513087.016062:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.98@tcp1 of length 224/224 into md 0xf97cbe19 [1] + 0 00000400:00000200:12.0:1616513087.016067:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789540 00000400:00000200:12.0:1616513087.016070:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513087.016072:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513087.016078:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789540 00000800:00000200:12.0:1616513087.016081:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.016084:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.016086:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513087.016094:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660155528128 00010000:00000200:0.0:1616513087.016104:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688660155528128/t0(0) o400->1fa238a9-04b3-f19d-0114-c050f5876079@10.31.128.224@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513087.016118:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660155528128, offset 192 00000400:00000200:0.0:1616513087.016125:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.224@tcp1 00000100:00000200:24.0:1616513087.016130:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699843385600 00000400:00000200:0.0:1616513087.016133:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.224@tcp1 routed destination 00010000:00000200:24.0:1616513087.016141:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688699843385600/t0(0) o400->95d140dc-6186-f8f2-9d9c-cc3823e29c41@10.31.141.98@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513087.016147:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.224@tcp1(10.31.128.224@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513087.016153:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513087.016156:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699843385600, offset 192 00000800:00000200:0.0:1616513087.016158:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513087.016161:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:24.0:1616513087.016162:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.98@tcp1 00000800:00000200:0.0:1616513087.016163:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.016168:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513087.016170:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.98@tcp1 routed destination 00000800:00000200:11.2:1616513087.016172:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513087.016174:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513087.016191:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.98@tcp1(10.31.141.98@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513087.016196:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513087.016198:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513087.016201:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.016202:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513087.016203:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513087.016205:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513087.016206:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513087.016208:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513087.016208:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.016211:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000800:00000200:24.0:1616513087.016213:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513087.016214:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.016219:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513087.016222:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.016227:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513087.016235:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.016241:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513087.016245:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513087.016249:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.224@tcp1 : PUT - for me 00000400:00000200:12.0:1616513087.016255:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.224@tcp1 of length 224 into portal 12 MB=0x5ffd3d4b77880 00000400:00000200:12.0:1616513087.016261:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.224@tcp1 of length 224/224 into md 0x865bbec1 [1] + 88032 00000400:00000200:12.0:1616513087.016265:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513087.016267:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513087.016274:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.016276:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.016277:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513087.016283:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.016290:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:4.0:1616513087.016294:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660155529344 00000400:00000200:26.0:1616513087.016295:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.016302:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:4.0:1616513087.016306:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d201f80 x1688660155529344/t0(0) o400->ed0c418f-0a80-b11b-9fa0-01d0af60854d@10.31.128.224@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:26.0:1616513087.016307:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4f68 00000800:00000200:26.0:1616513087.016310:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.016313:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513087.016320:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660155529344, offset 192 00000400:00000200:4.0:1616513087.016326:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.224@tcp1 00000400:00000200:4.0:1616513087.016333:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.224@tcp1 routed destination 00000400:00000200:4.0:1616513087.016347:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.224@tcp1(10.31.128.224@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.016354:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513087.016360:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.016362:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513087.016364:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513087.016369:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.016382:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513087.016384:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513087.016391:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.016397:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.016399:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.016401:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513087.016405:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.016408:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba5d8 00000800:00000200:28.0:1616513087.016411:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513087.016456:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.016465:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.016473:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.98@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.016481:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.98@tcp1 of length 224 into portal 12 MB=0x5ffdd124c2980 00000400:00000200:14.0:1616513087.016487:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.98@tcp1 of length 224/224 into md 0xf964d519 [1] + 58448 00000400:00000200:14.0:1616513087.016492:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.016495:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.016501:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.016504:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.016506:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.016509:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.016565:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699843389824 00010000:00000200:22.0:1616513087.016577:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7a400 x1688699843389824/t0(0) o400->35e62820-0481-8fec-c9c8-0991fe31d751@10.31.141.98@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.016590:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699843389824, offset 192 00000400:00000200:22.0:1616513087.016596:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.98@tcp1 00000400:00000200:22.0:1616513087.016603:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.98@tcp1 routed destination 00000400:00000200:22.0:1616513087.016617:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.98@tcp1(10.31.141.98@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.016622:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.016628:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.016630:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.016632:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.016637:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.016641:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.016651:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.016654:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.016656:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.016660:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.016662:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.016665:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b540 00000800:00000200:26.0:1616513087.016667:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.023713:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.023768:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.023777:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.023786:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.227@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.023796:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.227@tcp1 of length 224 into portal 26 MB=0x603e0e96f9200 00000400:00000200:2.0:1616513087.023803:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.227@tcp1 of length 224/224 into md 0xf97cbe29 [1] + 0 00000400:00000200:2.0:1616513087.023807:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788908 00000400:00000200:2.0:1616513087.023810:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.023813:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.023817:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788908 00000800:00000200:2.0:1616513087.023820:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.023821:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.023823:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.023825:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.023831:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513087.023839:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114384224768 00010000:00000200:24.0:1616513087.023849:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1693114384224768/t0(0) o400->b216fe55-6f70-0a00-58fe-3f8c8e05cc86@10.31.136.227@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.023863:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114384224768, offset 192 00000400:00000200:24.0:1616513087.023871:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.227@tcp1 00000400:00000200:24.0:1616513087.023879:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.227@tcp1 routed destination 00000400:00000200:24.0:1616513087.023894:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.227@tcp1(10.31.136.227@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.023900:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513087.023906:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513087.023906:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.023909:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.023911:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513087.023915:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513087.023916:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000100:00000200:24.0:1616513087.023922:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513087.023924:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.227@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.023933:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.227@tcp1 of length 224 into portal 12 MB=0x603e0e96f9280 00000400:00000200:20.0:1616513087.023940:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.227@tcp1 of length 224/224 into md 0x865bbec1 [1] + 88256 00000400:00000200:20.0:1616513087.023945:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.023947:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513087.023955:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513087.023957:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513087.023960:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513087.023967:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513087.023968:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513087.023970:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513087.023970:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513087.023977:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513087.023983:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb58f8 00000800:00000200:12.0:1616513087.023986:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513087.024019:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114384224896 00010000:00000200:4.0:1616513087.024031:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad1f80 x1693114384224896/t0(0) o400->4d74ff48-7335-afaf-9f90-a82453ee2475@10.31.136.227@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.024046:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114384224896, offset 192 00000400:00000200:4.0:1616513087.024052:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.227@tcp1 00000400:00000200:4.0:1616513087.024060:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.227@tcp1 routed destination 00000400:00000200:4.0:1616513087.024075:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.227@tcp1(10.31.136.227@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.024082:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.024087:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.024090:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.024092:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.024097:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.024102:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.024155:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.024162:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.024165:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.024171:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.024175:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb980 00000800:00000200:18.0:1616513087.024179:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.024182:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.040314:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.040371:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.040381:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.040390:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.197@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.040399:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.197@tcp1 of length 224 into portal 26 MB=0x5ffdc34c46d80 00000400:00000200:14.0:1616513087.040406:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.197@tcp1 of length 224/224 into md 0xf97cbe49 [1] + 0 00000400:00000200:14.0:1616513087.040411:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788110 00000400:00000200:14.0:1616513087.040414:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.040416:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.040420:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788110 00000800:00000200:14.0:1616513087.040424:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.040425:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.040427:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.040431:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.040466:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513087.040485:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696126729600 00010000:00000200:24.0:1616513087.040496:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688696126729600/t0(0) o400->32c778fb-3a9e-5df5-5612-430abcc8a67f@10.31.141.197@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.040510:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696126729600, offset 192 00000400:00000200:24.0:1616513087.040517:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.197@tcp1 00000400:00000200:24.0:1616513087.040524:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.197@tcp1 routed destination 00000400:00000200:24.0:1616513087.040539:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.197@tcp1(10.31.141.197@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513087.040539:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513087.040546:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513087.040548:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513087.040551:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.040554:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.040556:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513087.040557:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.197@tcp1 : PUT - for me 00000800:00000200:24.0:1616513087.040561:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513087.040566:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.197@tcp1 of length 224 into portal 12 MB=0x5ffdc34c46f80 00000100:00000200:24.0:1616513087.040567:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.040568:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513087.040572:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.197@tcp1 of length 224/224 into md 0xf964d519 [1] + 58672 00000400:00000200:18.0:1616513087.040577:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.040580:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513087.040587:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.040590:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.040592:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.040596:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513087.040640:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.040647:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.040650:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.040656:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:22.0:1616513087.040656:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696126730112 00000400:00000200:28.0:1616513087.040662:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4770 00000800:00000200:28.0:1616513087.040665:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.040668:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:22.0:1616513087.040669:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bc380 x1688696126730112/t0(0) o400->c7da4628-697d-0320-5ebd-180be8e9f7a4@10.31.141.197@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.040683:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696126730112, offset 192 00000400:00000200:22.0:1616513087.040690:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.197@tcp1 00000400:00000200:22.0:1616513087.040697:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.197@tcp1 routed destination 00000400:00000200:22.0:1616513087.040712:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.197@tcp1(10.31.141.197@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.040719:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.040724:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.040727:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.040730:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.040734:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.040739:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.040749:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.040755:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.040757:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.040760:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.040764:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.040768:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a330 00000800:00000200:2.0:1616513087.040772:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.044347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.044403:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.044413:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.044422:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.19@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.044432:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.19@tcp1 of length 224 into portal 12 MB=0x5ffd36f888180 00000400:00000200:0.0:1616513087.044451:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.19@tcp1 of length 224/224 into md 0x865bbec1 [1] + 88480 00000400:00000200:0.0:1616513087.044456:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513087.044457:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513087.044458:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513087.044465:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513087.044468:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513087.044468:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513087.044470:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.044473:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513087.044477:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.19@tcp1 : PUT - for me 00000400:00000200:12.0:1616513087.044486:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.19@tcp1 of length 224 into portal 26 MB=0x5ffd36f888100 00000400:00000200:12.0:1616513087.044492:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.19@tcp1 of length 224/224 into md 0xf97cbe59 [1] + 0 00000400:00000200:12.0:1616513087.044497:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fccb28 00000400:00000200:12.0:1616513087.044499:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513087.044502:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513087.044506:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccb28 00000800:00000200:12.0:1616513087.044510:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.044513:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.044514:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513087.044529:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658457952640 00010000:00000200:4.0:1616513087.044541:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86d100 x1688658457952640/t0(0) o400->b2d96ac6-76b0-ef3f-7ec5-4beef2cb4630@10.31.138.19@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.044555:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658457952640, offset 192 00000400:00000200:4.0:1616513087.044562:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.19@tcp1 00000100:00000200:24.0:1616513087.044568:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658457952512 00000400:00000200:4.0:1616513087.044569:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.19@tcp1 routed destination 00010000:00000200:24.0:1616513087.044578:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688658457952512/t0(0) o400->3cbe9736-5896-c642-34e5-ee5f5afe91fa@10.31.138.19@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:4.0:1616513087.044584:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.19@tcp1(10.31.138.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513087.044591:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658457952512, offset 192 00000800:00000200:4.0:1616513087.044591:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513087.044596:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513087.044597:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.19@tcp1 00000800:00000200:4.0:1616513087.044599:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513087.044601:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513087.044604:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.19@tcp1 routed destination 00000800:00000200:4.0:1616513087.044606:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.044611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513087.044618:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.19@tcp1(10.31.138.19@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.044628:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513087.044633:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.044635:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513087.044637:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513087.044642:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.044646:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513087.044648:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513087.044664:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.044671:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.044674:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.044675:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.044680:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513087.044684:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb6d8 00000800:00000200:20.0:1616513087.044688:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513087.044698:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.044704:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.044707:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.044713:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.044718:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5430 00000800:00000200:26.0:1616513087.044721:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.044724:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.045899:0:50210:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.045953:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.045963:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.045971:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.236@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.045981:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.236@tcp1 of length 224 into portal 26 MB=0x604a466e08000 00000400:00000200:14.0:1616513087.045988:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.236@tcp1 of length 224/224 into md 0xf97cbe89 [1] + 0 00000400:00000200:14.0:1616513087.045995:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc908 00000400:00000200:14.0:1616513087.045997:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.046000:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.046004:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc908 00000800:00000200:14.0:1616513087.046007:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.046008:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.046010:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.046015:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.046018:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.046019:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.046027:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.236@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.046036:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.236@tcp1 of length 224 into portal 12 MB=0x604a466e08280 00000400:00000200:2.0:1616513087.046042:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.236@tcp1 of length 224/224 into md 0xf964d519 [1] + 58896 00000400:00000200:2.0:1616513087.046048:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.046050:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513087.046057:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.046060:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.046062:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513087.046068:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954007400448 00010000:00000200:24.0:1616513087.046079:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1693954007400448/t0(0) o400->454cce3d-b7da-d47d-d987-3192cb4097b7@10.31.140.236@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.046093:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954007400448, offset 192 00000400:00000200:24.0:1616513087.046099:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.236@tcp1 00000400:00000200:24.0:1616513087.046107:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.236@tcp1 routed destination 00000100:00000200:22.0:1616513087.046127:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954007401088 00000400:00000200:24.0:1616513087.046131:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.236@tcp1(10.31.140.236@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.046137:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:22.0:1616513087.046139:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebe780 x1693954007401088/t0(0) o400->93716619-c802-b42a-989a-6d84a8b229af@10.31.140.236@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513087.046142:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.046145:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.046147:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.046151:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513087.046153:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954007401088, offset 192 00000800:00000200:11.2:1616513087.046156:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513087.046157:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513087.046160:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.236@tcp1 00000400:00000200:22.0:1616513087.046167:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.236@tcp1 routed destination 00000400:00000200:22.0:1616513087.046182:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.236@tcp1(10.31.140.236@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.046188:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.046194:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.046197:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.046199:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.046204:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.046208:0:50210:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513087.046211:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.046217:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.046220:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.046227:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.046232:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5078 00000800:00000200:28.0:1616513087.046236:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.046239:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513087.046261:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.046268:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.046271:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.046273:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.046278:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.046282:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b188 00000800:00000200:18.0:1616513087.046285:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.048058:0:50210:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.048113:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.048122:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.048131:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.237@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.048140:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.237@tcp1 of length 224 into portal 26 MB=0x5ffd360abcac0 00000400:00000200:2.0:1616513087.048147:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.237@tcp1 of length 224/224 into md 0xf97cbe99 [1] + 0 00000400:00000200:2.0:1616513087.048152:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04d48 00000400:00000200:2.0:1616513087.048154:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.048157:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.048161:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000800:00000200:2.0:1616513087.048164:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.048167:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.048169:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.048171:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.048181:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513087.048188:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.048190:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.237@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.048199:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.237@tcp1 of length 224 into portal 12 MB=0x5ffd360abcb40 00000400:00000200:26.0:1616513087.048205:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.237@tcp1 of length 224/224 into md 0xf964d519 [1] + 59120 00000400:00000200:26.0:1616513087.048210:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.048213:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513087.048220:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513087.048222:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658208606912 00000800:00000200:26.0:1616513087.048223:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.048225:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513087.048232:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688658208606912/t0(0) o400->47912d1f-eaeb-c523-12e0-ddd38ab7bc93@10.31.132.237@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.048246:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658208606912, offset 192 00000400:00000200:24.0:1616513087.048252:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.237@tcp1 00000400:00000200:24.0:1616513087.048258:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.237@tcp1 routed destination 00000400:00000200:24.0:1616513087.048272:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.237@tcp1(10.31.132.237@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.048278:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513087.048283:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:22.0:1616513087.048283:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658208607040 00000800:00000200:24.0:1616513087.048286:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.048294:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513087.048295:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dec00 x1688658208607040/t0(0) o400->9c309e38-69d2-c992-9116-06eb502eff68@10.31.132.237@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513087.048298:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513087.048304:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.048304:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513087.048309:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658208607040, offset 192 00000400:00000200:22.0:1616513087.048316:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.237@tcp1 00000800:00000200:0.0:1616513087.048317:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513087.048323:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.237@tcp1 routed destination 00000800:00000200:0.0:1616513087.048331:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.048334:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513087.048337:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.237@tcp1(10.31.132.237@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513087.048341:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513087.048344:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513087.048348:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5c28 00000800:00000200:22.0:1616513087.048349:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513087.048351:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513087.048352:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.048354:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513087.048354:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513087.048359:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.048365:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.048426:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.048442:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.048446:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.048449:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.048456:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.048460:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a990 00000800:00000200:18.0:1616513087.048464:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.076193:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.076251:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.076261:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513087.076271:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.195@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.076280:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.195@tcp1 of length 224 into portal 26 MB=0x5ffdbb1b63f00 00000400:00000200:28.0:1616513087.076288:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.195@tcp1 of length 224/224 into md 0xf97cbeb9 [1] + 0 00000400:00000200:28.0:1616513087.076293:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05078 00000400:00000200:28.0:1616513087.076296:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.076298:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513087.076303:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05078 00000800:00000200:12.0:1616513087.076305:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.076306:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.076309:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513087.076312:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.076315:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.076316:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513087.076325:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.195@tcp1 : PUT - for me 00000400:00000200:12.0:1616513087.076334:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.195@tcp1 of length 224 into portal 12 MB=0x5ffdbb1b641c0 00000400:00000200:12.0:1616513087.076340:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.195@tcp1 of length 224/224 into md 0x865bbec1 [1] + 88704 00000400:00000200:12.0:1616513087.076345:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513087.076348:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513087.076354:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.076358:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.076360:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513087.076367:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693927984896 00010000:00000200:24.0:1616513087.076378:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688693927984896/t0(0) o400->36136999-9e01-4041-56b1-c140597fa30e@10.31.141.195@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.076391:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693927984896, offset 192 00000400:00000200:24.0:1616513087.076396:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.195@tcp1 00000400:00000200:24.0:1616513087.076403:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.195@tcp1 routed destination 00000400:00000200:24.0:1616513087.076418:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.195@tcp1(10.31.141.195@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513087.076418:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693927985600 00000800:00000200:24.0:1616513087.076424:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513087.076430:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:4.0:1616513087.076431:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d202880 x1688693927985600/t0(0) o400->34d6290e-d89f-a7d8-1ebe-960a13eac238@10.31.141.195@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513087.076433:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.076446:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.076451:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.076457:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513087.076457:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693927985600, offset 192 00000100:00000200:24.0:1616513087.076459:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:4.0:1616513087.076464:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.195@tcp1 00000400:00000200:4.0:1616513087.076472:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.195@tcp1 routed destination 00000400:00000200:4.0:1616513087.076487:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.195@tcp1(10.31.141.195@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.076493:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.076500:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.076502:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.076505:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513087.076510:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513087.076510:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.076516:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513087.076517:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.076521:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.076522:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513087.076528:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513087.076534:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4ff0 00000800:00000200:20.0:1616513087.076537:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513087.076558:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.076571:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.076580:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513087.076582:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.076589:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.076593:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb650 00000800:00000200:14.0:1616513087.076597:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.083163:0:50214:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.083221:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.083231:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.083240:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.149@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.083249:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.149@tcp1 of length 224 into portal 26 MB=0x603fce5478040 00000400:00000200:2.0:1616513087.083258:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.149@tcp1 of length 224/224 into md 0xf97cbed9 [1] + 0 00000400:00000200:2.0:1616513087.083263:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05c28 00000400:00000200:2.0:1616513087.083266:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.083268:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.083272:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05c28 00000800:00000200:2.0:1616513087.083275:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.083276:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.083278:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.083283:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.083286:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513087.083286:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.083295:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.149@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.083304:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.149@tcp1 of length 224 into portal 12 MB=0x603fce54780c0 00000400:00000200:18.0:1616513087.083310:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.149@tcp1 of length 224/224 into md 0xf964d519 [1] + 59344 00000400:00000200:18.0:1616513087.083315:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.083318:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513087.083325:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.083329:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.083330:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513087.083336:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234573574208 00010000:00000200:24.0:1616513087.083347:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1693234573574208/t0(0) o400->4dbcef95-dd21-6f59-c1f7-56469d982836@10.31.138.149@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.083361:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234573574208, offset 192 00000400:00000200:24.0:1616513087.083367:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.149@tcp1 00000400:00000200:24.0:1616513087.083375:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.149@tcp1 routed destination 00000400:00000200:24.0:1616513087.083390:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.149@tcp1(10.31.138.149@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.083396:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:22.0:1616513087.083399:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234573574336 00000800:00000200:24.0:1616513087.083402:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.083404:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.083406:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513087.083411:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b9680 x1693234573574336/t0(0) o400->4975dc51-e607-04a0-5995-9b65d0250b03@10.31.138.149@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513087.083412:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.083418:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513087.083419:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513087.083426:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234573574336, offset 192 00000400:00000200:22.0:1616513087.083432:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.149@tcp1 00000400:00000200:22.0:1616513087.083454:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.149@tcp1 routed destination 00000400:00000200:22.0:1616513087.083471:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.149@tcp1(10.31.138.149@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.083471:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513087.083477:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.083478:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513087.083482:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513087.083484:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.083486:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.083488:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513087.083489:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513087.083493:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513087.083494:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4c38 00000800:00000200:7.2:1616513087.083497:0:50214:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.083498:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.083501:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513087.083507:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.083511:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.083513:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513087.083514:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513087.083518:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.083521:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a3b8 00000800:00000200:14.0:1616513087.083524:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.093758:0:50215:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513087.093766:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513087.093814:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.093824:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:28.0:1616513087.093834:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513087.093834:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.35@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.093842:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.35@tcp1 of length 224 into portal 12 MB=0x5ffe2190ebcc0 00000800:00000200:28.0:1616513087.093845:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513087.093849:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.35@tcp1 of length 224/224 into md 0xf964d519 [1] + 59568 00000400:00000200:2.0:1616513087.093854:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.093855:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.35@tcp1 : PUT - for me 00000100:00000200:2.0:1616513087.093857:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513087.093863:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513087.093864:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.35@tcp1 of length 224 into portal 26 MB=0x5ffe2190ebb80 00000800:00000200:2.0:1616513087.093866:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.093868:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513087.093870:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.35@tcp1 of length 224/224 into md 0xf97cbef9 [1] + 0 00000800:00000200:2.0:1616513087.093871:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513087.093875:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04088 00000400:00000200:28.0:1616513087.093878:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.093880:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513087.093885:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04088 00000800:00000200:28.0:1616513087.093888:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.093891:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.093893:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.093896:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513087.093933:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688721431641280 00010000:00000200:22.0:1616513087.093946:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eba400 x1688721431641280/t0(0) o400->4ce92fa1-bfcc-f3fb-603a-8d8443539499@10.31.141.35@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513087.093954:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688721431640960 00000100:00000200:22.0:1616513087.093960:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688721431641280, offset 192 00010000:00000200:24.0:1616513087.093965:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688721431640960/t0(0) o400->97f68ef0-4d2b-b91b-bb86-cd5544ec0112@10.31.141.35@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513087.093967:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.35@tcp1 00000400:00000200:22.0:1616513087.093974:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.35@tcp1 routed destination 00000100:00000200:24.0:1616513087.093981:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688721431640960, offset 192 00000400:00000200:24.0:1616513087.093987:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.35@tcp1 00000400:00000200:22.0:1616513087.093989:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.35@tcp1(10.31.141.35@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.093996:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513087.093998:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.35@tcp1 routed destination 00000800:00000200:22.0:1616513087.094002:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.094005:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513087.094007:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513087.094012:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513087.094013:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.35@tcp1(10.31.141.35@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513087.094016:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.094020:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513087.094025:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.094028:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513087.094030:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513087.094035:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.094040:0:50215:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513087.094042:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513087.094072:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513087.094079:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513087.094082:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513087.094088:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513087.094092:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4add0 00000800:00000200:26.0:1616513087.094096:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513087.094096:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.094099:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513087.094103:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.094105:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.094113:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.094119:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5cb0 00000800:00000200:26.0:1616513087.094122:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.094125:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.112101:0:50217:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.112158:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.112168:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.112177:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.156@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.112187:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.156@tcp1 of length 224 into portal 26 MB=0x5ffd59403a2c0 00000400:00000200:2.0:1616513087.112193:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.156@tcp1 of length 224/224 into md 0xf97cbf19 [1] + 0 00000400:00000200:2.0:1616513087.112198:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05430 00000400:00000200:2.0:1616513087.112201:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.112204:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.112208:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05430 00000800:00000200:2.0:1616513087.112211:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.112214:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.112214:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.112218:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513087.112270:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667659936448 00010000:00000200:24.0:1616513087.112281:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688667659936448/t0(0) o400->ccabf265-07cc-e5d9-dc4c-a0b06b0082fb@10.31.132.156@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.112294:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667659936448, offset 192 00000400:00000200:24.0:1616513087.112300:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.156@tcp1 00000400:00000200:24.0:1616513087.112307:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.156@tcp1 routed destination 00000400:00000200:24.0:1616513087.112321:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.156@tcp1(10.31.132.156@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.112327:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513087.112332:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.112335:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.112337:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.112341:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513087.112348:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.112348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.112402:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.112408:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.112411:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.112418:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.112423:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb54b8 00000800:00000200:28.0:1616513087.112427:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.112430:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.112479:0:50217:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513087.112534:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.112543:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513087.112550:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.156@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.112558:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.156@tcp1 of length 224 into portal 12 MB=0x5ffd59403a440 00000400:00000200:26.0:1616513087.112565:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.156@tcp1 of length 224/224 into md 0xf964d519 [1] + 59792 00000400:00000200:26.0:1616513087.112570:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.112572:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513087.112579:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.112582:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.112584:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.112587:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.112654:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667659936832 00010000:00000200:22.0:1616513087.112665:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647df500 x1688667659936832/t0(0) o400->7c5a1dfa-7e00-568c-3bae-46760ad4cf19@10.31.132.156@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.112679:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667659936832, offset 192 00000400:00000200:22.0:1616513087.112685:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.156@tcp1 00000400:00000200:22.0:1616513087.112691:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.156@tcp1 routed destination 00000400:00000200:22.0:1616513087.112705:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.156@tcp1(10.31.132.156@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.112711:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.112719:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.112722:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.112726:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.112730:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.112735:0:50217:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.112755:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.112761:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.112763:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513087.112765:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.112770:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.112774:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4acc0 00000800:00000200:18.0:1616513087.112777:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.135852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.135909:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.135919:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.135928:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.192@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.135939:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.192@tcp1 of length 224 into portal 26 MB=0x5ffd34d3b2bc0 00000400:00000200:14.0:1616513087.135945:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.192@tcp1 of length 224/224 into md 0xf97cbf29 [1] + 0 00000400:00000200:14.0:1616513087.135950:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb047f8 00000400:00000200:14.0:1616513087.135953:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.135956:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.135960:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb047f8 00000800:00000200:26.0:1616513087.135962:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.135963:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.135966:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513087.135969:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.135973:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513087.135975:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.135982:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.192@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.135991:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.192@tcp1 of length 224 into portal 12 MB=0x5ffd34d3b2c40 00000400:00000200:26.0:1616513087.135998:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.192@tcp1 of length 224/224 into md 0xf964d519 [1] + 60016 00000400:00000200:26.0:1616513087.136004:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.136006:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513087.136014:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.136017:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.136019:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513087.136026:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657882459072 00010000:00000200:24.0:1616513087.136038:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688657882459072/t0(0) o400->2bb220a9-f561-4c65-be24-c8613277b3a0@10.31.138.192@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.136052:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657882459072, offset 192 00000400:00000200:24.0:1616513087.136059:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.192@tcp1 00000400:00000200:24.0:1616513087.136067:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.192@tcp1 routed destination 00000100:00000200:22.0:1616513087.136080:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657882459200 00000400:00000200:24.0:1616513087.136083:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.192@tcp1(10.31.138.192@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.136090:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:22.0:1616513087.136093:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9b00 x1688657882459200/t0(0) o400->0fe222fd-cea3-f3bb-2413-f9a54b5966c0@10.31.138.192@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513087.136096:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.136098:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.136100:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.136106:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513087.136108:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657882459200, offset 192 00000100:00000200:24.0:1616513087.136112:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513087.136114:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.192@tcp1 00000800:00000200:11.2:1616513087.136114:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513087.136121:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.192@tcp1 routed destination 00000400:00000200:22.0:1616513087.136136:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.192@tcp1(10.31.138.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.136143:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.136149:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.136152:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.136154:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.136159:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.136165:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.136169:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513087.136176:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513087.136179:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513087.136186:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513087.136191:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb54b8 00000800:00000200:12.0:1616513087.136195:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.136198:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513087.136220:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.136228:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.136230:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.136232:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.136237:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.136241:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bd38 00000800:00000200:18.0:1616513087.136245:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.138981:0:50221:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513087.139035:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.139045:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513087.139054:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.19@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.139063:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.19@tcp1 of length 224 into portal 26 MB=0x5ffd41109d3c0 00000400:00000200:26.0:1616513087.139070:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.19@tcp1 of length 224/224 into md 0xf97cbf39 [1] + 0 00000400:00000200:26.0:1616513087.139075:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04000 00000400:00000200:26.0:1616513087.139080:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.139082:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513087.139086:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04000 00000800:00000200:26.0:1616513087.139089:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.139090:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.139092:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.139094:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513087.139151:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661167559616 00010000:00000200:24.0:1616513087.139162:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688661167559616/t0(0) o400->b6033e1e-97b4-d764-599e-fa39ec4d9c8e@10.31.129.19@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.139176:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661167559616, offset 192 00000400:00000200:24.0:1616513087.139183:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.19@tcp1 00000400:00000200:24.0:1616513087.139190:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.19@tcp1 routed destination 00000400:00000200:24.0:1616513087.139205:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.19@tcp1(10.31.129.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.139212:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513087.139217:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.139220:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.139222:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.139226:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.139231:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513087.139233:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513087.139295:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.139301:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.139304:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.139311:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.139316:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4c38 00000800:00000200:28.0:1616513087.139319:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.139322:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.139328:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513087.139330:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.139332:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.139333:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.139336:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.139582:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.139635:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.139644:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513087.139646:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513087.139653:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.19@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.139662:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.19@tcp1 of length 224 into portal 12 MB=0x5ffd41109d780 00000400:00000200:0.0:1616513087.139668:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.19@tcp1 of length 224/224 into md 0x865bbec1 [1] + 88928 00000400:00000200:0.0:1616513087.139673:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.139676:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513087.139683:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.139686:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.139688:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513087.139712:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661167560576 00010000:00000200:4.0:1616513087.139724:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde869f80 x1688661167560576/t0(0) o400->625da412-dfdc-ac83-3bf6-ed0ee3e09263@10.31.129.19@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.139739:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661167560576, offset 192 00000400:00000200:4.0:1616513087.139745:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.19@tcp1 00000400:00000200:4.0:1616513087.139753:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.19@tcp1 routed destination 00000400:00000200:4.0:1616513087.139768:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.19@tcp1(10.31.129.19@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.139775:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.139780:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.139783:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.139785:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.139790:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.139797:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.139813:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.139819:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.139822:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.139828:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.139832:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba440 00000800:00000200:14.0:1616513087.139836:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.139839:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.156405:0:50223:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.156460:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.156470:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.156479:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.168@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.156488:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.168@tcp1 of length 224 into portal 26 MB=0x5ffddff771280 00000400:00000200:2.0:1616513087.156495:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.168@tcp1 of length 224/224 into md 0xf97cbf69 [1] + 0 00000400:00000200:2.0:1616513087.156499:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05298 00000400:00000200:2.0:1616513087.156502:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.156505:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.156515:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05298 00000800:00000200:2.0:1616513087.156518:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.156519:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.156521:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.156526:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.156529:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.156530:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513087.156538:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.168@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.156547:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.168@tcp1 of length 224 into portal 12 MB=0x5ffddff771300 00000400:00000200:26.0:1616513087.156554:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.168@tcp1 of length 224/224 into md 0xf964d519 [1] + 60240 00000400:00000200:26.0:1616513087.156559:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.156562:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513087.156570:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.156573:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.156575:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513087.156590:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703822402176 00010000:00000200:24.0:1616513087.156600:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688703822402176/t0(0) o400->62a83d81-1fd6-c0a0-a4eb-6eda4a6fe7aa@10.31.136.168@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.156613:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703822402176, offset 192 00000400:00000200:24.0:1616513087.156619:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.168@tcp1 00000400:00000200:24.0:1616513087.156626:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.168@tcp1 routed destination 00000100:00000200:22.0:1616513087.156635:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703822402304 00000400:00000200:24.0:1616513087.156640:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.168@tcp1(10.31.136.168@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.156646:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:22.0:1616513087.156648:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c697500 x1688703822402304/t0(0) o400->d177d34a-5134-9829-29af-a35a68e21490@10.31.136.168@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513087.156651:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.156655:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.156657:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.156662:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513087.156662:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703822402304, offset 192 00000100:00000200:24.0:1616513087.156668:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513087.156668:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.168@tcp1 00000800:00000200:11.2:1616513087.156668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513087.156676:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.168@tcp1 routed destination 00000400:00000200:22.0:1616513087.156690:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.168@tcp1(10.31.136.168@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.156697:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.156703:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.156706:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.156708:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.156712:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.156718:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513087.156722:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.156729:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.156731:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513087.156738:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513087.156744:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5c28 00000800:00000200:20.0:1616513087.156748:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.156751:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513087.156773:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.156781:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.156784:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513087.156785:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.156791:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.156795:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bba0 00000800:00000200:18.0:1616513087.156799:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.169121:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.169177:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.169187:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.169196:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.140@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.169207:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.140@tcp1 of length 224 into portal 26 MB=0x603e4bb8ffc00 00000400:00000200:14.0:1616513087.169214:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.140@tcp1 of length 224/224 into md 0xf97cbf79 [1] + 0 00000400:00000200:14.0:1616513087.169218:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb043b8 00000400:00000200:14.0:1616513087.169221:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.169224:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.169228:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb043b8 00000800:00000200:26.0:1616513087.169230:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.169231:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.169234:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.169236:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513087.169290:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693130794466304 00010000:00000200:24.0:1616513087.169300:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1693130794466304/t0(0) o400->a21fdc0f-71a3-a146-3087-67f02790923b@10.31.136.140@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.169313:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693130794466304, offset 192 00000400:00000200:24.0:1616513087.169319:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.140@tcp1 00000400:00000200:24.0:1616513087.169327:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.140@tcp1 routed destination 00000400:00000200:24.0:1616513087.169341:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.140@tcp1(10.31.136.140@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.169347:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513087.169352:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.169354:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.169356:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.169360:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513087.169367:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.169367:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.169422:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.169429:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.169432:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.169452:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.169457:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5430 00000800:00000200:28.0:1616513087.169461:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.169464:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.169964:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.170018:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.170028:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513087.170037:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.140@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.170046:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.140@tcp1 of length 224 into portal 12 MB=0x603e4bb8ffd40 00000400:00000200:20.0:1616513087.170052:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.140@tcp1 of length 224/224 into md 0x865bbec1 [1] + 89152 00000400:00000200:20.0:1616513087.170058:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.170060:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513087.170067:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.170070:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513087.170072:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.170074:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513087.170133:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693130794466624 00010000:00000200:4.0:1616513087.170145:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad6300 x1693130794466624/t0(0) o400->83dc1b1a-2c62-89ed-bbf5-cee51892d9f9@10.31.136.140@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.170160:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693130794466624, offset 192 00000400:00000200:4.0:1616513087.170166:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.140@tcp1 00000400:00000200:4.0:1616513087.170174:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.140@tcp1 routed destination 00000400:00000200:4.0:1616513087.170191:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.140@tcp1(10.31.136.140@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.170197:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.170203:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.170206:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.170208:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.170213:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.170218:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.170272:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.170279:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.170282:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513087.170288:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.170293:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba990 00000800:00000200:2.0:1616513087.170297:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.170299:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.170774:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513087.170828:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.170837:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513087.170846:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.42@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.170856:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.42@tcp1 of length 224 into portal 26 MB=0x5ffdcac857800 00000400:00000200:26.0:1616513087.170862:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.42@tcp1 of length 224/224 into md 0xf97cbf99 [1] + 0 00000400:00000200:26.0:1616513087.170867:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04550 00000400:00000200:26.0:1616513087.170870:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.170873:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513087.170877:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04550 00000800:00000200:26.0:1616513087.170880:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.170881:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.170883:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.170889:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.170892:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.170892:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.170901:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.42@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.170910:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.42@tcp1 of length 224 into portal 12 MB=0x5ffdcac857a80 00000400:00000200:14.0:1616513087.170917:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.42@tcp1 of length 224/224 into md 0xf964d519 [1] + 60464 00000400:00000200:14.0:1616513087.170922:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.170924:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.170931:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.170934:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.170936:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513087.170941:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698135869440 00010000:00000200:24.0:1616513087.170952:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688698135869440/t0(0) o400->8d1ede9f-74d6-6873-12b1-7f799725f3e0@10.31.142.42@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.170966:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698135869440, offset 192 00000400:00000200:24.0:1616513087.170972:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.42@tcp1 00000400:00000200:24.0:1616513087.170980:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.42@tcp1 routed destination 00000400:00000200:24.0:1616513087.170995:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.42@tcp1(10.31.142.42@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513087.170996:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698135870080 00000800:00000200:24.0:1616513087.171002:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513087.171007:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513087.171008:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7e300 x1688698135870080/t0(0) o400->3b6dbcde-804f-b0f1-4deb-0ef208c0d24c@10.31.142.42@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513087.171010:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.171012:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.171017:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513087.171022:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698135870080, offset 192 00000800:00000200:11.2:1616513087.171022:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513087.171023:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513087.171031:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.42@tcp1 00000400:00000200:22.0:1616513087.171039:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.42@tcp1 routed destination 00000400:00000200:22.0:1616513087.171054:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.42@tcp1(10.31.142.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.171060:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.171066:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.171069:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.171071:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513087.171075:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513087.171076:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.171081:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513087.171082:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.171084:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.171091:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513087.171093:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513087.171096:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb58f8 00000800:00000200:28.0:1616513087.171100:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513087.171100:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513087.171103:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513087.171103:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513087.171104:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.171110:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.171114:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b5c8 00000800:00000200:18.0:1616513087.171118:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.215765:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.215821:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.215831:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513087.215841:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.123@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.215851:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.123@tcp1 of length 224 into portal 26 MB=0x5ffd3ca62c940 00000400:00000200:20.0:1616513087.215858:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.123@tcp1 of length 224/224 into md 0xf97cbfa9 [1] + 0 00000400:00000200:20.0:1616513087.215863:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05188 00000400:00000200:20.0:1616513087.215866:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.215869:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513087.215874:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05188 00000800:00000200:12.0:1616513087.215876:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.215878:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.215881:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.215884:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513087.215941:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659982207296 00010000:00000200:24.0:1616513087.215953:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688659982207296/t0(0) o400->8eafb4da-d104-1122-c248-46f79d4d646d@10.31.136.123@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513087.215969:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688659982207296, offset 192 00000400:00000200:24.0:1616513087.215975:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.123@tcp1 00000400:00000200:24.0:1616513087.215983:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.123@tcp1 routed destination 00000400:00000200:24.0:1616513087.215999:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.123@tcp1(10.31.136.123@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513087.216006:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513087.216012:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513087.216015:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.216017:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513087.216022:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.216027:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513087.216029:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513087.216082:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.216089:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.216092:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.216099:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.216105:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4990 00000800:00000200:28.0:1616513087.216109:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.216112:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.216294:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.216353:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.216363:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.216372:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.123@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.216381:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.123@tcp1 of length 224 into portal 12 MB=0x5ffd3ca62c9c0 00000400:00000200:2.0:1616513087.216388:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.123@tcp1 of length 224/224 into md 0xf964d519 [1] + 60688 00000400:00000200:2.0:1616513087.216393:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.216396:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513087.216403:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.216406:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.216408:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.216412:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.216470:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659982207424 00010000:00000200:22.0:1616513087.216484:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9b00 x1688659982207424/t0(0) o400->2d2a7203-6a61-2349-55fb-1d879034efd9@10.31.136.123@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.216498:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688659982207424, offset 192 00000400:00000200:22.0:1616513087.216505:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.123@tcp1 00000400:00000200:22.0:1616513087.216512:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.123@tcp1 routed destination 00000400:00000200:22.0:1616513087.216528:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.123@tcp1(10.31.136.123@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.216534:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.216541:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.216544:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.216546:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.216551:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.216556:0:50230:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.216610:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.216618:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.216622:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.216629:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.216633:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b760 00000800:00000200:26.0:1616513087.216637:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.216640:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.222713:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513087.222767:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.222776:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513087.222785:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.14@tcp1 : PUT - for me 00000400:00000200:12.0:1616513087.222794:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.14@tcp1 of length 224 into portal 26 MB=0x601827c21c880 00000800:00000200:7.2:1616513087.222799:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513087.222801:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.14@tcp1 of length 224/224 into md 0xf97cbfc9 [1] + 0 00000400:00000200:12.0:1616513087.222805:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb054b8 00000400:00000200:12.0:1616513087.222808:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513087.222811:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513087.222815:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb054b8 00000800:00000200:12.0:1616513087.222818:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.222819:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513087.222821:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513087.222823:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513087.222853:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.222864:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.222874:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.171@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.222883:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.171@tcp1 of length 224 into portal 26 MB=0x5ffde41a511c0 00000100:00000200:24.0:1616513087.222884:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509800228992 00000400:00000200:14.0:1616513087.222890:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.171@tcp1 of length 224/224 into md 0xf97cbfe9 [1] + 0 00010000:00000200:24.0:1616513087.222895:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1690509800228992/t0(0) o400->3dc14f82-2dcc-0d15-f088-4338d5a4833d@10.31.129.14@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513087.222895:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05a90 00000400:00000200:14.0:1616513087.222898:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.222900:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.222905:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a90 00000800:00000200:14.0:1616513087.222908:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.222908:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:24.0:1616513087.222909:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690509800228992, offset 192 00000800:00000200:14.0:1616513087.222910:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.222915:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:24.0:1616513087.222916:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.14@tcp1 00000800:00000200:14.0:1616513087.222918:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.222919:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513087.222923:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.14@tcp1 routed destination 00000100:00000200:8.0:1616513087.222925:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704932712896 00000400:00000200:2.0:1616513087.222927:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.171@tcp1 : PUT - for me 00010000:00000200:8.0:1616513087.222936:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688704932712896/t0(0) o400->4d712076-3e2f-9234-921e-e2a6630619d2@10.31.132.171@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513087.222937:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.171@tcp1 of length 224 into portal 12 MB=0x5ffde41a51240 00000400:00000200:24.0:1616513087.222939:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.14@tcp1(10.31.129.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513087.222943:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.171@tcp1 of length 224/224 into md 0xf964d519 [1] + 60912 00000800:00000200:24.0:1616513087.222945:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513087.222948:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513087.222950:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704932712896, offset 192 00000800:00000200:24.0:1616513087.222951:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:2.0:1616513087.222951:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513087.222954:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513087.222956:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513087.222957:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.171@tcp1 00000800:00000200:2.0:1616513087.222958:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513087.222961:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513087.222962:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.222964:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.222966:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513087.222966:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.171@tcp1 routed destination 00000100:00000200:24.0:1616513087.222968:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:8.0:1616513087.222981:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.171@tcp1(10.31.132.171@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.222988:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513087.222993:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.222996:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513087.222998:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513087.223003:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.223007:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513087.223010:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513087.223024:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.223030:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.223033:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:22.0:1616513087.223039:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704932713024 00000400:00000200:28.0:1616513087.223040:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.223045:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5870 00000800:00000200:28.0:1616513087.223049:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513087.223051:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c8d80 x1688704932713024/t0(0) o400->020c9fa5-a237-676e-467e-5d359681941e@10.31.132.171@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513087.223052:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513087.223060:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:22.0:1616513087.223065:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704932713024, offset 192 00000800:00000200:18.0:1616513087.223067:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.223070:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513087.223071:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.171@tcp1 00000800:00000200:14.0:1616513087.223071:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.223077:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513087.223079:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.171@tcp1 routed destination 00000400:00000200:18.0:1616513087.223082:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f707f8 00000800:00000200:18.0:1616513087.223086:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:22.0:1616513087.223094:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.171@tcp1(10.31.132.171@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.223101:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513087.223107:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.223111:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513087.223113:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513087.223118:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.223123:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.223134:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.223139:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.223141:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513087.223143:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.223148:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513087.223152:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bc28 00000800:00000200:20.0:1616513087.223156:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.223947:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.223957:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.223963:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513087.223967:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513087.223971:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.14@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.223979:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.14@tcp1 of length 224 into portal 12 MB=0x601827c21cd40 00000400:00000200:28.0:1616513087.223985:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.14@tcp1 of length 224/224 into md 0x865bbec1 [1] + 89376 00000400:00000200:28.0:1616513087.223990:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.223992:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513087.223998:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.224001:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.224004:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513087.224021:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509800230208 00010000:00000200:4.0:1616513087.224033:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa2400 x1690509800230208/t0(0) o400->59b04c51-cd67-f4f4-8c5e-228609ddeba8@10.31.129.14@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.224047:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690509800230208, offset 192 00000400:00000200:4.0:1616513087.224053:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.14@tcp1 00000400:00000200:4.0:1616513087.224061:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.14@tcp1 routed destination 00000400:00000200:4.0:1616513087.224076:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.14@tcp1(10.31.129.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.224083:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.224089:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.224092:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.224094:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.224099:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.224103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.224179:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.224185:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.224188:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.224195:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.224198:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb078 00000800:00000200:26.0:1616513087.224203:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.224205:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.249639:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513087.249696:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.249706:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513087.249714:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.238@tcp1 : PUT - for me 00000400:00000200:12.0:1616513087.249723:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.238@tcp1 of length 224 into portal 26 MB=0x5ffd57a41d9c0 00000400:00000200:12.0:1616513087.249730:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.238@tcp1 of length 224/224 into md 0xf97cbff9 [1] + 0 00000400:00000200:12.0:1616513087.249735:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04c38 00000400:00000200:12.0:1616513087.249738:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513087.249740:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513087.249745:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04c38 00000800:00000200:12.0:1616513087.249748:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.249751:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513087.249751:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513087.249755:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.249764:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667227806144 00010000:00000200:8.0:1616513087.249773:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688667227806144/t0(0) o400->31d9d83e-5033-a651-9d5c-afd22abd8c4d@10.31.131.238@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.249786:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667227806144, offset 192 00000400:00000200:8.0:1616513087.249791:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.238@tcp1 00000400:00000200:8.0:1616513087.249798:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.238@tcp1 routed destination 00000400:00000200:8.0:1616513087.249812:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.238@tcp1(10.31.131.238@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.249818:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.249823:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.249826:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.249827:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.249832:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.249837:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.249838:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513087.249842:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513087.249891:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.249897:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.249900:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.249907:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513087.249909:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513087.249913:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71078 00000800:00000200:0.0:1616513087.249916:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513087.249918:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513087.249919:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513087.249927:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.238@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.249935:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.238@tcp1 of length 224 into portal 12 MB=0x5ffd57a41dac0 00000400:00000200:14.0:1616513087.249941:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.238@tcp1 of length 224/224 into md 0xf964d519 [1] + 61136 00000400:00000200:14.0:1616513087.249946:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.249949:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.249955:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.249958:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.249960:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.249963:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.250021:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667227806400 00010000:00000200:22.0:1616513087.250034:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a78900 x1688667227806400/t0(0) o400->97f59a00-9054-2f81-4198-b8f481486bfc@10.31.131.238@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.250048:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667227806400, offset 192 00000400:00000200:22.0:1616513087.250054:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.238@tcp1 00000400:00000200:22.0:1616513087.250061:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.238@tcp1 routed destination 00000400:00000200:22.0:1616513087.250076:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.238@tcp1(10.31.131.238@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.250082:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.250088:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.250091:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.250093:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.250098:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.250103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.250154:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.250161:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.250164:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.250170:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.250174:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b078 00000800:00000200:26.0:1616513087.250177:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.250180:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.269396:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513087.269454:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.269463:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.269472:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.124@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.269482:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.124@tcp1 of length 224 into portal 26 MB=0x6019c8406b080 00000400:00000200:18.0:1616513087.269488:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.124@tcp1 of length 224/224 into md 0xf97cc019 [1] + 0 00000400:00000200:18.0:1616513087.269493:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb058f8 00000400:00000200:18.0:1616513087.269496:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.269499:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513087.269503:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb058f8 00000800:00000200:18.0:1616513087.269506:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.269509:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.269509:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513087.269516:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.269519:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.269520:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.269528:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.124@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.269537:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.124@tcp1 of length 224 into portal 12 MB=0x6019c8406b100 00000400:00000200:14.0:1616513087.269544:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.124@tcp1 of length 224/224 into md 0xf964d519 [1] + 61360 00000400:00000200:14.0:1616513087.269549:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.269552:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.269558:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.269563:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.269565:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.269588:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690621601820800 00010000:00000200:8.0:1616513087.269599:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1690621601820800/t0(0) o400->4c362506-2354-e163-de9e-d2fa4b63e894@10.31.140.124@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.269613:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690621601820800, offset 192 00000400:00000200:8.0:1616513087.269620:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.124@tcp1 00000100:00000200:22.0:1616513087.269623:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690621601820928 00000400:00000200:8.0:1616513087.269627:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.124@tcp1 routed destination 00010000:00000200:22.0:1616513087.269635:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7bf00 x1690621601820928/t0(0) o400->d7a12095-fc93-adc8-2f1c-309685eb9bf8@10.31.140.124@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:8.0:1616513087.269642:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.124@tcp1(10.31.140.124@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513087.269649:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690621601820928, offset 192 00000800:00000200:8.0:1616513087.269653:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513087.269655:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.124@tcp1 00000800:00000200:8.0:1616513087.269658:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.269661:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513087.269663:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.124@tcp1 routed destination 00000800:00000200:8.0:1616513087.269663:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.269668:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.269675:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513087.269678:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.124@tcp1(10.31.140.124@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.269684:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.269693:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.269696:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.269698:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.269703:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.269708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513087.269715:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.269784:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513087.269784:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.269791:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513087.269792:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513087.269794:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513087.269795:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.269795:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513087.269801:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.269801:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.269805:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a5d8 00000400:00000200:20.0:1616513087.269807:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f703b8 00000800:00000200:2.0:1616513087.269809:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513087.269810:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.269813:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.274483:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.274541:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.274550:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.274559:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.54@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.274568:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.54@tcp1 of length 224 into portal 26 MB=0x5ffdd3937cd40 00000400:00000200:0.0:1616513087.274575:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.54@tcp1 of length 224/224 into md 0xf97cc029 [1] + 0 00000400:00000200:0.0:1616513087.274580:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05540 00000400:00000200:0.0:1616513087.274583:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.274585:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513087.274590:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05540 00000800:00000200:0.0:1616513087.274593:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.274595:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.274595:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.274599:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.274652:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700496366912 00010000:00000200:8.0:1616513087.274663:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688700496366912/t0(0) o400->d75f99a0-9425-5538-edc3-308cd45f72f0@10.31.129.54@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513087.274671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:8.0:1616513087.274677:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700496366912, offset 192 00000400:00000200:8.0:1616513087.274683:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.54@tcp1 00000400:00000200:8.0:1616513087.274691:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.54@tcp1 routed destination 00000400:00000200:8.0:1616513087.274705:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.54@tcp1(10.31.129.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.274712:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.274718:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.274721:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.274723:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.274728:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513087.274737:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513087.274740:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:8.0:1616513087.274743:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513087.274748:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.54@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.274757:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.54@tcp1 of length 224 into portal 12 MB=0x5ffdd3937d0c0 00000400:00000200:20.0:1616513087.274763:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.54@tcp1 of length 224/224 into md 0x865bbec1 [1] + 89600 00000400:00000200:20.0:1616513087.274769:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.274771:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513087.274778:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513087.274781:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513087.274783:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513087.274786:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.274790:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.274792:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513087.274794:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.274797:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513087.274801:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71430 00000800:00000200:20.0:1616513087.274804:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513087.274842:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700496367808 00010000:00000200:4.0:1616513087.274855:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5100 x1688700496367808/t0(0) o400->aa808893-7f77-1a9e-4397-5ce546119017@10.31.129.54@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.274869:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700496367808, offset 192 00000400:00000200:4.0:1616513087.274875:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.54@tcp1 00000400:00000200:4.0:1616513087.274883:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.54@tcp1 routed destination 00000400:00000200:4.0:1616513087.274898:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.54@tcp1(10.31.129.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.274904:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.274910:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.274912:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.274916:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.274921:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.274926:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.274937:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.274943:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.274945:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.274951:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.274955:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba4c8 00000800:00000200:14.0:1616513087.274959:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.274962:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.277481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.277538:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.277547:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.277556:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.153@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.277565:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.153@tcp1 of length 224 into portal 26 MB=0x5ffd80bf23f40 00000400:00000200:0.0:1616513087.277571:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.153@tcp1 of length 224/224 into md 0xf97cc059 [1] + 0 00000400:00000200:0.0:1616513087.277576:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04bb0 00000400:00000200:0.0:1616513087.277579:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.277581:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513087.277585:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:0.0:1616513087.277588:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.277591:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513087.277592:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.277596:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.277648:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678261997376 00010000:00000200:8.0:1616513087.277659:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688678261997376/t0(0) o400->78c3aa65-749b-18ae-dac7-f7ed639d82d2@10.31.129.153@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.277673:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688678261997376, offset 192 00000400:00000200:8.0:1616513087.277679:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.153@tcp1 00000400:00000200:8.0:1616513087.277687:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.153@tcp1 routed destination 00000400:00000200:8.0:1616513087.277702:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.153@tcp1(10.31.129.153@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.277718:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.277724:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.277726:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.277730:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513087.277732:0:50236:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513087.277734:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.277739:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.277740:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513087.277785:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.277795:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:28.0:1616513087.277803:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513087.277804:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.153@tcp1 : PUT - for me 00000800:00000200:28.0:1616513087.277809:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.277812:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.277812:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.153@tcp1 of length 224 into portal 12 MB=0x5ffd80bf24240 00000400:00000200:18.0:1616513087.277818:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.153@tcp1 of length 224/224 into md 0xf964d519 [1] + 61584 00000400:00000200:28.0:1616513087.277819:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.277823:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.277825:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71ba0 00000100:00000200:18.0:1616513087.277826:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513087.277828:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.277831:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513087.277834:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.277837:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.277838:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.277842:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.277901:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678261998144 00010000:00000200:22.0:1616513087.277913:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bda00 x1688678261998144/t0(0) o400->eb1f0b2e-9b64-3cfd-c5a4-6c2faf490e16@10.31.129.153@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.277938:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688678261998144, offset 192 00000400:00000200:22.0:1616513087.277944:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.153@tcp1 00000400:00000200:22.0:1616513087.277952:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.153@tcp1 routed destination 00000400:00000200:22.0:1616513087.277967:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.153@tcp1(10.31.129.153@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.277973:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.277979:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.277982:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.277984:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.277989:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.277993:0:50236:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.278044:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.278051:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.278053:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.278059:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.278063:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a2a8 00000800:00000200:14.0:1616513087.278067:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.278070:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.302308:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513087.302360:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513087.302362:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.302372:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.302381:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.120@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.302390:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.120@tcp1 of length 224 into portal 12 MB=0x5fd27b2b05280 00000400:00000200:2.0:1616513087.302396:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.120@tcp1 of length 224/224 into md 0xf964d519 [1] + 61808 00000400:00000200:2.0:1616513087.302401:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.302404:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513087.302411:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.302414:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.302415:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513087.302416:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513087.302418:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513087.302426:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513087.302449:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.120@tcp1 : PUT - for me 00000400:00000200:16.0:1616513087.302458:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.120@tcp1 of length 224 into portal 26 MB=0x5fd27b2b05200 00000400:00000200:16.0:1616513087.302465:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.120@tcp1 of length 224/224 into md 0xf97cc069 [1] + 0 00000400:00000200:16.0:1616513087.302470:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04cc0 00000400:00000200:16.0:1616513087.302472:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513087.302475:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1616513087.302477:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513087.302480:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04cc0 00000800:00000200:16.0:1616513087.302483:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513087.302486:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513087.302487:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513087.302497:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1685721827005056 00000100:00000200:8.0:1616513087.302499:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1685721827004928 00010000:00000200:8.0:1616513087.302509:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1685721827004928/t0(0) o400->d94e7f42-7453-26fe-47d5-728f41883a5d@10.31.143.120@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:22.0:1616513087.302510:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cba80 x1685721827005056/t0(0) o400->a5f99442-e339-7641-3c18-adebef9789e7@10.31.143.120@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513087.302522:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1685721827004928, offset 192 00000100:00000200:22.0:1616513087.302524:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1685721827005056, offset 192 00000400:00000200:8.0:1616513087.302528:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.120@tcp1 00000400:00000200:22.0:1616513087.302530:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.120@tcp1 00000400:00000200:8.0:1616513087.302535:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.120@tcp1 routed destination 00000400:00000200:22.0:1616513087.302538:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.120@tcp1 routed destination 00000400:00000200:8.0:1616513087.302550:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.120@tcp1(10.31.143.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513087.302553:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.120@tcp1(10.31.143.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.302556:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513087.302560:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513087.302562:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.302564:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513087.302565:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.302566:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513087.302568:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.302570:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513087.302571:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513087.302575:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513087.302576:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.302577:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513087.302579:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.302589:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.302595:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.302598:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.302604:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.302608:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4abb0 00000800:00000200:14.0:1616513087.302612:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.302614:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513087.302640:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.302647:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.302650:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.302656:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.302662:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f702a8 00000800:00000200:28.0:1616513087.302665:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.302668:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.311082:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.311138:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.311148:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.311157:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.2@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.311167:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.2@tcp1 of length 224 into portal 26 MB=0x5ffd410a7ef40 00000400:00000200:0.0:1616513087.311174:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.2@tcp1 of length 224/224 into md 0xf97cc079 [1] + 0 00000400:00000200:0.0:1616513087.311179:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04440 00000400:00000200:0.0:1616513087.311182:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.311185:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513087.311190:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04440 00000800:00000200:16.0:1616513087.311192:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.311193:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.311196:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.311198:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.311200:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661161144128 00010000:00000200:8.0:1616513087.311206:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688661161144128/t0(0) o400->3579ff8c-94b4-716b-4e03-5b1baeff233d@10.31.133.2@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.311216:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661161144128, offset 192 00000400:00000200:8.0:1616513087.311221:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.2@tcp1 00000400:00000200:8.0:1616513087.311226:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.2@tcp1 routed destination 00000400:00000200:8.0:1616513087.311238:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.2@tcp1(10.31.133.2@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.311244:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.311248:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.311250:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.311252:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.311254:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.311258:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.311259:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.311312:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.311319:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.311322:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.311329:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.311334:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71d38 00000800:00000200:28.0:1616513087.311338:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.311341:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.311665:0:50239:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.311724:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.311734:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513087.311737:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.311743:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.2@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.311752:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.2@tcp1 of length 224 into portal 12 MB=0x5ffd410a7f040 00000400:00000200:2.0:1616513087.311758:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.2@tcp1 of length 224/224 into md 0xf964d519 [1] + 62032 00000400:00000200:2.0:1616513087.311765:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.311768:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513087.311774:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.311777:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.311779:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.311840:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661161144384 00010000:00000200:22.0:1616513087.311852:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c8000 x1688661161144384/t0(0) o400->8282cbc7-6439-69d3-33df-0d88accf5f53@10.31.133.2@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.311866:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661161144384, offset 192 00000400:00000200:22.0:1616513087.311872:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.2@tcp1 00000400:00000200:22.0:1616513087.311880:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.2@tcp1 routed destination 00000400:00000200:22.0:1616513087.311894:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.2@tcp1(10.31.133.2@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.311903:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.311909:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.311912:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.311914:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.311919:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.311923:0:50239:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.311934:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.311940:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.311943:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.311949:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.311953:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a880 00000800:00000200:14.0:1616513087.311956:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.311959:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.326879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513087.326934:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.326943:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.326952:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.113@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.326961:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.113@tcp1 of length 224 into portal 26 MB=0x6000f1321b1c0 00000400:00000200:18.0:1616513087.326967:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.113@tcp1 of length 224/224 into md 0xf97cc0a9 [1] + 0 00000400:00000200:18.0:1616513087.326972:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05d38 00000400:00000200:18.0:1616513087.326974:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.326977:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513087.326981:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05d38 00000800:00000200:18.0:1616513087.326984:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.326986:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.326988:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.326990:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513087.326998:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688914605748672 00000800:00000200:2.0:1616513087.327001:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:8.0:1616513087.327007:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688914605748672/t0(0) o400->433a98dd-38d5-64e6-fb21-93884fe25c19@10.31.138.113@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513087.327010:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.113@tcp1 : PUT - for me 00000100:00000200:8.0:1616513087.327019:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688914605748672, offset 192 00000400:00000200:2.0:1616513087.327019:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.113@tcp1 of length 224 into portal 12 MB=0x6000f1321b280 00000400:00000200:8.0:1616513087.327025:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.113@tcp1 00000400:00000200:2.0:1616513087.327025:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.113@tcp1 of length 224/224 into md 0xf964d519 [1] + 62256 00000400:00000200:2.0:1616513087.327030:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513087.327031:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.113@tcp1 routed destination 00000100:00000200:2.0:1616513087.327033:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513087.327039:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:8.0:1616513087.327043:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.113@tcp1(10.31.138.113@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513087.327043:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.327044:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.327047:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513087.327049:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.327054:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.327056:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.327058:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.327062:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.327068:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.327069:0:5991:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513087.327107:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688914605748864 00010000:00000200:22.0:1616513087.327119:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c8480 x1688914605748864/t0(0) o400->8a138965-a194-1b59-0489-267f5e455629@10.31.138.113@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513087.327125:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:22.0:1616513087.327133:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688914605748864, offset 192 00000800:00000200:16.0:1616513087.327133:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513087.327135:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513087.327140:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.113@tcp1 00000400:00000200:16.0:1616513087.327142:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513087.327142:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513087.327147:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.113@tcp1 routed destination 00000400:00000200:16.0:1616513087.327148:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70d48 00000800:00000200:16.0:1616513087.327151:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513087.327162:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.113@tcp1(10.31.138.113@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.327168:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.327174:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.327176:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.327179:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.327183:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.327188:0:50239:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.327243:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.327250:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.327253:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.327253:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.327260:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.327264:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b980 00000800:00000200:26.0:1616513087.327268:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.335862:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.335918:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.335927:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513087.335936:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.142@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.335946:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.142@tcp1 of length 224 into portal 26 MB=0x60007dffb0200 00000400:00000200:28.0:1616513087.335952:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.142@tcp1 of length 224/224 into md 0xf97cc0b9 [1] + 0 00000400:00000200:28.0:1616513087.335957:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb055c8 00000400:00000200:28.0:1616513087.335960:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.335963:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513087.335968:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb055c8 00000800:00000200:28.0:1616513087.335971:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.335972:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.335976:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513087.335979:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.335982:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.335982:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688883682804224 00000800:00000200:20.0:1616513087.335983:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00010000:00000200:8.0:1616513087.335990:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688883682804224/t0(0) o400->20233cde-431d-3b01-969f-bb95e56bc9e7@10.31.140.142@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513087.335993:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.142@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.336002:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.142@tcp1 of length 224 into portal 12 MB=0x60007dffb0440 00000100:00000200:8.0:1616513087.336002:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688883682804224, offset 192 00000400:00000200:8.0:1616513087.336008:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.142@tcp1 00000400:00000200:20.0:1616513087.336013:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.142@tcp1 of length 224/224 into md 0x865bbec1 [1] + 89824 00000400:00000200:8.0:1616513087.336015:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.142@tcp1 routed destination 00000400:00000200:20.0:1616513087.336018:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.336020:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513087.336028:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.142@tcp1(10.31.140.142@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513087.336033:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.336034:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513087.336036:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.336038:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513087.336039:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.336041:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.336043:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.336047:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:4.0:1616513087.336050:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688883682804800 00000800:00000200:11.2:1616513087.336051:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.336053:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:4.0:1616513087.336063:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0d80 x1688883682804800/t0(0) o400->2c310d52-2a1e-901e-01ac-88969b1cd7c0@10.31.140.142@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.336078:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688883682804800, offset 192 00000400:00000200:4.0:1616513087.336085:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.142@tcp1 00000400:00000200:4.0:1616513087.336092:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.142@tcp1 routed destination 00000800:00000200:16.0:1616513087.336102:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:4.0:1616513087.336107:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.142@tcp1(10.31.140.142@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513087.336110:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513087.336112:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513087.336114:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513087.336114:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513087.336119:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:4.0:1616513087.336120:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.336123:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513087.336125:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f715c8 00000800:00000200:4.0:1616513087.336125:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513087.336128:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:4.0:1616513087.336130:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.336136:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.336190:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.336197:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.336200:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513087.336206:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.336211:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba4c8 00000800:00000200:2.0:1616513087.336215:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.336218:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.380416:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513087.380449:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.380460:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.380470:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.67@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.380481:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.67@tcp1 of length 224 into portal 26 MB=0x5ffe06354b440 00000400:00000200:18.0:1616513087.380488:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.67@tcp1 of length 224/224 into md 0xf97cc0d9 [1] + 0 00000400:00000200:18.0:1616513087.380493:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04dd0 00000400:00000200:18.0:1616513087.380496:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.380499:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513087.380504:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04dd0 00000800:00000200:18.0:1616513087.380508:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.380511:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.380513:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.380517:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.380528:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714087806016 00010000:00000200:8.0:1616513087.380540:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688714087806016/t0(0) o400->34f9ffab-b6cb-9c6d-154a-770d4cd645c2@10.31.129.67@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.380553:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688714087806016, offset 192 00000400:00000200:8.0:1616513087.380559:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.67@tcp1 00000400:00000200:8.0:1616513087.380567:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.67@tcp1 routed destination 00000400:00000200:8.0:1616513087.380581:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.67@tcp1(10.31.129.67@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.380587:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.380593:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.380595:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.380597:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.380603:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.380607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.380609:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513087.380625:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.380634:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.380643:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.67@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.380652:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.67@tcp1 of length 224 into portal 12 MB=0x5ffe06354b840 00000400:00000200:0.0:1616513087.380658:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.67@tcp1 of length 224/224 into md 0x865bbec1 [1] + 90048 00000400:00000200:0.0:1616513087.380663:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.380668:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513087.380676:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513087.380679:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:0.0:1616513087.380679:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513087.380687:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513087.380688:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513087.380690:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.380691:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513087.380698:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513087.380704:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70908 00000800:00000200:20.0:1616513087.380708:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513087.380737:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714087807040 00010000:00000200:4.0:1616513087.380750:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86b600 x1688714087807040/t0(0) o400->5d4c6ce2-847c-7b4e-9510-28518798db43@10.31.129.67@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.380762:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688714087807040, offset 192 00000400:00000200:4.0:1616513087.380767:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.67@tcp1 00000400:00000200:4.0:1616513087.380774:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.67@tcp1 routed destination 00000400:00000200:4.0:1616513087.380787:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.67@tcp1(10.31.129.67@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.380792:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.380798:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.380800:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.380802:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.380807:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.380813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.380825:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.380833:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.380836:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513087.380842:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.380846:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb078 00000800:00000200:2.0:1616513087.380850:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.380853:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.438868:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513087.438929:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.438948:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.438957:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.188@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.438970:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.188@tcp1 of length 224 into portal 26 MB=0x5ffd4351dfd80 00000400:00000200:18.0:1616513087.438977:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.188@tcp1 of length 224/224 into md 0xf97cc0e9 [1] + 0 00000400:00000200:18.0:1616513087.438982:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05210 00000400:00000200:18.0:1616513087.438985:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.438989:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513087.438993:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05210 00000800:00000200:26.0:1616513087.438994:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.438997:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.439000:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.439002:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.439018:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661772860800 00010000:00000200:8.0:1616513087.439029:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688661772860800/t0(0) o400->8da74a25-c7d6-5d3a-08c6-21c28a62fada@10.31.136.188@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.439044:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661772860800, offset 192 00000400:00000200:8.0:1616513087.439050:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.188@tcp1 00000400:00000200:8.0:1616513087.439058:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.188@tcp1 routed destination 00000400:00000200:8.0:1616513087.439072:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.188@tcp1(10.31.136.188@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.439079:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.439085:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.439087:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.439089:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.439096:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.439102:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.439102:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513087.439158:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513087.439167:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513087.439170:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513087.439176:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513087.439183:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70aa0 00000800:00000200:16.0:1616513087.439187:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513087.439190:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.439414:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.439467:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.439476:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.439484:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.188@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.439493:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.188@tcp1 of length 224 into portal 12 MB=0x5ffd4351dfe00 00000400:00000200:2.0:1616513087.439501:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.188@tcp1 of length 224/224 into md 0xf964d519 [1] + 62480 00000400:00000200:2.0:1616513087.439506:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.439509:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513087.439516:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.439520:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.439522:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.439526:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.439609:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661772860928 00010000:00000200:22.0:1616513087.439623:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53ce300 x1688661772860928/t0(0) o400->96ac3d8b-b3cd-02fa-6129-e6eb709884da@10.31.136.188@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.439638:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661772860928, offset 192 00000400:00000200:22.0:1616513087.439644:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.188@tcp1 00000400:00000200:22.0:1616513087.439651:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.188@tcp1 routed destination 00000400:00000200:22.0:1616513087.439665:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.188@tcp1(10.31.136.188@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.439671:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.439677:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.439680:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.439682:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.439687:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.439692:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.439703:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.439709:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.439711:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.439713:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.439718:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.439722:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b870 00000800:00000200:26.0:1616513087.439726:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.481439:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513087.481463:0:50259:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513087.481495:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.481505:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513087.481515:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.98@tcp1 : PUT - for me 00000800:00000200:14.0:1616513087.481524:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513087.481526:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.98@tcp1 of length 224 into portal 26 MB=0x5ffdcbb9c0680 00000400:00000200:28.0:1616513087.481533:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.98@tcp1 of length 224/224 into md 0xf97cc119 [1] + 0 00000800:00000200:14.0:1616513087.481535:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513087.481537:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04660 00000400:00000200:28.0:1616513087.481541:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.481544:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.481545:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.98@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.481550:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04660 00000800:00000200:20.0:1616513087.481550:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.481553:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513087.481554:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.98@tcp1 of length 224 into portal 12 MB=0x5ffdcbb9c0880 00000800:00000200:28.0:1616513087.481557:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.481558:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513087.481563:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.98@tcp1 of length 224/224 into md 0xf964d519 [1] + 62704 00000400:00000200:14.0:1616513087.481569:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.481572:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:8.0:1616513087.481574:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698389005952 00000800:00000200:14.0:1616513087.481580:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.481580:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.481583:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.481585:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:8.0:1616513087.481586:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688698389005952/t0(0) o400->150559ee-c643-eda0-eafd-62731f420e08@10.31.142.98@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.481602:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698389005952, offset 192 00000400:00000200:8.0:1616513087.481609:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.98@tcp1 00000400:00000200:8.0:1616513087.481617:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.98@tcp1 routed destination 00000400:00000200:8.0:1616513087.481633:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.98@tcp1(10.31.142.98@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.481639:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.481646:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.481649:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.481652:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.481657:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513087.481659:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698389006464 00000800:00000200:11.2:1616513087.481662:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.481664:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:22.0:1616513087.481673:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7d580 x1688698389006464/t0(0) o400->e974719b-3dd1-f128-f616-051747e75889@10.31.142.98@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.481689:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698389006464, offset 192 00000400:00000200:22.0:1616513087.481695:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.98@tcp1 00000400:00000200:22.0:1616513087.481703:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.98@tcp1 routed destination 00000800:00000200:16.0:1616513087.481715:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513087.481718:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.98@tcp1(10.31.142.98@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513087.481723:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513087.481725:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513087.481726:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513087.481731:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513087.481733:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513087.481734:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.481736:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513087.481739:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71a90 00000800:00000200:22.0:1616513087.481742:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513087.481742:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513087.481745:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.481747:0:50259:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.481812:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.481820:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.481822:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.481829:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.481833:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a550 00000800:00000200:18.0:1616513087.481837:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.481840:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.482083:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.482095:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.482102:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513087.482104:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.482111:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.17@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.482121:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.17@tcp1 of length 224 into portal 26 MB=0x60143df9867c0 00000400:00000200:20.0:1616513087.482128:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.17@tcp1 of length 224/224 into md 0xf97cc129 [1] + 0 00000400:00000200:20.0:1616513087.482133:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04f68 00000400:00000200:20.0:1616513087.482135:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.482138:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513087.482143:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:20.0:1616513087.482146:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.482149:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.482151:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.482153:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690240886007744 00010000:00000200:8.0:1616513087.482158:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1690240886007744/t0(0) o400->f3285e1a-2030-433c-bfe5-eab4ec3f69e8@10.31.136.17@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.482168:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690240886007744, offset 192 00000400:00000200:8.0:1616513087.482173:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.17@tcp1 00000400:00000200:8.0:1616513087.482178:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.17@tcp1 routed destination 00000400:00000200:8.0:1616513087.482190:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.17@tcp1(10.31.136.17@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.482195:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.482200:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.482202:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.482203:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.482206:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.482210:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.482211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513087.482220:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513087.482224:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513087.482225:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.482230:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513087.482231:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513087.482234:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71980 00000800:00000200:16.0:1616513087.482236:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.482242:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.482251:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.482257:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513087.482259:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513087.482263:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.17@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.482270:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.17@tcp1 of length 224 into portal 12 MB=0x60143df986940 00000400:00000200:28.0:1616513087.482275:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.17@tcp1 of length 224/224 into md 0x865bbec1 [1] + 90272 00000400:00000200:28.0:1616513087.482279:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.482281:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513087.482288:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.482290:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.482291:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513087.482345:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690240886008128 00010000:00000200:4.0:1616513087.482354:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa0900 x1690240886008128/t0(0) o400->96bc9dfa-cca1-55d0-30b1-7d8027bb7534@10.31.136.17@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.482366:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690240886008128, offset 192 00000400:00000200:4.0:1616513087.482372:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.17@tcp1 00000400:00000200:4.0:1616513087.482378:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.17@tcp1 routed destination 00000400:00000200:4.0:1616513087.482389:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.17@tcp1(10.31.136.17@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.482394:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.482398:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.482401:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.482402:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.482406:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.482414:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.482427:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.482432:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.482455:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513087.482462:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.482466:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba990 00000800:00000200:2.0:1616513087.482470:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.482473:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.505229:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.505287:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.505297:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.505306:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.172@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.505315:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.172@tcp1 of length 224 into portal 26 MB=0x5ffdbdc4ffdc0 00000400:00000200:0.0:1616513087.505322:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.172@tcp1 of length 224/224 into md 0xf97cc139 [1] + 0 00000400:00000200:0.0:1616513087.505326:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb046e8 00000400:00000200:0.0:1616513087.505329:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.505332:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513087.505337:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb046e8 00000800:00000200:0.0:1616513087.505340:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513087.505342:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513087.505343:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.505349:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.505352:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513087.505353:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:8.0:1616513087.505356:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694642703808 00000400:00000200:16.0:1616513087.505362:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.172@tcp1 : PUT - for me 00010000:00000200:8.0:1616513087.505366:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688694642703808/t0(0) o400->535c977f-523e-f5f6-47e0-64a52f14f54a@10.31.141.172@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513087.505371:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.172@tcp1 of length 224 into portal 12 MB=0x5ffdbdc4ffe40 00000400:00000200:16.0:1616513087.505377:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.172@tcp1 of length 224/224 into md 0x865bbec1 [1] + 90496 00000400:00000200:16.0:1616513087.505383:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513087.505383:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694642703808, offset 192 00000100:00000200:16.0:1616513087.505385:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513087.505392:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513087.505394:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.172@tcp1 00000800:00000200:16.0:1616513087.505396:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513087.505397:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513087.505400:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.172@tcp1 routed destination 00000400:00000200:8.0:1616513087.505414:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.172@tcp1(10.31.141.172@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.505420:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.505426:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.505428:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.505430:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:11.2:1616513087.505440:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.505441:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.505448:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:4.0:1616513087.505463:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694642703936 00010000:00000200:4.0:1616513087.505475:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd2400 x1688694642703936/t0(0) o400->716c9985-b534-b6b7-c9b0-83eaa0bd1cdb@10.31.141.172@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.505490:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694642703936, offset 192 00000800:00000200:28.0:1616513087.505495:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:4.0:1616513087.505496:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.172@tcp1 00000800:00000200:28.0:1616513087.505502:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513087.505504:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.172@tcp1 routed destination 00000400:00000200:28.0:1616513087.505505:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.505506:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513087.505511:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.505517:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71540 00000400:00000200:4.0:1616513087.505519:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.172@tcp1(10.31.141.172@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513087.505521:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:4.0:1616513087.505526:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.505532:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.505535:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.505537:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.505542:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.505549:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.505605:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.505612:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.505614:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.505621:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.505625:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba440 00000800:00000200:18.0:1616513087.505629:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.505632:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.515452:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.515509:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.515519:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.515528:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.237@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.515537:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.237@tcp1 of length 224 into portal 26 MB=0x603e103f1c940 00000400:00000200:2.0:1616513087.515544:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.237@tcp1 of length 224/224 into md 0xf97cc149 [1] + 0 00000400:00000200:2.0:1616513087.515549:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04908 00000400:00000200:2.0:1616513087.515552:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.515555:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.515559:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04908 00000800:00000200:2.0:1616513087.515563:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.515564:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.515565:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.515572:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.515574:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.515575:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513087.515575:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114828966208 00010000:00000200:8.0:1616513087.515583:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1693114828966208/t0(0) o400->4e780524-3516-e996-62ac-7c387401ecd5@10.31.136.237@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513087.515584:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.237@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.515594:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.237@tcp1 of length 224 into portal 12 MB=0x603e103f1ca00 00000100:00000200:8.0:1616513087.515594:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114828966208, offset 192 00000400:00000200:14.0:1616513087.515600:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.237@tcp1 of length 224/224 into md 0xf964d519 [1] + 62928 00000400:00000200:8.0:1616513087.515600:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.237@tcp1 00000400:00000200:8.0:1616513087.515605:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.237@tcp1 routed destination 00000400:00000200:14.0:1616513087.515606:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.515608:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.515615:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513087.515618:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.237@tcp1(10.31.136.237@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513087.515619:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.515621:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513087.515624:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.515628:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.515631:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.515632:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.515636:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.515642:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.515644:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513087.515685:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114828966400 00010000:00000200:22.0:1616513087.515698:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7c800 x1693114828966400/t0(0) o400->2053a2a9-9161-6dbc-4866-c3e7eb6d19d9@10.31.136.237@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513087.515700:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.515707:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.515709:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:22.0:1616513087.515712:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114828966400, offset 192 00000400:00000200:20.0:1616513087.515716:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513087.515718:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.237@tcp1 00000400:00000200:20.0:1616513087.515721:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71870 00000400:00000200:22.0:1616513087.515726:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.237@tcp1 routed destination 00000800:00000200:20.0:1616513087.515726:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.515730:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513087.515740:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.237@tcp1(10.31.136.237@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.515747:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.515753:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.515756:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.515758:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.515763:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.515769:0:50263:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.515833:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.515840:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.515843:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513087.515844:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.515851:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.515855:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba90 00000800:00000200:26.0:1616513087.515859:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.520210:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.520278:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.520288:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.520297:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.167@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.520306:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.167@tcp1 of length 224 into portal 26 MB=0x5ffd679e51580 00000400:00000200:0.0:1616513087.520313:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.167@tcp1 of length 224/224 into md 0xf97cc159 [1] + 0 00000400:00000200:0.0:1616513087.520318:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04aa0 00000400:00000200:0.0:1616513087.520321:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.520323:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513087.520328:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04aa0 00000800:00000200:0.0:1616513087.520331:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.520334:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.520336:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.520339:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.520342:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688671516693888 00010000:00000200:8.0:1616513087.520348:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688671516693888/t0(0) o400->0df77d17-0893-2f1c-b5a5-1a34ffa0fd6a@10.31.138.167@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513087.520349:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513087.520358:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.167@tcp1 : PUT - for me 00000100:00000200:8.0:1616513087.520359:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688671516693888, offset 192 00000400:00000200:8.0:1616513087.520365:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.167@tcp1 00000400:00000200:28.0:1616513087.520367:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.167@tcp1 of length 224 into portal 12 MB=0x5ffd679e51600 00000400:00000200:8.0:1616513087.520371:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.167@tcp1 routed destination 00000400:00000200:28.0:1616513087.520373:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.167@tcp1 of length 224/224 into md 0x865bbec1 [1] + 90720 00000400:00000200:28.0:1616513087.520379:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.520381:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513087.520383:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.167@tcp1(10.31.138.167@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513087.520388:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.520389:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513087.520391:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.520393:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513087.520393:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.520395:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.520396:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513087.520397:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.520400:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.520405:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.520413:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513087.520473:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688671516694016 00000800:00000200:16.0:1616513087.520477:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513087.520484:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:4.0:1616513087.520486:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa0000 x1688671516694016/t0(0) o400->bf564b52-7545-34b6-1d27-fbf6f9a37eb9@10.31.138.167@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513087.520487:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.520488:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513087.520493:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513087.520499:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71870 00000100:00000200:4.0:1616513087.520503:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688671516694016, offset 192 00000800:00000200:16.0:1616513087.520504:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:4.0:1616513087.520510:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.167@tcp1 00000400:00000200:4.0:1616513087.520517:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.167@tcp1 routed destination 00000400:00000200:4.0:1616513087.520533:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.167@tcp1(10.31.138.167@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.520539:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.520545:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.520549:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.520551:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.520556:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.520562:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.520617:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.520624:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.520627:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.520633:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.520637:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb650 00000800:00000200:18.0:1616513087.520641:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.520644:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.531793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.531856:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.531866:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.531874:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.172@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.531884:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.172@tcp1 of length 224 into portal 26 MB=0x6018330e2d440 00000400:00000200:2.0:1616513087.531890:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.172@tcp1 of length 224/224 into md 0xf97cc169 [1] + 0 00000400:00000200:2.0:1616513087.531903:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb044c8 00000400:00000200:2.0:1616513087.531906:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513087.531909:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:2.0:1616513087.531909:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.531913:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb044c8 00000800:00000200:2.0:1616513087.531916:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.531919:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.531921:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.531930:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512832779328 00010000:00000200:8.0:1616513087.531938:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1690512832779328/t0(0) o400->211a469f-ad73-ea18-e950-e7d76e6afb47@10.31.132.172@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.531950:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512832779328, offset 192 00000400:00000200:8.0:1616513087.531956:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.172@tcp1 00000400:00000200:8.0:1616513087.531963:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.172@tcp1 routed destination 00000400:00000200:8.0:1616513087.531976:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.172@tcp1(10.31.132.172@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.531982:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.531987:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.531989:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.531991:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.531995:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.532001:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.532004:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.532059:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.532065:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.532068:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513087.532068:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513087.532075:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.532080:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71980 00000800:00000200:28.0:1616513087.532084:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.532087:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513087.532138:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.532147:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.532156:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.172@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.532167:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.172@tcp1 of length 224 into portal 12 MB=0x6018330e2d580 00000400:00000200:18.0:1616513087.532174:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.172@tcp1 of length 224/224 into md 0xf964d519 [1] + 63152 00000400:00000200:18.0:1616513087.532179:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.532182:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513087.532189:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.532192:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.532192:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.532197:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.532255:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512832779648 00010000:00000200:22.0:1616513087.532267:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b8000 x1690512832779648/t0(0) o400->7191231a-497a-60d6-230c-9d562d267d58@10.31.132.172@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.532281:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512832779648, offset 192 00000400:00000200:22.0:1616513087.532288:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.172@tcp1 00000400:00000200:22.0:1616513087.532295:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.172@tcp1 routed destination 00000400:00000200:22.0:1616513087.532312:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.172@tcp1(10.31.132.172@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.532318:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.532324:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.532327:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.532329:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.532334:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.532339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.532350:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.532355:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.532358:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513087.532359:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.532365:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.532369:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a000 00000800:00000200:26.0:1616513087.532372:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.550706:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.550761:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.550771:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.550780:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.203@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.550790:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.203@tcp1 of length 224 into portal 26 MB=0x6002fe1819000 00000400:00000200:0.0:1616513087.550797:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.203@tcp1 of length 224/224 into md 0xf97cc189 [1] + 0 00000400:00000200:0.0:1616513087.550802:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb045d8 00000400:00000200:0.0:1616513087.550804:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.550807:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513087.550812:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb045d8 00000800:00000200:0.0:1616513087.550815:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513087.550816:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.550818:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.550820:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.550827:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689055507091456 00010000:00000200:8.0:1616513087.550835:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1689055507091456/t0(0) o400->036927b2-3a2c-4119-2461-d3338da35419@10.31.128.203@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.550846:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689055507091456, offset 192 00000400:00000200:8.0:1616513087.550851:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.203@tcp1 00000400:00000200:8.0:1616513087.550858:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.203@tcp1 routed destination 00000400:00000200:8.0:1616513087.550870:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.203@tcp1(10.31.128.203@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.550876:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.550881:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.550883:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.550885:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.550889:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.550894:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.550894:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513087.550949:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.550956:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.550958:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.550965:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.550971:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70aa0 00000800:00000200:28.0:1616513087.550974:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.550977:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.551419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.551475:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.551485:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.551494:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.203@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.551504:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.203@tcp1 of length 224 into portal 12 MB=0x6002fe1819500 00000400:00000200:14.0:1616513087.551510:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.203@tcp1 of length 224/224 into md 0xf964d519 [1] + 63376 00000400:00000200:14.0:1616513087.551516:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.551519:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.551526:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.551529:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.551530:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.551534:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.551592:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689055507092736 00010000:00000200:22.0:1616513087.551604:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7c380 x1689055507092736/t0(0) o400->aa199513-816a-88d0-6c20-5d922c121ef2@10.31.128.203@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.551619:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689055507092736, offset 192 00000400:00000200:22.0:1616513087.551625:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.203@tcp1 00000400:00000200:22.0:1616513087.551633:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.203@tcp1 routed destination 00000400:00000200:22.0:1616513087.551648:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.203@tcp1(10.31.128.203@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.551655:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.551660:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.551663:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.551665:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.551670:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.551688:0:50269:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.551742:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.551750:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.551752:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513087.551759:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.551763:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a908 00000800:00000200:2.0:1616513087.551767:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.551770:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.567938:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513087.567995:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.568004:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.568013:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.65@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.568023:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.65@tcp1 of length 224 into portal 26 MB=0x5ff49e9d9a200 00000400:00000200:18.0:1616513087.568029:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.65@tcp1 of length 224/224 into md 0xf97cc199 [1] + 0 00000400:00000200:18.0:1616513087.568034:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb056d8 00000400:00000200:18.0:1616513087.568036:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.568039:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513087.568043:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb056d8 00000800:00000200:18.0:1616513087.568047:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.568050:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.568050:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513087.568057:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.568060:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.568060:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513087.568061:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688067804602880 00000400:00000200:14.0:1616513087.568069:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.65@tcp1 : PUT - for me 00010000:00000200:8.0:1616513087.568070:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688067804602880/t0(0) o400->9f80529f-9958-9517-cd79-370bff497008@10.31.132.65@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513087.568078:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.65@tcp1 of length 224 into portal 12 MB=0x5ff49e9d9a280 00000100:00000200:8.0:1616513087.568082:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688067804602880, offset 192 00000400:00000200:14.0:1616513087.568084:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.65@tcp1 of length 224/224 into md 0xf964d519 [1] + 63600 00000400:00000200:8.0:1616513087.568087:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.65@tcp1 00000400:00000200:14.0:1616513087.568091:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.568094:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513087.568094:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.65@tcp1 routed destination 00000800:00000200:14.0:1616513087.568101:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.568104:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.568106:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513087.568107:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.65@tcp1(10.31.132.65@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.568116:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.568121:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.568124:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.568125:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.568130:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.568135:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.568138:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513087.568171:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688067804603008 00010000:00000200:22.0:1616513087.568184:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a79200 x1688067804603008/t0(0) o400->ef984be8-e197-3fe5-686d-79fc2944ae4d@10.31.132.65@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513087.568193:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:22.0:1616513087.568198:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688067804603008, offset 192 00000800:00000200:16.0:1616513087.568200:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513087.568203:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513087.568205:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.65@tcp1 00000400:00000200:16.0:1616513087.568210:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513087.568212:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.65@tcp1 routed destination 00000400:00000200:16.0:1616513087.568215:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f715c8 00000800:00000200:16.0:1616513087.568219:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513087.568222:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513087.568227:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.65@tcp1(10.31.132.65@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.568233:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.568240:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.568242:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.568247:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.568251:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.568257:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.568312:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.568319:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.568322:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.568323:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.568328:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.568332:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a4c8 00000800:00000200:26.0:1616513087.568336:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.578256:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.578312:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.578322:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513087.578331:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.109@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.578340:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.109@tcp1 of length 224 into portal 26 MB=0x603e439c02e40 00000400:00000200:28.0:1616513087.578346:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.109@tcp1 of length 224/224 into md 0xf97cc1b9 [1] + 0 00000400:00000200:28.0:1616513087.578351:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05980 00000400:00000200:28.0:1616513087.578354:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.578356:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513087.578363:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05980 00000800:00000200:20.0:1616513087.578365:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.578366:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.578369:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.578371:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.578381:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693128616586816 00010000:00000200:8.0:1616513087.578389:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1693128616586816/t0(0) o400->53a76047-e5ef-cb5a-b0f3-55241e0edcf9@10.31.141.109@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.578402:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693128616586816, offset 192 00000400:00000200:8.0:1616513087.578408:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.109@tcp1 00000400:00000200:8.0:1616513087.578415:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.109@tcp1 routed destination 00000400:00000200:8.0:1616513087.578429:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.109@tcp1(10.31.141.109@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.578441:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.578447:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.578450:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.578452:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.578456:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.578462:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.578463:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513087.578502:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513087.578532:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513087.578540:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513087.578542:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513087.578549:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513087.578553:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513087.578555:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71d38 00000800:00000200:16.0:1616513087.578558:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513087.578563:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.578572:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.578582:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.109@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.578591:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.109@tcp1 of length 224 into portal 12 MB=0x603e439c02f40 00000400:00000200:2.0:1616513087.578597:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.109@tcp1 of length 224/224 into md 0xf964d519 [1] + 63824 00000400:00000200:2.0:1616513087.578602:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.578605:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513087.578611:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.578614:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.578616:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.578618:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.578676:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693128616587072 00010000:00000200:22.0:1616513087.578689:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cad00 x1693128616587072/t0(0) o400->1cab7f8c-c1e3-1ed6-e447-6bbe966aef13@10.31.141.109@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.578704:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693128616587072, offset 192 00000400:00000200:22.0:1616513087.578710:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.109@tcp1 00000400:00000200:22.0:1616513087.578717:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.109@tcp1 routed destination 00000400:00000200:22.0:1616513087.578732:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.109@tcp1(10.31.141.109@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.578738:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.578744:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.578747:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.578749:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.578754:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.578760:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.578817:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.578824:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.578826:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.578833:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.578836:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a110 00000800:00000200:18.0:1616513087.578840:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.578843:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.587185:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.587241:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.587251:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.587260:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.222@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.587269:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.222@tcp1 of length 224 into portal 26 MB=0x5ffd5586dc380 00000400:00000200:2.0:1616513087.587275:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.222@tcp1 of length 224/224 into md 0xf97cc1c9 [1] + 0 00000400:00000200:2.0:1616513087.587280:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05b18 00000400:00000200:2.0:1616513087.587282:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.587287:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.587291:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:14.0:1616513087.587295:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.587295:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.587297:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.587301:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.587304:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.587305:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513087.587305:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666660258688 00010000:00000200:8.0:1616513087.587312:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688666660258688/t0(0) o400->067bbb30-e69d-f368-ed29-0503e6a25f23@10.31.132.222@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513087.587314:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.222@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.587323:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.222@tcp1 of length 224 into portal 12 MB=0x5ffd5586dc400 00000100:00000200:8.0:1616513087.587323:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666660258688, offset 192 00000400:00000200:8.0:1616513087.587328:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.222@tcp1 00000400:00000200:14.0:1616513087.587329:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.222@tcp1 of length 224/224 into md 0xf964d519 [1] + 64048 00000400:00000200:8.0:1616513087.587334:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.222@tcp1 routed destination 00000400:00000200:14.0:1616513087.587335:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.587337:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.587344:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.587347:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513087.587347:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.222@tcp1(10.31.132.222@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513087.587349:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513087.587353:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.587358:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.587360:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.587363:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.587367:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.587371:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.587374:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513087.587409:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666660258816 00010000:00000200:22.0:1616513087.587420:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7a880 x1688666660258816/t0(0) o400->536b478c-120a-cc0e-910a-a03cd75192bc@10.31.132.222@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.587445:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666660258816, offset 192 00000800:00000200:20.0:1616513087.587447:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513087.587451:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.222@tcp1 00000800:00000200:20.0:1616513087.587453:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.587456:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513087.587458:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.222@tcp1 routed destination 00000400:00000200:20.0:1616513087.587463:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513087.587468:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71ba0 00000400:00000200:22.0:1616513087.587472:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.222@tcp1(10.31.132.222@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513087.587472:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.587475:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513087.587478:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.587484:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.587486:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.587488:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.587493:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.587498:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.587551:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.587558:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.587561:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513087.587562:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.587568:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.587571:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba08 00000800:00000200:26.0:1616513087.587575:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.591574:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513087.591590:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.591599:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.591608:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.137@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.591618:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.137@tcp1 of length 224 into portal 26 MB=0x603fcd6200080 00000400:00000200:18.0:1616513087.591627:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.137@tcp1 of length 224/224 into md 0xf97cc1d9 [1] + 0 00000400:00000200:18.0:1616513087.591632:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04b28 00000400:00000200:18.0:1616513087.591634:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.591637:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513087.591642:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04b28 00000800:00000200:14.0:1616513087.591644:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.591645:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.591647:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513087.591650:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:8.0:1616513087.591651:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234319327360 00000800:00000200:18.0:1616513087.591653:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.591654:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:8.0:1616513087.591656:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1693234319327360/t0(0) o400->fa1ee5b1-0787-9958-79f8-36b8e1c76be1@10.31.138.137@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513087.591663:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.137@tcp1 : PUT - for me 00000100:00000200:8.0:1616513087.591665:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234319327360, offset 192 00000400:00000200:8.0:1616513087.591669:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.137@tcp1 00000400:00000200:14.0:1616513087.591672:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.137@tcp1 of length 224 into portal 12 MB=0x603fcd6200140 00000400:00000200:8.0:1616513087.591675:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.137@tcp1 routed destination 00000400:00000200:14.0:1616513087.591679:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.137@tcp1 of length 224/224 into md 0xf964d519 [1] + 64272 00000400:00000200:14.0:1616513087.591684:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.591686:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513087.591686:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.137@tcp1(10.31.138.137@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.591691:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513087.591693:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513087.591695:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513087.591696:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513087.591697:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513087.591698:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513087.591699:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.591701:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.591705:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.591707:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513087.591762:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234319327552 00000800:00000200:0.0:1616513087.591767:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:22.0:1616513087.591774:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a78000 x1693234319327552/t0(0) o400->b141e775-cc06-4246-3e1f-b1e8ea270a0c@10.31.138.137@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513087.591774:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.591777:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.591784:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:22.0:1616513087.591787:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234319327552, offset 192 00000400:00000200:0.0:1616513087.591789:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f703b8 00000800:00000200:0.0:1616513087.591793:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513087.591794:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.137@tcp1 00000800:00000200:0.0:1616513087.591796:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513087.591802:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.137@tcp1 routed destination 00000400:00000200:22.0:1616513087.591816:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.137@tcp1(10.31.138.137@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.591822:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.591827:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.591830:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.591832:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.591836:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.591841:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.591893:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.591900:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.591903:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.591903:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.591909:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.591913:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b320 00000800:00000200:26.0:1616513087.591917:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513087.594587:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927216, portal 4 00000100:00000200:6.0:1616513087.594591:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927216, offset 0 00000400:00000200:6.0:1616513087.594597:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:6.0:1616513087.594602:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:6.0:1616513087.594612:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513087.594617:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:6.0:1616513087.594623:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513087.594625:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:6.0:1616513087.594627:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:6.0:1616513087.594631:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513087.594691:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:15.0:1616513087.594693:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927232, portal 4 00000100:00000200:15.0:1616513087.594699:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927232, offset 0 00000400:00000200:15.0:1616513087.594707:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:1.0:1616513087.594712:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:15.0:1616513087.594714:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:1.0:1616513087.594718:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513087.594721:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:15.0:1616513087.594728:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:1.0:1616513087.594729:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:15.0:1616513087.594735:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000100:00000200:1.0:1616513087.594737:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c8a2b600 x1693067725927216/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513094 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:15.0:1616513087.594742:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513087.594745:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:15.0:1616513087.594748:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:1.0:1616513087.594750:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf017e8 00000800:00000200:15.0:1616513087.594754:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:1.0:1616513087.594756:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513087.594759:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:27.2:1616513087.594802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513087.594824:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513087.594829:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513087.594832:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513087.594835:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513087.594839:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513087.594847:0:6476:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc566780 x1693067725927232/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513094 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.2:1616513087.594857:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:27.0:1616513087.594859:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0a110 00000800:00000200:27.0:1616513087.594863:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:1.0:1616513087.594867:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513087.594874:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513087.594877:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513087.594882:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513087.594891:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627330 00000400:00000200:1.0:1616513087.594898:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffe85 [1] + 192 00000400:00000200:1.0:1616513087.594903:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513087.594908:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c8a2b600 x1693067725927216/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513094 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513087.594923:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513087.594926:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513087.594930:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:6.0:1616513087.594931:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c8a2b600 x1693067725927216/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513094 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:6.0:1616513087.594944:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c8a2b600 x1693067725927216/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513094 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513087.594954:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:27.2:1616513087.594973:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513087.594984:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513087.594990:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513087.594992:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513087.594998:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513087.595006:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627340 00000400:00000200:27.0:1616513087.595012:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166e9d [1] + 192 00000400:00000200:27.0:1616513087.595017:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513087.595022:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc566780 x1693067725927232/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513094 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513087.595036:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513087.595039:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513087.595041:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:15.0:1616513087.595043:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc566780 x1693067725927232/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513094 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513087.595056:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc566780 x1693067725927232/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513094 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513087.595066:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000800:00000200:11.2:1616513087.596342:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.596397:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.596407:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513087.596416:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.106@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.596426:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.106@tcp1 of length 224 into portal 26 MB=0x60444d34c3100 00000400:00000200:20.0:1616513087.596444:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.106@tcp1 of length 224/224 into md 0xf97cc1f9 [1] + 0 00000400:00000200:20.0:1616513087.596449:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05f58 00000800:00000200:28.0:1616513087.596451:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.596452:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.596454:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513087.596459:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05f58 00000800:00000200:20.0:1616513087.596463:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.596465:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.596467:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.596474:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693543509537024 00010000:00000200:8.0:1616513087.596481:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1693543509537024/t0(0) o400->8ad19ade-2d8f-08d6-cfea-e95a92375047@10.31.143.106@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.596492:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693543509537024, offset 192 00000400:00000200:8.0:1616513087.596497:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.106@tcp1 00000400:00000200:8.0:1616513087.596503:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.106@tcp1 routed destination 00000400:00000200:8.0:1616513087.596516:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.106@tcp1(10.31.143.106@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.596522:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.596526:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.596528:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.596530:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.596534:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.596538:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.596539:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.596595:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.596602:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.596605:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.596611:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.596617:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f707f8 00000800:00000200:0.0:1616513087.596620:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.596623:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.596629:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513087.596632:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513087.596637:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.106@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.596645:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.106@tcp1 of length 224 into portal 12 MB=0x60444d34c3200 00000400:00000200:0.0:1616513087.596651:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.106@tcp1 of length 224/224 into md 0x865bbec1 [1] + 90944 00000400:00000200:0.0:1616513087.596656:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.596659:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513087.596667:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.596669:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.596671:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513087.596731:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693543509537280 00010000:00000200:4.0:1616513087.596744:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86b180 x1693543509537280/t0(0) o400->d850a506-ff15-42b9-25bb-527854cdc49a@10.31.143.106@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.596758:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693543509537280, offset 192 00000400:00000200:4.0:1616513087.596764:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.106@tcp1 00000400:00000200:4.0:1616513087.596772:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.106@tcp1 routed destination 00000400:00000200:4.0:1616513087.596787:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.106@tcp1(10.31.143.106@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.596793:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.596799:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.596802:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.596804:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.596809:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.596815:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.596867:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.596874:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.596877:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.596883:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.596887:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb6d8 00000800:00000200:14.0:1616513087.596890:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.596895:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.603127:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513087.603182:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.603191:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.603200:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.22@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.603209:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.22@tcp1 of length 224 into portal 26 MB=0x60194bf5e2580 00000400:00000200:18.0:1616513087.603216:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.22@tcp1 of length 224/224 into md 0xf97cc209 [1] + 0 00000400:00000200:18.0:1616513087.603221:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05e48 00000400:00000200:18.0:1616513087.603224:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.603226:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513087.603231:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05e48 00000800:00000200:18.0:1616513087.603234:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.603236:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.603236:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.603243:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:8.0:1616513087.603243:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588237669760 00000800:00000200:26.0:1616513087.603246:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513087.603246:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:8.0:1616513087.603249:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1690588237669760/t0(0) o400->e517e414-1553-e5ab-6fc6-3cabf83df327@10.31.141.22@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513087.603255:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.22@tcp1 : PUT - for me 00000100:00000200:8.0:1616513087.603261:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588237669760, offset 192 00000400:00000200:26.0:1616513087.603264:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.22@tcp1 of length 224 into portal 12 MB=0x60194bf5e2600 00000400:00000200:8.0:1616513087.603266:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.22@tcp1 00000400:00000200:26.0:1616513087.603270:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.22@tcp1 of length 224/224 into md 0xf964d519 [1] + 64496 00000400:00000200:8.0:1616513087.603272:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.22@tcp1 routed destination 00000400:00000200:26.0:1616513087.603276:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.603281:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513087.603285:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.22@tcp1(10.31.141.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513087.603287:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.603290:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513087.603291:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513087.603293:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513087.603296:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.603298:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.603299:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.603303:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.603307:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.603309:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513087.603366:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588237669888 00000800:00000200:20.0:1616513087.603371:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:22.0:1616513087.603377:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695a00 x1690588237669888/t0(0) o400->4b7fbfc5-282d-c8a4-b048-2c14ab931e41@10.31.141.22@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513087.603378:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.603381:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513087.603387:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:22.0:1616513087.603390:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588237669888, offset 192 00000400:00000200:20.0:1616513087.603393:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f716d8 00000400:00000200:22.0:1616513087.603395:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.22@tcp1 00000800:00000200:20.0:1616513087.603396:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.603399:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513087.603401:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.22@tcp1 routed destination 00000400:00000200:22.0:1616513087.603414:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.22@tcp1(10.31.141.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.603420:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.603425:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.603427:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.603429:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:7.2:1616513087.603440:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513087.603445:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.603494:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.603501:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.603504:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.603505:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.603510:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.603514:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aaa0 00000800:00000200:2.0:1616513087.603518:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.640517:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513087.640586:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.640595:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513087.640605:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.157@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.640615:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.157@tcp1 of length 224 into portal 26 MB=0x5ffd6ab5bfb40 00000400:00000200:26.0:1616513087.640622:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.157@tcp1 of length 224/224 into md 0xf97cc219 [1] + 0 00000400:00000200:26.0:1616513087.640627:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04110 00000400:00000200:26.0:1616513087.640630:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.640633:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513087.640637:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04110 00000800:00000200:26.0:1616513087.640641:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.640641:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513087.640642:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513087.640644:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.640645:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.640666:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672346569536 00010000:00000200:8.0:1616513087.640674:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688672346569536/t0(0) o400->03bd4a46-f4c5-61e4-e1b9-29894abc503d@10.31.138.157@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.640687:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688672346569536, offset 192 00000400:00000200:8.0:1616513087.640692:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.157@tcp1 00000400:00000200:8.0:1616513087.640699:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.157@tcp1 routed destination 00000800:00000200:16.0:1616513087.640700:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513087.640709:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513087.640713:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.157@tcp1(10.31.138.157@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.640719:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513087.640722:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.157@tcp1 : PUT - for me 00000800:00000200:8.0:1616513087.640724:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.640726:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.640728:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:16.0:1616513087.640731:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.157@tcp1 of length 224 into portal 12 MB=0x5ffd6ab5bfc00 00000800:00000200:8.0:1616513087.640732:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513087.640737:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.157@tcp1 of length 224/224 into md 0x865bbec1 [1] + 91168 00000100:00000200:8.0:1616513087.640738:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513087.640742:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513087.640745:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513087.640752:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513087.640756:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513087.640757:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:16.0:1616513087.640764:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513087.640765:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513087.640767:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.640768:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.640775:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.640780:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70a18 00000800:00000200:0.0:1616513087.640784:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513087.640818:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672346569728 00010000:00000200:4.0:1616513087.640832:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd3f00 x1688672346569728/t0(0) o400->a5f2a9f6-3394-5656-283e-1b46c178e385@10.31.138.157@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513087.640847:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688672346569728, offset 192 00000400:00000200:4.0:1616513087.640853:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.157@tcp1 00000400:00000200:4.0:1616513087.640861:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.157@tcp1 routed destination 00000400:00000200:4.0:1616513087.640876:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.157@tcp1(10.31.138.157@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513087.640883:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513087.640890:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.640893:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513087.640895:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.640900:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.640905:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.640959:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.640966:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.640969:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.640976:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.640980:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb980 00000800:00000200:18.0:1616513087.640984:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.640987:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.649710:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.649765:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.649774:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.649784:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.241@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.649793:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.241@tcp1 of length 224 into portal 26 MB=0x6022fd411b980 00000400:00000200:14.0:1616513087.649800:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.241@tcp1 of length 224/224 into md 0xf97cc249 [1] + 0 00000400:00000200:14.0:1616513087.649805:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05ba0 00000400:00000200:14.0:1616513087.649808:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.649810:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.649815:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ba0 00000800:00000200:14.0:1616513087.649818:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.649820:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.649821:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.649824:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.649829:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691254304913792 00010000:00000200:8.0:1616513087.649836:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1691254304913792/t0(0) o400->16f22125-d5ea-f24c-1afd-4b34f3691979@10.31.142.241@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.649847:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691254304913792, offset 192 00000400:00000200:8.0:1616513087.649852:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.241@tcp1 00000800:00000200:7.2:1616513087.649853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513087.649858:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.241@tcp1 routed destination 00000400:00000200:8.0:1616513087.649872:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.241@tcp1(10.31.142.241@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.649877:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.649882:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.649884:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.649886:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.649889:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.649894:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.649896:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513087.649906:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.649916:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.649925:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.241@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.649935:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.241@tcp1 of length 224 into portal 12 MB=0x6022fd411ba80 00000400:00000200:18.0:1616513087.649942:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.241@tcp1 of length 224/224 into md 0xf964d519 [1] + 64720 00000400:00000200:18.0:1616513087.649947:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.649950:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513087.649953:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513087.649957:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513087.649960:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513087.649960:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513087.649962:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.649962:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.649965:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513087.649969:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.649975:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71298 00000800:00000200:28.0:1616513087.649979:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.649982:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513087.650019:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691254304914048 00010000:00000200:22.0:1616513087.650030:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bc800 x1691254304914048/t0(0) o400->5eb691a7-4de6-86ad-e0ef-282647ac759e@10.31.142.241@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.650043:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691254304914048, offset 192 00000400:00000200:22.0:1616513087.650049:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.241@tcp1 00000400:00000200:22.0:1616513087.650056:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.241@tcp1 routed destination 00000400:00000200:22.0:1616513087.650068:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.241@tcp1(10.31.142.241@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.650074:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.650079:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.650084:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.650086:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.650091:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.650095:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.650105:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.650111:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.650113:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.650115:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.650120:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.650124:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b650 00000800:00000200:26.0:1616513087.650128:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.653967:0:5832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.654024:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.654033:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.654042:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.172@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.654052:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.172@tcp1 of length 224 into portal 26 MB=0x5ffd67e176680 00000400:00000200:0.0:1616513087.654058:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.172@tcp1 of length 224/224 into md 0xf97cc259 [1] + 0 00000400:00000200:0.0:1616513087.654063:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05dc0 00000400:00000200:0.0:1616513087.654066:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.654069:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513087.654073:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05dc0 00000800:00000200:0.0:1616513087.654076:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.654078:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.654079:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.654082:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.654083:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688671587100288 00010000:00000200:8.0:1616513087.654088:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688671587100288/t0(0) o400->19ab2877-13e8-25a4-db51-9469486041c2@10.31.129.172@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.654097:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688671587100288, offset 192 00000400:00000200:8.0:1616513087.654101:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.172@tcp1 00000400:00000200:8.0:1616513087.654107:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.172@tcp1 routed destination 00000400:00000200:8.0:1616513087.654119:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.172@tcp1(10.31.129.172@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.654124:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.654127:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.654129:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.654131:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.654134:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.654137:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.654142:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.654195:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.654202:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.654204:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513087.654211:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.654217:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71650 00000800:00000200:28.0:1616513087.654220:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.654223:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.654490:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513087.654544:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.654553:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513087.654560:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513087.654562:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.172@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.654570:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.172@tcp1 of length 224 into portal 12 MB=0x5ffd67e176800 00000800:00000200:2.0:1616513087.654570:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.654577:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.172@tcp1 of length 224/224 into md 0xf964d519 [1] + 64944 00000400:00000200:2.0:1616513087.654579:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.25@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.654582:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.654585:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:2.0:1616513087.654589:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.25@tcp1 of length 224 into portal 26 MB=0x5faa5fc631e00 00000800:00000200:18.0:1616513087.654591:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513087.654594:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513087.654595:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.25@tcp1 of length 224/224 into md 0xf97cc289 [1] + 0 00000800:00000200:18.0:1616513087.654596:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.654599:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.654600:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04330 00000400:00000200:2.0:1616513087.654603:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.654606:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.654610:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04330 00000800:00000200:2.0:1616513087.654613:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.654616:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.654618:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.654619:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682965694455296 00010000:00000200:8.0:1616513087.654623:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1682965694455296/t0(0) o400->ab17f7e0-20d5-1e00-6c03-8aef5f028977@10.31.132.25@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.654632:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1682965694455296, offset 192 00000400:00000200:8.0:1616513087.654637:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.25@tcp1 00000400:00000200:8.0:1616513087.654642:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.25@tcp1 routed destination 00000400:00000200:8.0:1616513087.654654:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.25@tcp1(10.31.132.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513087.654658:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688671587100672 00000800:00000200:8.0:1616513087.654659:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513087.654663:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.654666:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513087.654667:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513087.654670:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:22.0:1616513087.654671:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bec00 x1688671587100672/t0(0) o400->016d57c9-4602-a3e6-b2e1-dc4e707663a1@10.31.129.172@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513087.654674:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513087.654675:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513087.654686:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688671587100672, offset 192 00000400:00000200:22.0:1616513087.654692:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.172@tcp1 00000400:00000200:22.0:1616513087.654699:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.172@tcp1 routed destination 00000400:00000200:22.0:1616513087.654715:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.172@tcp1(10.31.129.172@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.654721:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513087.654727:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513087.654728:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513087.654730:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513087.654732:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513087.654735:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513087.654737:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513087.654738:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.654739:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513087.654743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513087.654745:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.654750:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71320 00000800:00000200:26.0:1616513087.654754:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513087.654754:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.654759:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.654762:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.654764:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.654769:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513087.654774:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ac38 00000800:00000200:20.0:1616513087.654778:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.654780:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.654792:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.654801:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513087.654803:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513087.654811:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.25@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.654819:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.25@tcp1 of length 224 into portal 12 MB=0x5faa5fc631f80 00000400:00000200:14.0:1616513087.654826:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.25@tcp1 of length 224/224 into md 0xf964d519 [1] + 65168 00000400:00000200:14.0:1616513087.654831:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.654833:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.654840:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.654843:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.654845:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.654852:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682965694455680 00010000:00000200:22.0:1616513087.654859:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7cc80 x1682965694455680/t0(0) o400->cc54499d-d74c-1b4f-fe61-6827baa89cb2@10.31.132.25@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.654870:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1682965694455680, offset 192 00000400:00000200:22.0:1616513087.654876:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.25@tcp1 00000400:00000200:22.0:1616513087.654882:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.25@tcp1 routed destination 00000400:00000200:22.0:1616513087.654895:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.25@tcp1(10.31.132.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.654901:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.654906:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.654908:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.654910:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.654913:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.654918:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.654926:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.654929:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.654932:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513087.654934:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.654937:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.654941:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bed0 00000800:00000200:18.0:1616513087.654944:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.655287:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.655297:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.655303:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513087.655307:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513087.655311:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.36@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.655321:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.36@tcp1 of length 224 into portal 26 MB=0x604f56797adc0 00000400:00000200:28.0:1616513087.655327:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.36@tcp1 of length 224/224 into md 0xf97cc299 [1] + 0 00000400:00000200:28.0:1616513087.655332:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04990 00000400:00000200:28.0:1616513087.655334:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.655337:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513087.655342:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:28.0:1616513087.655344:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.655347:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.655349:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.655350:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694301911756224 00010000:00000200:8.0:1616513087.655354:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1694301911756224/t0(0) o400->38f8fb29-ab73-7226-f09d-ed0cffda196d@10.31.138.36@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.655363:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694301911756224, offset 192 00000400:00000200:8.0:1616513087.655367:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.36@tcp1 00000400:00000200:8.0:1616513087.655373:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.36@tcp1 routed destination 00000400:00000200:8.0:1616513087.655384:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.36@tcp1(10.31.138.36@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.655389:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.655393:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.655395:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.655396:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.655399:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.655402:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.655403:0:5832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.655413:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.655417:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.655419:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513087.655421:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513087.655425:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.655429:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71f58 00000800:00000200:0.0:1616513087.655432:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.655684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513087.655696:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513087.655704:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513087.655706:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513087.655714:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.36@tcp1 : PUT - for me 00000400:00000200:16.0:1616513087.655723:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.36@tcp1 of length 224 into portal 12 MB=0x604f56797aec0 00000400:00000200:16.0:1616513087.655729:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.36@tcp1 of length 224/224 into md 0x865bbec1 [1] + 91392 00000400:00000200:16.0:1616513087.655734:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513087.655739:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513087.655747:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513087.655750:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513087.655752:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513087.655810:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694301911756480 00010000:00000200:12.0:1616513087.655822:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5580 x1694301911756480/t0(0) o400->70ab3673-1dcf-0ed0-ca4f-a7452abcd8bd@10.31.138.36@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.655837:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694301911756480, offset 192 00000400:00000200:12.0:1616513087.655843:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.36@tcp1 00000400:00000200:12.0:1616513087.655851:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.36@tcp1 routed destination 00000400:00000200:12.0:1616513087.655866:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.36@tcp1(10.31.138.36@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.655873:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.655879:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.655881:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.655883:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.655888:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.655892:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.655901:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.655905:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.655907:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.655909:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.655913:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.655916:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ef68 00000800:00000200:2.0:1616513087.655919:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.658112:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.658165:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.658174:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513087.658183:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.206@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.658192:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.206@tcp1 of length 224 into portal 26 MB=0x60195629b88c0 00000400:00000200:20.0:1616513087.658199:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.206@tcp1 of length 224/224 into md 0xf97cc2b9 [1] + 0 00000400:00000200:20.0:1616513087.658204:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb053a8 00000400:00000200:20.0:1616513087.658206:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.658209:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513087.658214:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb053a8 00000800:00000200:20.0:1616513087.658217:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.658218:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.658220:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513087.658225:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513087.658228:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.658228:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590976379072 00000800:00000200:0.0:1616513087.658229:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00010000:00000200:8.0:1616513087.658235:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1690590976379072/t0(0) o400->3e43cae7-1766-40b3-c6b3-2063a7708acc@10.31.140.206@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513087.658237:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.206@tcp1 : PUT - for me 00000100:00000200:8.0:1616513087.658245:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690590976379072, offset 192 00000400:00000200:0.0:1616513087.658246:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.206@tcp1 of length 224 into portal 12 MB=0x60195629b8940 00000400:00000200:8.0:1616513087.658250:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.206@tcp1 00000400:00000200:0.0:1616513087.658255:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.206@tcp1 of length 224/224 into md 0x865bbec1 [1] + 91616 00000400:00000200:8.0:1616513087.658256:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.206@tcp1 routed destination 00000400:00000200:0.0:1616513087.658262:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.658264:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513087.658277:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.206@tcp1(10.31.140.206@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513087.658281:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.658282:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513087.658284:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.658286:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513087.658287:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.658289:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.658290:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.658294:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.658299:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.658299:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513087.658332:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590976379200 00010000:00000200:12.0:1616513087.658344:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86da00 x1690590976379200/t0(0) o400->cf3b7819-614b-323a-bb2b-68d7d71e79cf@10.31.140.206@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513087.658346:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513087.658353:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513087.658356:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513087.658358:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513087.658358:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690590976379200, offset 192 00000400:00000200:24.0:1616513087.658363:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513087.658364:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.206@tcp1 00000400:00000200:24.0:1616513087.658369:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f706e8 00000400:00000200:12.0:1616513087.658372:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.206@tcp1 routed destination 00000800:00000200:24.0:1616513087.658377:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:12.0:1616513087.658387:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.206@tcp1(10.31.140.206@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.658395:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.658400:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.658403:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.658405:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.658410:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.658415:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.658498:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.658504:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.658507:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.658514:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.658519:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e220 00000800:00000200:26.0:1616513087.658523:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.658526:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.676157:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.676211:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.676221:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513087.676232:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.20@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.676241:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.20@tcp1 of length 224 into portal 26 MB=0x6018274b2a7c0 00000400:00000200:28.0:1616513087.676248:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.20@tcp1 of length 224/224 into md 0xf97cc2c9 [1] + 0 00000400:00000200:28.0:1616513087.676253:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05320 00000400:00000200:28.0:1616513087.676256:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.676259:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513087.676264:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05320 00000800:00000200:28.0:1616513087.676267:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.676269:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.676270:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.676274:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.676280:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509675505600 00010000:00000200:8.0:1616513087.676289:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1690509675505600/t0(0) o400->24103c6d-f3f7-a98d-0284-58d966ce12cd@10.31.129.20@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.676300:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690509675505600, offset 192 00000400:00000200:8.0:1616513087.676306:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.20@tcp1 00000400:00000200:8.0:1616513087.676311:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.20@tcp1 routed destination 00000400:00000200:8.0:1616513087.676323:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.20@tcp1(10.31.129.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.676329:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.676333:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.676336:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.676337:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.676341:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.676346:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.676347:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513087.676401:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.676409:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.676411:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513087.676414:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.676418:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513087.676424:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70dd0 00000800:00000200:20.0:1616513087.676428:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.676575:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.676631:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.676640:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.676649:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.20@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.676657:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.20@tcp1 of length 224 into portal 12 MB=0x6018274b2ac00 00000400:00000200:14.0:1616513087.676663:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.20@tcp1 of length 224/224 into md 0xf964d519 [1] + 65392 00000400:00000200:14.0:1616513087.676668:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.676672:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.676678:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.676681:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.676683:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.676685:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.676745:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509675506688 00010000:00000200:22.0:1616513087.676757:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a79b00 x1690509675506688/t0(0) o400->fe5b2a60-2462-1a84-6970-8a2e05032183@10.31.129.20@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.676783:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690509675506688, offset 192 00000400:00000200:22.0:1616513087.676789:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.20@tcp1 00000400:00000200:22.0:1616513087.676796:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.20@tcp1 routed destination 00000400:00000200:22.0:1616513087.676810:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.20@tcp1(10.31.129.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.676817:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.676823:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.676825:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.676827:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.676832:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.676838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.676897:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.676904:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.676906:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.676913:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.676917:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aff0 00000800:00000200:26.0:1616513087.676921:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.676924:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.684271:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.684327:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.684336:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.684345:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.136@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.684353:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.136@tcp1 of length 224 into portal 26 MB=0x603fcda7a53c0 00000400:00000200:0.0:1616513087.684360:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.136@tcp1 of length 224/224 into md 0xf97cc2e9 [1] + 0 00000400:00000200:0.0:1616513087.684365:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb057e8 00000400:00000200:0.0:1616513087.684368:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.684371:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513087.684376:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb057e8 00000800:00000200:0.0:1616513087.684379:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.684382:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.684383:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:8.0:1616513087.684390:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234392355776 00000800:00000200:0.0:1616513087.684391:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.684394:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.684395:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00010000:00000200:8.0:1616513087.684399:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1693234392355776/t0(0) o400->4a8ecdaa-218b-b5f3-6af7-5a0b97d54af0@10.31.138.136@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513087.684403:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.136@tcp1 : PUT - for me 00000100:00000200:8.0:1616513087.684411:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234392355776, offset 192 00000400:00000200:28.0:1616513087.684412:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.136@tcp1 of length 224 into portal 12 MB=0x603fcda7a5440 00000400:00000200:8.0:1616513087.684417:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.136@tcp1 00000400:00000200:28.0:1616513087.684422:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.136@tcp1 of length 224/224 into md 0x865bbec1 [1] + 91840 00000400:00000200:8.0:1616513087.684423:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.136@tcp1 routed destination 00000400:00000200:28.0:1616513087.684427:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.684430:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513087.684445:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.136@tcp1(10.31.138.136@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513087.684450:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.684451:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513087.684454:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.684456:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513087.684457:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.684459:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.684461:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.684465:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.684470:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.684471:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513087.684512:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234392355904 00010000:00000200:12.0:1616513087.684525:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5a00 x1693234392355904/t0(0) o400->9e06aa08-7662-d39f-e801-f389851e4f8e@10.31.138.136@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513087.684526:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.684533:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.684536:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.684538:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513087.684539:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234392355904, offset 192 00000400:00000200:20.0:1616513087.684543:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513087.684545:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.136@tcp1 00000400:00000200:20.0:1616513087.684550:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70440 00000400:00000200:12.0:1616513087.684552:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.136@tcp1 routed destination 00000800:00000200:20.0:1616513087.684558:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:12.0:1616513087.684567:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.136@tcp1(10.31.138.136@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.684573:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.684579:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.684582:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.684584:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.684589:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.684595:0:50283:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.684651:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.684658:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.684661:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.684667:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.684672:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f760 00000800:00000200:14.0:1616513087.684675:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.684678:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.718663:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513087.718720:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513087.718730:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513087.718740:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.18@tcp1 : PUT - for me 00000400:00000200:4.0:1616513087.718752:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.18@tcp1 of length 224 into portal 26 MB=0x5fffad6865640 00000400:00000200:4.0:1616513087.718759:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.18@tcp1 of length 224/224 into md 0xf97cc309 [1] + 0 00000400:00000200:4.0:1616513087.718764:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04a18 00000400:00000200:4.0:1616513087.718768:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513087.718771:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513087.718775:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513087.718776:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:4.0:1616513087.718779:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513087.718782:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513087.718784:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.718796:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688827689588288 00010000:00000200:8.0:1616513087.718807:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688827689588288/t0(0) o400->9a788486-162a-f537-fcd7-ebad04e51a8c@10.31.142.18@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.718821:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688827689588288, offset 192 00000400:00000200:8.0:1616513087.718828:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.18@tcp1 00000400:00000200:8.0:1616513087.718836:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.18@tcp1 routed destination 00000400:00000200:8.0:1616513087.718851:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.18@tcp1(10.31.142.18@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.718857:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.718863:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.718866:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.718868:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.718873:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.718877:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.718879:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513087.718931:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.718951:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.718953:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.718960:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.718965:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f718f8 00000800:00000200:0.0:1616513087.718969:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.718972:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.718976:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.718987:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.718996:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513087.718998:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513087.719005:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.18@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.719014:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.18@tcp1 of length 224 into portal 12 MB=0x5fffad6865840 00000400:00000200:28.0:1616513087.719020:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.18@tcp1 of length 224/224 into md 0x865bbec1 [1] + 92064 00000400:00000200:28.0:1616513087.719025:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.719028:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513087.719035:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.719038:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.719040:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513087.719100:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688827689588800 00010000:00000200:12.0:1616513087.719114:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa6c00 x1688827689588800/t0(0) o400->18a21bd5-2a62-2b4c-908d-63cdc873fa32@10.31.142.18@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.719128:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688827689588800, offset 192 00000400:00000200:12.0:1616513087.719134:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.18@tcp1 00000400:00000200:12.0:1616513087.719142:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.18@tcp1 routed destination 00000400:00000200:12.0:1616513087.719157:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.18@tcp1(10.31.142.18@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.719164:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.719170:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.719173:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.719176:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.719181:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.719187:0:50286:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.719243:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.719251:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.719254:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.719260:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.719264:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f3a8 00000800:00000200:26.0:1616513087.719268:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.719271:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.723858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.723872:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.723881:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.723890:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.246@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.723900:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.246@tcp1 of length 224 into portal 26 MB=0x603ff5f290e00 00000400:00000200:14.0:1616513087.723907:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.246@tcp1 of length 224/224 into md 0xf97cc319 [1] + 0 00000400:00000200:14.0:1616513087.723912:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04198 00000400:00000200:14.0:1616513087.723914:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.723917:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.723921:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04198 00000800:00000200:14.0:1616513087.723924:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.723925:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.723927:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.723930:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.723939:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245208333824 00010000:00000200:8.0:1616513087.723947:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1693245208333824/t0(0) o400->7a85355a-427e-b41e-2787-c4592c302eb4@10.31.132.246@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.723959:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245208333824, offset 192 00000400:00000200:8.0:1616513087.723965:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.246@tcp1 00000400:00000200:8.0:1616513087.723972:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.246@tcp1 routed destination 00000400:00000200:8.0:1616513087.723984:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.246@tcp1(10.31.132.246@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.723990:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.723995:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.723997:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.723999:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.724003:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.724009:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.724009:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513087.724063:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.724070:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.724073:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.724079:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.724085:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70dd0 00000800:00000200:0.0:1616513087.724088:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.724091:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.724172:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513087.724225:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.724234:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513087.724243:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.246@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.724251:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.246@tcp1 of length 224 into portal 12 MB=0x603ff5f290f00 00000400:00000200:26.0:1616513087.724258:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.246@tcp1 of length 224/224 into md 0xf964d519 [1] + 65616 00000400:00000200:26.0:1616513087.724263:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.724266:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513087.724274:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.724277:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.724279:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.724283:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.724339:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245208334080 00010000:00000200:22.0:1616513087.724350:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c694380 x1693245208334080/t0(0) o400->e639735e-9672-e658-d275-909e673eff2d@10.31.132.246@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.724364:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245208334080, offset 192 00000400:00000200:22.0:1616513087.724369:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.246@tcp1 00000400:00000200:22.0:1616513087.724376:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.246@tcp1 routed destination 00000400:00000200:22.0:1616513087.724390:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.246@tcp1(10.31.132.246@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.724396:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.724401:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.724404:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.724406:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.724410:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.724415:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.724425:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.724431:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.724441:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.724447:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513087.724453:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.724467:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4be48 00000800:00000200:2.0:1616513087.724471:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.725187:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.725240:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.725249:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513087.725250:0:50287:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513087.725258:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.164@tcp1 : PUT - for me 00000800:00000200:18.0:1616513087.725263:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513087.725267:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.164@tcp1 of length 224 into portal 12 MB=0x6052000ecde40 00000800:00000200:18.0:1616513087.725270:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513087.725272:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513087.725276:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.164@tcp1 of length 224/224 into md 0x865bbec1 [1] + 92288 00000400:00000200:18.0:1616513087.725279:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.164@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.725281:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.725284:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513087.725288:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.164@tcp1 of length 224 into portal 26 MB=0x6052000ecdd40 00000800:00000200:20.0:1616513087.725290:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.725293:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513087.725295:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513087.725295:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.164@tcp1 of length 224/224 into md 0xf97cc339 [1] + 0 00000800:00000200:20.0:1616513087.725298:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513087.725301:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05a08 00000400:00000200:18.0:1616513087.725304:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.725307:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513087.725311:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a08 00000800:00000200:18.0:1616513087.725314:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.725317:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.725319:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.725321:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694484872879424 00010000:00000200:8.0:1616513087.725325:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1694484872879424/t0(0) o400->1a21ec75-48ec-6f94-1ccc-4f37c1a83c64@10.31.136.164@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.725335:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694484872879424, offset 192 00000400:00000200:8.0:1616513087.725339:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.164@tcp1 00000400:00000200:8.0:1616513087.725345:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.164@tcp1 routed destination 00000100:00000200:12.0:1616513087.725355:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694484872879680 00000400:00000200:8.0:1616513087.725357:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.164@tcp1(10.31.136.164@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.725365:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513087.725367:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad3180 x1694484872879680/t0(0) o400->e291faf2-2875-ddf6-1928-34263459db77@10.31.136.164@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:8.0:1616513087.725369:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.725371:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.725372:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.725375:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.725379:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.725379:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513087.725381:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694484872879680, offset 192 00000400:00000200:12.0:1616513087.725388:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.164@tcp1 00000800:00000200:0.0:1616513087.725388:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.725392:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.725394:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513087.725395:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.164@tcp1 routed destination 00000400:00000200:0.0:1616513087.725399:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.725402:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71f58 00000800:00000200:0.0:1616513087.725405:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.725407:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:12.0:1616513087.725410:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.164@tcp1(10.31.136.164@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.725416:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.725422:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.725425:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.725427:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.725432:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.725436:0:50287:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.725446:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513087.725450:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.725452:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513087.725454:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513087.725458:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.725461:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ed48 00000800:00000200:14.0:1616513087.725464:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.730514:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513087.730541:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513087.730593:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513087.730599:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513087.730602:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513087.730608:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513087.730611:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.193@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.730617:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.193@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.730621:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.193@tcp1 of length 224 into portal 26 MB=0x605a072c12240 00000400:00000200:26.0:1616513087.730626:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.193@tcp1 of length 224 into portal 12 MB=0x605a072c12300 00000400:00000200:20.0:1616513087.730627:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.193@tcp1 of length 224/224 into md 0xf97cc369 [1] + 0 00000400:00000200:20.0:1616513087.730633:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05cb0 00000400:00000200:20.0:1616513087.730636:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.730638:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.193@tcp1 of length 224/224 into md 0xf964d519 [1] + 65840 00000400:00000200:26.0:1616513087.730645:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.730645:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513087.730648:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.730649:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05cb0 00000100:00000200:26.0:1616513087.730652:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513087.730652:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.730655:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.730657:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513087.730658:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.730661:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.730663:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513087.730663:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036538430016 00000800:00000200:26.0:1616513087.730666:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:8.0:1616513087.730670:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1695036538430016/t0(0) o400->644e70e9-a16f-6dea-ce44-518f52088c7d@10.31.129.193@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.730681:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036538430016, offset 192 00000400:00000200:8.0:1616513087.730685:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.193@tcp1 00000400:00000200:8.0:1616513087.730691:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.193@tcp1 routed destination 00000400:00000200:8.0:1616513087.730702:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.193@tcp1(10.31.129.193@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.730707:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.730712:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.730714:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.730715:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.730719:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513087.730722:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036538430208 00000800:00000200:11.2:1616513087.730723:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.730724:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:22.0:1616513087.730734:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691b00 x1695036538430208/t0(0) o400->ff6238b4-c3f4-5f27-6cbd-981f1e426abd@10.31.129.193@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.730749:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036538430208, offset 192 00000400:00000200:22.0:1616513087.730755:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.193@tcp1 00000400:00000200:22.0:1616513087.730762:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.193@tcp1 routed destination 00000400:00000200:22.0:1616513087.730777:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.193@tcp1(10.31.129.193@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.730792:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.730798:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.730800:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.730802:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.730807:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513087.730808:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513087.730812:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513087.730815:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.730817:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.730824:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.730830:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71650 00000800:00000200:0.0:1616513087.730833:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.730836:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513087.730882:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.730890:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.730892:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.730893:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.730899:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.730903:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4be48 00000800:00000200:2.0:1616513087.730907:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.737065:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.737119:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.737128:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513087.737137:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.94@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.737147:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.94@tcp1 of length 224 into portal 26 MB=0x5ffdbfaa8af40 00000400:00000200:28.0:1616513087.737154:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.94@tcp1 of length 224/224 into md 0xf97cc379 [1] + 0 00000400:00000200:28.0:1616513087.737158:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04e58 00000400:00000200:28.0:1616513087.737161:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.737164:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513087.737168:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:28.0:1616513087.737171:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.737172:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.737174:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513087.737178:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695151832896 00000800:00000200:28.0:1616513087.737180:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.737183:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.737183:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00010000:00000200:8.0:1616513087.737183:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688695151832896/t0(0) o400->36935e00-2835-6d96-4bd9-73d484fde496@10.31.141.94@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513087.737192:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.94@tcp1 : PUT - for me 00000100:00000200:8.0:1616513087.737193:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695151832896, offset 192 00000400:00000200:8.0:1616513087.737198:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.94@tcp1 00000400:00000200:20.0:1616513087.737201:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.94@tcp1 of length 224 into portal 12 MB=0x5ffdbfaa8b1c0 00000400:00000200:8.0:1616513087.737203:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.94@tcp1 routed destination 00000400:00000200:20.0:1616513087.737208:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.94@tcp1 of length 224/224 into md 0x865bbec1 [1] + 92512 00000400:00000200:20.0:1616513087.737213:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.737216:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513087.737216:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.94@tcp1(10.31.141.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.737221:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513087.737222:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.737225:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513087.737226:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.737227:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513087.737228:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.737230:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.737233:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.737237:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.737237:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513087.737246:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.737254:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.737257:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513087.737259:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513087.737264:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.737269:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70a18 00000800:00000200:0.0:1616513087.737273:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513087.737287:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695151833536 00010000:00000200:12.0:1616513087.737299:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad4c80 x1688695151833536/t0(0) o400->8aab2ab2-0d1e-cf12-1e2a-788e9e2cadd4@10.31.141.94@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.737313:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695151833536, offset 192 00000400:00000200:12.0:1616513087.737320:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.94@tcp1 00000400:00000200:12.0:1616513087.737327:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.94@tcp1 routed destination 00000400:00000200:12.0:1616513087.737343:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.94@tcp1(10.31.141.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.737349:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.737355:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.737366:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.737368:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.737373:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.737379:0:50288:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.737448:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.737454:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.737457:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.737463:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.737467:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f7e8 00000800:00000200:18.0:1616513087.737471:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.737473:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.751671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.751723:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.751732:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513087.751741:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.149@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.751751:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.149@tcp1 of length 224 into portal 12 MB=0x5ffd705ff42c0 00000400:00000200:20.0:1616513087.751757:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.149@tcp1 of length 224/224 into md 0x865bbec1 [1] + 92736 00000400:00000200:20.0:1616513087.751762:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.751764:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513087.751770:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.751773:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513087.751774:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.751777:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513087.751831:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673867219648 00010000:00000200:12.0:1616513087.751842:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0480 x1688673867219648/t0(0) o400->41cd033e-4d3d-2e8a-f892-1eba490cc3b5@10.31.129.149@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.751855:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688673867219648, offset 192 00000400:00000200:12.0:1616513087.751861:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.149@tcp1 00000400:00000200:12.0:1616513087.751868:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.149@tcp1 routed destination 00000400:00000200:12.0:1616513087.751884:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.149@tcp1(10.31.129.149@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.751890:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513087.751896:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.751898:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513087.751900:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.751905:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.751909:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513087.751928:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.751941:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.751950:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.751958:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.149@tcp1 : PUT - for me 00000800:00000200:0.0:1616513087.751960:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.751967:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513087.751968:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.149@tcp1 of length 224 into portal 26 MB=0x5ffd705ff3f40 00000400:00000200:0.0:1616513087.751969:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.751974:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.149@tcp1 of length 224/224 into md 0xf97cc389 [1] + 0 00000400:00000200:0.0:1616513087.751975:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.751979:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05870 00000400:00000200:0.0:1616513087.751979:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e660 00000400:00000200:14.0:1616513087.751982:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.751983:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:14.0:1616513087.751984:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1616513087.751986:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513087.751988:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05870 00000800:00000200:14.0:1616513087.751991:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.751994:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.751995:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.751999:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.752004:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673867218752 00010000:00000200:8.0:1616513087.752011:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688673867218752/t0(0) o400->5618fb95-116b-69ab-5e78-ce417da1648e@10.31.129.149@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.752023:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688673867218752, offset 192 00000400:00000200:8.0:1616513087.752028:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.149@tcp1 00000400:00000200:8.0:1616513087.752034:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.149@tcp1 routed destination 00000400:00000200:8.0:1616513087.752046:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.149@tcp1(10.31.129.149@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.752051:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513087.752056:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.752058:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513087.752060:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513087.752063:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513087.752068:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513087.752069:0:50290:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.752122:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.752130:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.752132:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.752138:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.752143:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f707f8 00000800:00000200:18.0:1616513087.752147:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.752149:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.760106:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.760160:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.760170:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513087.760178:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.111@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.760187:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.111@tcp1 of length 224 into portal 26 MB=0x6050b9a4825c0 00000400:00000200:20.0:1616513087.760194:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.111@tcp1 of length 224/224 into md 0xf97cc3a9 [1] + 0 00000400:00000200:20.0:1616513087.760199:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05650 00000400:00000200:20.0:1616513087.760201:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.760204:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513087.760209:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:20.0:1616513087.760212:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.760214:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.760215:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513087.760220:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694397251462592 00000800:00000200:20.0:1616513087.760221:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.760224:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513087.760224:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:8.0:1616513087.760226:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1694397251462592/t0(0) o400->85c80927-a574-eb7b-2960-546abe6bde9b@10.31.143.111@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513087.760233:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.111@tcp1 : PUT - for me 00000100:00000200:8.0:1616513087.760236:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694397251462592, offset 192 00000400:00000200:8.0:1616513087.760240:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.111@tcp1 00000400:00000200:28.0:1616513087.760242:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.111@tcp1 of length 224 into portal 12 MB=0x6050b9a482640 00000400:00000200:8.0:1616513087.760245:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.111@tcp1 routed destination 00000400:00000200:28.0:1616513087.760248:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.111@tcp1 of length 224/224 into md 0x865bbec1 [1] + 92960 00000400:00000200:28.0:1616513087.760253:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.760256:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513087.760256:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.111@tcp1(10.31.143.111@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.760261:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513087.760263:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.760265:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513087.760266:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513087.760267:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.760268:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513087.760269:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.760272:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.760276:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.760276:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513087.760284:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.760292:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.760295:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513087.760297:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513087.760302:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.760307:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71ba0 00000800:00000200:0.0:1616513087.760311:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513087.760329:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694397251462720 00010000:00000200:12.0:1616513087.760342:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa3f00 x1694397251462720/t0(0) o400->f8e0886e-5bec-c207-3b78-a9a909cf0c3f@10.31.143.111@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.760356:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694397251462720, offset 192 00000400:00000200:12.0:1616513087.760362:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.111@tcp1 00000400:00000200:12.0:1616513087.760369:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.111@tcp1 routed destination 00000400:00000200:12.0:1616513087.760384:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.111@tcp1(10.31.143.111@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.760391:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.760397:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.760399:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.760402:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.760406:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.760413:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.760467:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.760474:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.760477:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.760480:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.760484:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.760488:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa08 00000800:00000200:2.0:1616513087.760492:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.787343:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513087.787399:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.787409:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513087.787411:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513087.787418:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.141@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.787428:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.141@tcp1 of length 224 into portal 12 MB=0x5ffe431a1ac00 00000400:00000200:18.0:1616513087.787445:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.141@tcp1 of length 224/224 into md 0xf964d519 [1] + 66064 00000400:00000200:18.0:1616513087.787450:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513087.787451:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:18.0:1616513087.787453:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513087.787461:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.787464:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.787466:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513087.787472:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.787482:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513087.787491:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.141@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.787499:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.141@tcp1 of length 224 into portal 26 MB=0x5ffe431a1a9c0 00000400:00000200:28.0:1616513087.787506:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.141@tcp1 of length 224/224 into md 0xf97cc3d9 [1] + 0 00000400:00000200:28.0:1616513087.787511:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb042a8 00000400:00000200:28.0:1616513087.787514:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.787518:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513087.787523:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb042a8 00000800:00000200:28.0:1616513087.787526:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:22.0:1616513087.787528:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688730433858560 00000800:00000200:20.0:1616513087.787528:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.787529:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.787532:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.787540:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688730433857984 00010000:00000200:22.0:1616513087.787541:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645be300 x1688730433858560/t0(0) o400->479ecb3a-ca06-b5f9-70bc-2df4a596e5c8@10.31.141.141@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:8.0:1616513087.787548:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688730433857984/t0(0) o400->d2afa6e1-e1a1-7dae-c709-80daa513dcab@10.31.141.141@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513087.787556:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688730433858560, offset 192 00000100:00000200:8.0:1616513087.787560:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688730433857984, offset 192 00000400:00000200:22.0:1616513087.787562:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.141@tcp1 00000400:00000200:8.0:1616513087.787565:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.141@tcp1 00000400:00000200:22.0:1616513087.787570:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.141@tcp1 routed destination 00000400:00000200:8.0:1616513087.787571:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.141@tcp1 routed destination 00000400:00000200:8.0:1616513087.787584:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.141@tcp1(10.31.141.141@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513087.787586:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.141@tcp1(10.31.141.141@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.787590:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.787593:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.787596:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.787598:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.787599:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.787600:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.787601:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513087.787604:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.787604:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513087.787608:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.787609:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513087.787609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513087.787614:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513087.787620:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.787626:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.787629:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.787629:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513087.787635:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513087.787636:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513087.787640:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f715c8 00000400:00000200:0.0:1616513087.787640:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.787644:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.787646:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513087.787647:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.787651:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aff0 00000800:00000200:0.0:1616513087.787655:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.787658:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.812086:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.812142:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.812152:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513087.812160:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.163@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.812170:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.163@tcp1 of length 224 into portal 12 MB=0x6018332edf8c0 00000400:00000200:20.0:1616513087.812176:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.163@tcp1 of length 224/224 into md 0x865bbec1 [1] + 93184 00000400:00000200:20.0:1616513087.812182:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.812184:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513087.812191:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.812194:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.812196:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.812200:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513087.812254:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512867064000 00010000:00000200:12.0:1616513087.812267:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5e80 x1690512867064000/t0(0) o400->0662225a-96f0-364f-231f-9642d6431ca4@10.31.132.163@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.812281:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512867064000, offset 192 00000400:00000200:12.0:1616513087.812287:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.163@tcp1 00000400:00000200:12.0:1616513087.812295:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.163@tcp1 routed destination 00000400:00000200:12.0:1616513087.812310:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.163@tcp1(10.31.132.163@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.812316:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513087.812322:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.812325:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513087.812329:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.812334:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.812339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.812393:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.812400:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.812402:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.812409:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.812413:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f540 00000800:00000200:0.0:1616513087.812418:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.812420:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.813567:0:50296:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513087.813624:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.813633:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513087.813642:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.163@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.813651:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.163@tcp1 of length 224 into portal 26 MB=0x6018332edf6c0 00000400:00000200:26.0:1616513087.813658:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.163@tcp1 of length 224/224 into md 0xf97cc3e9 [1] + 0 00000400:00000200:26.0:1616513087.813662:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05ed0 00000400:00000200:26.0:1616513087.813665:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.813668:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513087.813672:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:26.0:1616513087.813675:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.813678:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.813679:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.813682:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.813689:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512867063488 00010000:00000200:8.0:1616513087.813698:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1690512867063488/t0(0) o400->cdc49e0f-0944-6a9a-f97a-dd07bb3d30ce@10.31.132.163@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.813711:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512867063488, offset 192 00000400:00000200:8.0:1616513087.813716:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.163@tcp1 00000400:00000200:8.0:1616513087.813723:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.163@tcp1 routed destination 00000400:00000200:8.0:1616513087.813737:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.163@tcp1(10.31.132.163@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.813744:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513087.813749:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.813752:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513087.813754:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513087.813758:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.813762:0:50296:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513087.813764:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513087.813816:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.813823:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.813826:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.813833:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.813838:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71980 00000800:00000200:14.0:1616513087.813842:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.813845:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.823903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.823961:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.823971:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513087.823980:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.88@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.823988:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.88@tcp1 of length 224 into portal 26 MB=0x5f90cd150f080 00000400:00000200:28.0:1616513087.823995:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.88@tcp1 of length 224/224 into md 0xf97cc409 [1] + 0 00000400:00000200:28.0:1616513087.824000:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04ee0 00000400:00000200:28.0:1616513087.824003:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.824006:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513087.824010:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04ee0 00000800:00000200:28.0:1616513087.824013:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.824015:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.824016:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.824022:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513087.824025:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.824026:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513087.824034:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.88@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.824043:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.88@tcp1 of length 224 into portal 12 MB=0x5f90cd150f100 00000400:00000200:20.0:1616513087.824049:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.88@tcp1 of length 224/224 into md 0x865bbec1 [1] + 93408 00000400:00000200:20.0:1616513087.824054:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.824057:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513087.824063:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.824066:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.824068:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.824073:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681208330219648 00010000:00000200:8.0:1616513087.824084:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1681208330219648/t0(0) o400->4447f8e5-a73d-e40a-e120-7f7be8194975@10.31.131.88@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.824099:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681208330219648, offset 192 00000400:00000200:8.0:1616513087.824105:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.88@tcp1 00000400:00000200:8.0:1616513087.824112:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.88@tcp1 routed destination 00000100:00000200:12.0:1616513087.824128:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681208330219776 00000400:00000200:8.0:1616513087.824128:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.88@tcp1(10.31.131.88@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.824135:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513087.824140:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0900 x1681208330219776/t0(0) o400->0fa972da-887a-de2a-5817-b5cdc012c24b@10.31.131.88@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:8.0:1616513087.824141:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.824143:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.824146:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.824150:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513087.824154:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681208330219776, offset 192 00000800:00000200:11.2:1616513087.824155:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.824157:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513087.824161:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.88@tcp1 00000400:00000200:12.0:1616513087.824168:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.88@tcp1 routed destination 00000400:00000200:12.0:1616513087.824183:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.88@tcp1(10.31.131.88@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.824189:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.824195:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.824197:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.824199:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.824204:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.824212:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.824227:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513087.824234:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513087.824237:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513087.824238:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513087.824244:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513087.824250:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71980 00000800:00000200:4.0:1616513087.824254:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513087.824281:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.824287:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.824289:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.824296:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.824300:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e000 00000800:00000200:26.0:1616513087.824303:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.824306:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.824630:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513087.824630:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513087.824640:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.824646:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513087.824650:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.824653:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.212@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.824660:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.212@tcp1 of length 224 into portal 12 MB=0x5ffd561267780 00000400:00000200:20.0:1616513087.824665:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.212@tcp1 of length 224/224 into md 0x865bbec1 [1] + 93632 00000400:00000200:20.0:1616513087.824669:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.824671:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513087.824675:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.824677:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.824679:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513087.824684:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513087.824687:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666806581120 00000800:00000200:14.0:1616513087.824694:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00010000:00000200:12.0:1616513087.824694:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad2400 x1688666806581120/t0(0) o400->3b8d85b5-0851-bfdf-7ebf-694d5e586f65@10.31.132.212@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513087.824703:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.212@tcp1 : PUT - for me 00000100:00000200:12.0:1616513087.824705:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666806581120, offset 192 00000400:00000200:12.0:1616513087.824710:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.212@tcp1 00000400:00000200:14.0:1616513087.824712:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.212@tcp1 of length 224 into portal 26 MB=0x5ffd561267700 00000400:00000200:12.0:1616513087.824716:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.212@tcp1 routed destination 00000400:00000200:14.0:1616513087.824718:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.212@tcp1 of length 224/224 into md 0xf97cc419 [1] + 0 00000400:00000200:14.0:1616513087.824723:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04770 00000400:00000200:14.0:1616513087.824726:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.824728:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.824738:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000400:00000200:12.0:1616513087.824740:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.212@tcp1(10.31.132.212@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513087.824741:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.824743:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513087.824745:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513087.824746:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.824748:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513087.824750:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:8.0:1616513087.824750:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666806580992 00000800:00000200:12.0:1616513087.824752:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513087.824753:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:8.0:1616513087.824755:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688666806580992/t0(0) o400->f7dfd410-c0cb-b3a6-8bdf-5accd0022ad9@10.31.132.212@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513087.824756:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.824761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.824766:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666806580992, offset 192 00000800:00000200:28.0:1616513087.824770:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:8.0:1616513087.824771:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.212@tcp1 00000800:00000200:28.0:1616513087.824773:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.824776:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513087.824776:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.212@tcp1 routed destination 00000800:00000200:4.0:1616513087.824778:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513087.824781:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.824784:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ea18 00000800:00000200:28.0:1616513087.824786:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513087.824788:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.212@tcp1(10.31.132.212@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.824794:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513087.824798:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.824801:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513087.824802:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513087.824805:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513087.824809:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513087.824810:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.824818:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.824822:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.824824:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.824828:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.824829:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.824833:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71ba0 00000800:00000200:26.0:1616513087.824836:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.849974:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.850032:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.850042:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.850051:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.253@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.850060:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.253@tcp1 of length 224 into portal 12 MB=0x601981b25df40 00000400:00000200:0.0:1616513087.850066:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.253@tcp1 of length 224/224 into md 0x865bbec1 [1] + 93856 00000400:00000200:0.0:1616513087.850071:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.850074:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513087.850081:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.850084:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513087.850085:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.850088:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.850099:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513087.850107:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.253@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.850116:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.253@tcp1 of length 224 into portal 26 MB=0x601981b25dec0 00000400:00000200:20.0:1616513087.850123:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.253@tcp1 of length 224/224 into md 0xf97cc429 [1] + 0 00000400:00000200:20.0:1616513087.850128:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05100 00000400:00000200:20.0:1616513087.850131:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.850133:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513087.850138:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000800:00000200:20.0:1616513087.850142:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:12.0:1616513087.850143:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690602662387520 00000800:00000200:20.0:1616513087.850145:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513087.850146:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513087.850151:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:12.0:1616513087.850155:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86e300 x1690602662387520/t0(0) o400->67b81bfe-6e20-c000-e1cc-523a5f3f6d60@10.31.141.253@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513087.850157:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690602662387392 00010000:00000200:8.0:1616513087.850166:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1690602662387392/t0(0) o400->31571327-bec4-aa3b-b228-4162f9bc152b@10.31.141.253@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513087.850168:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690602662387520, offset 192 00000400:00000200:12.0:1616513087.850175:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.253@tcp1 00000100:00000200:8.0:1616513087.850179:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690602662387392, offset 192 00000400:00000200:12.0:1616513087.850182:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.253@tcp1 routed destination 00000400:00000200:8.0:1616513087.850184:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.253@tcp1 00000400:00000200:12.0:1616513087.850195:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.253@tcp1(10.31.141.253@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.850202:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:8.0:1616513087.850203:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.253@tcp1 routed destination 00000800:00000200:12.0:1616513087.850210:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.850213:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513087.850215:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513087.850216:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.253@tcp1(10.31.141.253@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.850219:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513087.850222:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513087.850224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.850228:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.850231:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513087.850232:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513087.850236:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.850237:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513087.850242:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513087.850243:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513087.850243:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513087.850245:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.850247:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513087.850251:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.850255:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eb28 00000800:00000200:28.0:1616513087.850259:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513087.850298:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.850305:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.850308:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513087.850314:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.850320:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70a18 00000800:00000200:2.0:1616513087.850324:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.850327:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.854771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.854825:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.854835:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513087.854840:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513087.854844:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.61@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.854853:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.61@tcp1 of length 224 into portal 26 MB=0x60189ef392840 00000400:00000200:20.0:1616513087.854860:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.61@tcp1 of length 224/224 into md 0xf97cc459 [1] + 0 00000400:00000200:20.0:1616513087.854865:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04220 00000400:00000200:20.0:1616513087.854868:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.854871:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513087.854875:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:20.0:1616513087.854878:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.854881:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513087.854883:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.854886:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690541795911744 00010000:00000200:8.0:1616513087.854892:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1690541795911744/t0(0) o400->265bfc91-bb60-4527-febf-050570e310fe@10.31.128.61@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.854903:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690541795911744, offset 192 00000400:00000200:8.0:1616513087.854908:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.61@tcp1 00000400:00000200:8.0:1616513087.854914:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.61@tcp1 routed destination 00000400:00000200:8.0:1616513087.854927:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.61@tcp1(10.31.128.61@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.854933:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.854938:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.854940:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.854942:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.854945:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.854949:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.854950:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.855003:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.855010:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.855013:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.855019:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.855025:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71f58 00000800:00000200:0.0:1616513087.855028:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.855031:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.855124:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513087.855135:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513087.855143:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513087.855145:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513087.855152:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.61@tcp1 : PUT - for me 00000400:00000200:4.0:1616513087.855161:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.61@tcp1 of length 224 into portal 12 MB=0x60189ef392c00 00000400:00000200:4.0:1616513087.855167:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.61@tcp1 of length 224/224 into md 0x865bbec1 [1] + 94080 00000400:00000200:4.0:1616513087.855173:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513087.855175:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513087.855182:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513087.855185:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513087.855187:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513087.855260:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690541795912704 00010000:00000200:12.0:1616513087.855273:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23ba80 x1690541795912704/t0(0) o400->9f08772d-69a9-8b1d-c1b5-ce37aa23d8d6@10.31.128.61@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.855287:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690541795912704, offset 192 00000400:00000200:12.0:1616513087.855294:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.61@tcp1 00000400:00000200:12.0:1616513087.855302:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.61@tcp1 routed destination 00000400:00000200:12.0:1616513087.855317:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.61@tcp1(10.31.128.61@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.855324:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.855329:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.855332:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.855334:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.855339:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.855364:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.855382:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.855388:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.855393:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513087.855400:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.855404:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777edd0 00000800:00000200:18.0:1616513087.855408:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.855411:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:31.0:1616513087.871548:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513087.871556:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:7.2:1616513087.880887:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.880943:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.880953:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513087.880960:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.880963:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.231@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.880973:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.231@tcp1 of length 224 into portal 26 MB=0x5ffd4478ef200 00000400:00000200:2.0:1616513087.880979:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.231@tcp1 of length 224/224 into md 0xf97cc469 [1] + 0 00000400:00000200:2.0:1616513087.880984:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04880 00000400:00000200:2.0:1616513087.880989:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.880992:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.880996:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:2.0:1616513087.881000:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.881003:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.881005:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.881015:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662082253312 00010000:00000200:8.0:1616513087.881024:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688662082253312/t0(0) o400->3b44568e-fc9c-1eba-a4b7-29147ec74927@10.31.132.231@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.881036:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662082253312, offset 192 00000400:00000200:8.0:1616513087.881042:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.231@tcp1 00000400:00000200:8.0:1616513087.881049:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.231@tcp1 routed destination 00000400:00000200:8.0:1616513087.881062:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.231@tcp1(10.31.132.231@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.881068:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.881073:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.881075:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.881077:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.881082:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.881087:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.881088:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.881143:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.881150:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.881152:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.881161:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.881167:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f718f8 00000800:00000200:0.0:1616513087.881170:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.881173:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.881549:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513087.881603:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513087.881612:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513087.881621:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.231@tcp1 : PUT - for me 00000400:00000200:20.0:1616513087.881630:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.231@tcp1 of length 224 into portal 12 MB=0x5ffd4478ef3c0 00000400:00000200:20.0:1616513087.881637:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.231@tcp1 of length 224/224 into md 0x865bbec1 [1] + 94304 00000400:00000200:20.0:1616513087.881642:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513087.881644:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513087.881651:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513087.881654:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513087.881655:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513087.881659:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513087.881715:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662082253760 00010000:00000200:12.0:1616513087.881727:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad6780 x1688662082253760/t0(0) o400->a82c0776-1eed-74f5-24e5-129dfd37214a@10.31.132.231@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.881742:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662082253760, offset 192 00000400:00000200:12.0:1616513087.881748:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.231@tcp1 00000400:00000200:12.0:1616513087.881755:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.231@tcp1 routed destination 00000400:00000200:12.0:1616513087.881770:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.231@tcp1(10.31.132.231@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.881777:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.881783:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.881787:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.881789:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.881794:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.881799:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.881854:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.881862:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.881864:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513087.881867:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.881871:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.881875:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ee58 00000800:00000200:18.0:1616513087.881879:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.890551:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513087.890612:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.890622:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.890630:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.134@tcp1 : PUT - for me 00000800:00000200:28.0:1616513087.890633:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513087.890640:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.134@tcp1 of length 224 into portal 26 MB=0x5ffd4e25f61c0 00000800:00000200:28.0:1616513087.890643:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513087.890646:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.134@tcp1 of length 224/224 into md 0xf97cc479 [1] + 0 00000400:00000200:0.0:1616513087.890651:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42cc0 00000400:00000200:28.0:1616513087.890654:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.134@tcp1 : PUT - for me 00000400:00000200:0.0:1616513087.890654:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513087.890656:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513087.890661:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42cc0 00000400:00000200:28.0:1616513087.890664:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.134@tcp1 of length 224 into portal 12 MB=0x5ffd4e25f6580 00000800:00000200:0.0:1616513087.890664:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513087.890667:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513087.890669:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513087.890670:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.134@tcp1 of length 224/224 into md 0x865bbec1 [1] + 94528 00000800:00000200:0.0:1616513087.890671:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513087.890675:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513087.890677:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664679604672 00000100:00000200:28.0:1616513087.890678:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513087.890685:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:8.0:1616513087.890686:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688664679604672/t0(0) o400->26104676-9147-ef36-082b-d9e36a4c2515@10.31.136.134@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513087.890688:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.890690:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513087.890698:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664679604672, offset 192 00000400:00000200:8.0:1616513087.890703:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.134@tcp1 00000400:00000200:8.0:1616513087.890710:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.134@tcp1 routed destination 00000400:00000200:8.0:1616513087.890722:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.134@tcp1(10.31.136.134@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.890727:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.890732:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.890735:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.890736:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.890741:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.890745:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.890746:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513087.890757:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664679605632 00010000:00000200:12.0:1616513087.890769:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5e80 x1688664679605632/t0(0) o400->3ca1f647-61f7-7427-983d-46f431e1d915@10.31.136.134@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.890784:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664679605632, offset 192 00000400:00000200:12.0:1616513087.890790:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.134@tcp1 00000800:00000200:20.0:1616513087.890798:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513087.890798:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.134@tcp1 routed destination 00000800:00000200:20.0:1616513087.890806:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513087.890808:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513087.890810:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513087.890813:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.134@tcp1(10.31.136.134@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513087.890815:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:12.0:1616513087.890820:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513087.890821:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70a18 00000800:00000200:20.0:1616513087.890824:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513087.890825:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.890828:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.890830:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.890835:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.890841:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.890898:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.890905:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.890908:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513087.890915:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.890918:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e198 00000800:00000200:14.0:1616513087.890922:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.890925:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.892178:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513087.892235:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513087.892244:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513087.892253:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.101@tcp1 : PUT - for me 00000400:00000200:4.0:1616513087.892263:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.101@tcp1 of length 224 into portal 26 MB=0x60215a5993600 00000400:00000200:4.0:1616513087.892271:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.101@tcp1 of length 224/224 into md 0xf97cc489 [1] + 0 00000400:00000200:4.0:1616513087.892276:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5b18 00000400:00000200:4.0:1616513087.892278:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513087.892281:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513087.892286:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5b18 00000800:00000200:4.0:1616513087.892289:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513087.892292:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513087.892292:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513087.892294:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691141856114176 00000800:00000200:4.0:1616513087.892296:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513087.892299:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1691141856114176/t0(0) o400->4202d7de-78f7-cd89-5099-83a871583dbc@10.31.140.101@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.892308:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691141856114176, offset 192 00000400:00000200:8.0:1616513087.892312:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.101@tcp1 00000400:00000200:8.0:1616513087.892317:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.101@tcp1 routed destination 00000400:00000200:8.0:1616513087.892328:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.101@tcp1(10.31.140.101@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.892333:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.892337:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.892339:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.892340:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.892342:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.892346:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.892346:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513087.892400:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.892407:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.892410:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.892416:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.892422:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71980 00000800:00000200:0.0:1616513087.892426:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.892429:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.892916:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513087.892927:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.892934:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513087.892936:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513087.892942:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.101@tcp1 : PUT - for me 00000400:00000200:28.0:1616513087.892951:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.101@tcp1 of length 224 into portal 12 MB=0x60215a5993bc0 00000400:00000200:28.0:1616513087.892957:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.101@tcp1 of length 224/224 into md 0x865bbec1 [1] + 94752 00000400:00000200:28.0:1616513087.892962:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513087.892964:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513087.892970:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513087.892973:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513087.892974:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513087.893034:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691141856115648 00010000:00000200:12.0:1616513087.893046:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa1f80 x1691141856115648/t0(0) o400->8495e1cd-1aa5-f6da-c35b-0a6b2f734adf@10.31.140.101@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.893060:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691141856115648, offset 192 00000400:00000200:12.0:1616513087.893067:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.101@tcp1 00000400:00000200:12.0:1616513087.893074:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.101@tcp1 routed destination 00000400:00000200:12.0:1616513087.893089:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.101@tcp1(10.31.140.101@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.893096:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.893102:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.893104:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.893107:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.893111:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.893116:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.893168:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.893175:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.893178:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513087.893184:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.893188:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e880 00000800:00000200:26.0:1616513087.893192:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.893195:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.901303:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.901357:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.901367:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.901376:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.162@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.901385:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.162@tcp1 of length 224 into portal 26 MB=0x60468b2780c80 00000400:00000200:14.0:1616513087.901391:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.162@tcp1 of length 224/224 into md 0xf97cc4a9 [1] + 0 00000400:00000200:14.0:1616513087.901396:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5298 00000400:00000200:14.0:1616513087.901399:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.901402:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.901406:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5298 00000800:00000200:14.0:1616513087.901409:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.901411:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.901412:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.901418:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.901421:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513087.901422:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693697577585792 00000800:00000200:2.0:1616513087.901422:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:8.0:1616513087.901429:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1693697577585792/t0(0) o400->4b8f493b-1aba-e621-8a74-393a6022bd87@10.31.140.162@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513087.901431:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.162@tcp1 : PUT - for me 00000100:00000200:8.0:1616513087.901446:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693697577585792, offset 192 00000400:00000200:8.0:1616513087.901452:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.162@tcp1 00000400:00000200:8.0:1616513087.901458:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.162@tcp1 routed destination 00000400:00000200:2.0:1616513087.901458:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.162@tcp1 of length 224 into portal 12 MB=0x60468b2780d00 00000400:00000200:2.0:1616513087.901465:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.162@tcp1 of length 224/224 into md 0xf964d519 [1] + 66288 00000400:00000200:8.0:1616513087.901470:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.162@tcp1(10.31.140.162@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513087.901470:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.901473:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513087.901475:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.901480:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513087.901480:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513087.901482:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.901484:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513087.901484:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.901486:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513087.901488:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.901493:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.901494:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513087.901516:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513087.901522:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513087.901524:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513087.901531:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513087.901536:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f718f8 00000800:00000200:0.0:1616513087.901539:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513087.901542:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513087.901552:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693697577585920 00010000:00000200:22.0:1616513087.901564:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cf500 x1693697577585920/t0(0) o400->3ba4dd4a-3687-9309-c2b6-724a2df8877f@10.31.140.162@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.901579:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693697577585920, offset 192 00000400:00000200:22.0:1616513087.901585:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.162@tcp1 00000400:00000200:22.0:1616513087.901592:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.162@tcp1 routed destination 00000400:00000200:22.0:1616513087.901607:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.162@tcp1(10.31.140.162@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.901613:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.901619:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.901622:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.901624:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.901629:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.901633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.901686:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.901693:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.901695:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.901697:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.901702:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.901706:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bed0 00000800:00000200:18.0:1616513087.901710:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.934752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513087.934807:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.934817:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513087.934826:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.208@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.934838:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.208@tcp1 of length 224 into portal 26 MB=0x6005057101640 00000400:00000200:26.0:1616513087.934845:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.208@tcp1 of length 224/224 into md 0xf97cc4b9 [1] + 0 00000400:00000200:26.0:1616513087.934850:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4dd0 00000400:00000200:26.0:1616513087.934853:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.934856:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513087.934861:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4dd0 00000800:00000200:2.0:1616513087.934862:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.934865:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.934867:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.934869:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513087.934882:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689194918319680 00010000:00000200:8.0:1616513087.934892:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1689194918319680/t0(0) o400->3d6ce07d-d5c6-1f16-698a-aa463dc08e15@10.31.141.208@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513087.934901:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:8.0:1616513087.934905:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689194918319680, offset 192 00000400:00000200:8.0:1616513087.934911:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.208@tcp1 00000400:00000200:8.0:1616513087.934918:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.208@tcp1 routed destination 00000400:00000200:8.0:1616513087.934932:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.208@tcp1(10.31.141.208@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.934938:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.934944:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.934947:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.934948:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.934953:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513087.934959:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513087.934960:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513087.934968:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.934978:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.934987:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.208@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.934996:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.208@tcp1 of length 224 into portal 12 MB=0x6005057101880 00000400:00000200:14.0:1616513087.935002:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.208@tcp1 of length 224/224 into md 0xf964d519 [1] + 66512 00000400:00000200:14.0:1616513087.935008:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.935010:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513087.935012:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513087.935017:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513087.935019:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513087.935021:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.935023:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.935025:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.935028:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513087.935028:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513087.935033:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f718f8 00000800:00000200:4.0:1616513087.935037:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513087.935040:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513087.935085:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689194918320256 00010000:00000200:22.0:1616513087.935099:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7e780 x1689194918320256/t0(0) o400->b5c97707-d6a0-5b83-3075-f3fcae2d131a@10.31.141.208@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.935113:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689194918320256, offset 192 00000400:00000200:22.0:1616513087.935119:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.208@tcp1 00000400:00000200:22.0:1616513087.935127:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.208@tcp1 routed destination 00000400:00000200:22.0:1616513087.935142:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.208@tcp1(10.31.141.208@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.935149:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.935155:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.935158:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.935160:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.935166:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.935170:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.935179:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.935185:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.935187:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513087.935189:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.935194:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.935198:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ac38 00000800:00000200:2.0:1616513087.935202:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.960614:0:50313:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513087.960669:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.960680:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.960689:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.149@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.960700:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.149@tcp1 of length 224 into portal 26 MB=0x5f92179529540 00000400:00000200:18.0:1616513087.960706:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.149@tcp1 of length 224/224 into md 0xf97cc4d9 [1] + 0 00000400:00000200:18.0:1616513087.960711:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5d38 00000400:00000200:18.0:1616513087.960714:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.960717:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513087.960722:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5d38 00000800:00000200:18.0:1616513087.960725:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.960726:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.960729:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.960733:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513087.960736:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.960737:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513087.960744:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681297048245568 00000400:00000200:14.0:1616513087.960746:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.149@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.960755:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.149@tcp1 of length 224 into portal 12 MB=0x5f921795295c0 00010000:00000200:8.0:1616513087.960755:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1681297048245568/t0(0) o400->2b2cbb3a-3408-97f3-5f4a-afaf1d256b10@10.31.132.149@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513087.960762:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.149@tcp1 of length 224/224 into md 0xf964d519 [1] + 66736 00000400:00000200:14.0:1616513087.960767:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513087.960768:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681297048245568, offset 192 00000100:00000200:14.0:1616513087.960770:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513087.960774:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.149@tcp1 00000800:00000200:14.0:1616513087.960778:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513087.960781:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.149@tcp1 routed destination 00000800:00000200:14.0:1616513087.960782:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.960784:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513087.960795:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.149@tcp1(10.31.132.149@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.960804:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.960810:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.960813:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.960815:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.960820:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.960827:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513087.960848:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681297048245696 00000800:00000200:11.2:1616513087.960861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513087.960862:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7f980 x1681297048245696/t0(0) o400->aa3f76bc-80f5-d68a-5250-afcec6142ea8@10.31.132.149@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.960877:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681297048245696, offset 192 00000400:00000200:22.0:1616513087.960884:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.149@tcp1 00000400:00000200:22.0:1616513087.960891:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.149@tcp1 routed destination 00000400:00000200:22.0:1616513087.960907:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.149@tcp1(10.31.132.149@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.960913:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.960920:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.960922:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.960924:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.960930:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.960934:0:50313:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.960987:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.960994:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.960997:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.960998:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.961004:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.961008:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b650 00000800:00000200:2.0:1616513087.961012:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513087.964450:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.964454:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513087.964456:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513087.964461:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513087.964464:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000800:00000200:8.0:1616513087.964467:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513087.964469:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.969050:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513087.969103:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.969112:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513087.969120:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.57@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.969129:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.57@tcp1 of length 224 into portal 26 MB=0x6002fb69c59c0 00000400:00000200:26.0:1616513087.969136:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.57@tcp1 of length 224/224 into md 0xf97cc4f9 [1] + 0 00000400:00000200:26.0:1616513087.969140:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb43b8 00000400:00000200:26.0:1616513087.969142:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.969145:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513087.969149:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb43b8 00000800:00000200:26.0:1616513087.969152:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.969154:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.969156:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513087.969158:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513087.969164:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689054787426752 00000800:00000200:14.0:1616513087.969169:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:8.0:1616513087.969172:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1689054787426752/t0(0) o400->f686928d-0a13-c547-0523-25f3da76db09@10.31.133.57@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513087.969177:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.57@tcp1 : PUT - for me 00000100:00000200:8.0:1616513087.969183:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689054787426752, offset 192 00000400:00000200:14.0:1616513087.969186:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.57@tcp1 of length 224 into portal 12 MB=0x6002fb69c5a40 00000400:00000200:8.0:1616513087.969188:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.57@tcp1 00000400:00000200:14.0:1616513087.969192:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.57@tcp1 of length 224/224 into md 0xf964d519 [1] + 66960 00000400:00000200:8.0:1616513087.969194:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.57@tcp1 routed destination 00000400:00000200:14.0:1616513087.969197:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.969200:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.969206:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:8.0:1616513087.969206:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.57@tcp1(10.31.133.57@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513087.969209:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.969211:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513087.969212:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513087.969214:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513087.969217:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.969219:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.969220:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.969224:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.969229:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.969265:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513087.969273:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689054787426880 00000800:00000200:4.0:1616513087.969278:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513087.969284:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:22.0:1616513087.969286:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a79f80 x1689054787426880/t0(0) o400->081911d1-a6cf-b3dd-99dc-ab6df0408edd@10.31.133.57@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:4.0:1616513087.969287:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513087.969293:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513087.969298:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000100:00000200:22.0:1616513087.969301:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689054787426880, offset 192 00000800:00000200:4.0:1616513087.969301:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513087.969304:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513087.969308:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.57@tcp1 00000800:00000200:11.2:1616513087.969311:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513087.969315:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.57@tcp1 routed destination 00000800:00000200:8.0:1616513087.969320:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.969325:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.969327:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.969329:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513087.969330:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513087.969332:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.57@tcp1(10.31.133.57@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.969332:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513087.969338:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.969344:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.969346:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.969349:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.969353:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.969357:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.969410:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.969417:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513087.969420:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513087.969421:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513087.969426:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513087.969430:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aaa0 00000800:00000200:2.0:1616513087.969448:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.974378:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513087.974388:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513087.974393:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513087.974400:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.158@tcp1 : PUT - for me 00000400:00000200:4.0:1616513087.974409:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.158@tcp1 of length 224 into portal 26 MB=0x603e14377d380 00000400:00000200:4.0:1616513087.974414:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.158@tcp1 of length 224/224 into md 0xf97cc509 [1] + 0 00000400:00000200:4.0:1616513087.974419:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4bb0 00000400:00000200:4.0:1616513087.974421:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513087.974423:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513087.974427:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4bb0 00000800:00000200:4.0:1616513087.974430:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513087.974433:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513087.974435:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.974442:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115894715264 00000800:00000200:4.0:1616513087.974443:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513087.974447:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1693115894715264/t0(0) o400->cbbf02f3-5f8e-d5fe-a3c5-85ee16ed3f1c@10.31.136.158@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513087.974456:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693115894715264, offset 192 00000400:00000200:8.0:1616513087.974460:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.158@tcp1 00000400:00000200:8.0:1616513087.974466:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.158@tcp1 routed destination 00000400:00000200:8.0:1616513087.974477:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.158@tcp1(10.31.136.158@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513087.974482:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513087.974486:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513087.974488:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.974489:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.974492:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513087.974495:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.974496:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.974506:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.974510:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513087.974511:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513087.974520:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513087.974522:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:8.0:1616513087.974524:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513087.974526:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513087.974589:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513087.974596:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513087.974601:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513087.974606:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513087.974608:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.158@tcp1 : PUT - for me 00000400:00000200:4.0:1616513087.974613:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.158@tcp1 of length 224 into portal 12 MB=0x603e14377d480 00000400:00000200:4.0:1616513087.974618:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.158@tcp1 of length 224/224 into md 0x865bbec1 [1] + 94976 00000400:00000200:4.0:1616513087.974622:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513087.974624:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513087.974629:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513087.974631:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513087.974632:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513087.974652:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115894715520 00010000:00000200:12.0:1616513087.974664:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23f500 x1693115894715520/t0(0) o400->1bb7ee79-2cb0-4e74-60d7-1a723721b766@10.31.136.158@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.974689:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693115894715520, offset 192 00000400:00000200:12.0:1616513087.974695:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.158@tcp1 00000400:00000200:12.0:1616513087.974702:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.158@tcp1 routed destination 00000400:00000200:12.0:1616513087.974717:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.158@tcp1(10.31.136.158@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.974724:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.974730:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.974733:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.974735:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513087.974739:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.974744:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.974803:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.974810:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513087.974813:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513087.974816:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513087.974819:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513087.974823:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ebb0 00000800:00000200:14.0:1616513087.974827:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.984018:0:50317:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513087.984069:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.984078:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513087.984086:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.71@tcp1 : PUT - for me 00000400:00000200:26.0:1616513087.984095:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.71@tcp1 of length 224 into portal 12 MB=0x5ffe07cce8f80 00000400:00000200:26.0:1616513087.984100:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.71@tcp1 of length 224/224 into md 0xf964d519 [1] + 67184 00000400:00000200:26.0:1616513087.984105:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513087.984107:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513087.984114:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.984116:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513087.984118:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513087.984120:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513087.984136:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513087.984146:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.984151:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513087.984157:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.71@tcp1 : PUT - for me 00000400:00000200:8.0:1616513087.984164:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.71@tcp1 of length 224 into portal 26 MB=0x5ffe07cce8c00 00000400:00000200:8.0:1616513087.984169:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.71@tcp1 of length 224/224 into md 0xf97cc529 [1] + 0 00000400:00000200:8.0:1616513087.984173:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb57e8 00000400:00000200:8.0:1616513087.984175:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513087.984176:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714515222400 00000100:00000200:8.0:1616513087.984177:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513087.984181:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb57e8 00000800:00000200:8.0:1616513087.984183:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.984185:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513087.984186:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513087.984188:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c696c00 x1688714515222400/t0(0) o400->0fc194ef-7336-b918-565b-9593cf4c370e@10.31.129.71@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:8.0:1616513087.984189:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513087.984201:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688714515222400, offset 192 00000400:00000200:22.0:1616513087.984207:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.71@tcp1 00000400:00000200:22.0:1616513087.984215:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.71@tcp1 routed destination 00000400:00000200:22.0:1616513087.984228:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.71@tcp1(10.31.129.71@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.984234:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513087.984240:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.984242:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513087.984244:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513087.984249:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513087.984253:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714515221504 00000800:00000200:11.2:1616513087.984253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:12.0:1616513087.984264:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688714515221504/t0(0) o400->24cf953e-60d5-bd9b-cab1-640ea739f8a1@10.31.129.71@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513087.984278:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688714515221504, offset 192 00000400:00000200:12.0:1616513087.984285:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.71@tcp1 00000400:00000200:12.0:1616513087.984292:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.71@tcp1 routed destination 00000400:00000200:12.0:1616513087.984307:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.71@tcp1(10.31.129.71@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.984314:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513087.984319:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513087.984320:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513087.984322:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513087.984324:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513087.984327:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513087.984329:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513087.984330:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513087.984330:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513087.984333:0:50317:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513087.984335:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513087.984337:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513087.984341:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b320 00000800:00000200:28.0:1616513087.984345:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513087.984387:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513087.984393:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513087.984398:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.984398:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513087.984404:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513087.984410:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:18.0:1616513087.984413:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.991298:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513087.991351:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513087.991360:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513087.991369:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.120@tcp1 : PUT - for me 00000400:00000200:2.0:1616513087.991378:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.120@tcp1 of length 224 into portal 26 MB=0x5ffd4c7fec1c0 00000400:00000200:2.0:1616513087.991385:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.120@tcp1 of length 224/224 into md 0xf97cc539 [1] + 0 00000400:00000200:2.0:1616513087.991389:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb53a8 00000400:00000200:2.0:1616513087.991392:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513087.991395:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513087.991399:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb53a8 00000800:00000200:26.0:1616513087.991402:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513087.991402:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513087.991405:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513087.991406:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513087.991464:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664237064640 00010000:00000200:12.0:1616513087.991475:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688664237064640/t0(0) o400->3476fffa-0105-db68-732d-523a54b68303@10.31.136.120@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513087.991489:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664237064640, offset 192 00000400:00000200:12.0:1616513087.991495:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.120@tcp1 00000400:00000200:12.0:1616513087.991503:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.120@tcp1 routed destination 00000400:00000200:12.0:1616513087.991518:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.120@tcp1(10.31.136.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.991526:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513087.991532:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513087.991534:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513087.991537:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.991542:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513087.991548:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.991549:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513087.991560:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513087.991564:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513087.991565:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513087.991570:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513087.991573:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e3b8 00000800:00000200:8.0:1616513087.991576:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513087.991578:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513087.992144:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.992198:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.992206:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.992216:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.120@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.992224:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.120@tcp1 of length 224 into portal 12 MB=0x5ffd4c7fec300 00000400:00000200:14.0:1616513087.992230:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.120@tcp1 of length 224/224 into md 0xf964d519 [1] + 67408 00000400:00000200:14.0:1616513087.992235:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.992237:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513087.992243:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.992246:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.992248:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.992250:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513087.992306:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664237064960 00010000:00000200:22.0:1616513087.992317:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7f500 x1688664237064960/t0(0) o400->b6212658-7bb0-3d74-2ee7-2d9fd547959b@10.31.136.120@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513087.992331:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664237064960, offset 192 00000400:00000200:22.0:1616513087.992337:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.120@tcp1 00000400:00000200:22.0:1616513087.992343:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.120@tcp1 routed destination 00000400:00000200:22.0:1616513087.992357:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.120@tcp1(10.31.136.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.992363:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.992368:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513087.992370:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.992374:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513087.992378:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513087.992383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513087.992391:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.992396:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.992398:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513087.992401:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.992404:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.992407:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba08 00000800:00000200:26.0:1616513087.992411:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513087.994614:0:50318:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513087.994667:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513087.994676:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513087.994685:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.17@tcp1 : PUT - for me 00000400:00000200:14.0:1616513087.994694:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.17@tcp1 of length 224 into portal 26 MB=0x5ffdb4db51200 00000400:00000200:14.0:1616513087.994701:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.17@tcp1 of length 224/224 into md 0xf97cc569 [1] + 0 00000400:00000200:14.0:1616513087.994706:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5210 00000400:00000200:14.0:1616513087.994708:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513087.994711:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513087.994715:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5210 00000800:00000200:18.0:1616513087.994717:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.994718:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513087.994721:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513087.994723:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513087.994726:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.994727:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513087.994734:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.17@tcp1 : PUT - for me 00000400:00000200:18.0:1616513087.994742:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.17@tcp1 of length 224 into portal 12 MB=0x5ffdb4db51380 00000400:00000200:18.0:1616513087.994747:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.17@tcp1 of length 224/224 into md 0xf964d519 [1] + 67632 00000400:00000200:18.0:1616513087.994752:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513087.994754:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513087.994760:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513087.994762:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513087.994764:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513087.994776:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692250186240 00010000:00000200:12.0:1616513087.994786:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688692250186240/t0(0) o400->9dd8638a-d292-ee66-706f-a9f87528d0a2@10.31.141.17@tcp1:44/0 lens 224/224 e 0 to 0 dl 1616513179 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513087.994799:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692250186240, offset 192 00000400:00000200:12.0:1616513087.994805:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.17@tcp1 00000400:00000200:12.0:1616513087.994811:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.17@tcp1 routed destination 00000400:00000200:12.0:1616513087.994825:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.17@tcp1(10.31.141.17@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513087.994831:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513087.994836:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:22.0:1616513087.994839:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692250186624 00000800:00000200:12.0:1616513087.994839:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513087.994841:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513087.994845:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513087.994850:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bbf00 x1688692250186624/t0(0) o400->165970b0-b3a8-5024-6000-2f738e084945@10.31.141.17@tcp1:713/0 lens 224/224 e 0 to 0 dl 1616513093 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513087.994851:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513087.994852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513087.994863:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692250186624, offset 192 00000400:00000200:22.0:1616513087.994869:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.17@tcp1 00000400:00000200:22.0:1616513087.994876:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.17@tcp1 routed destination 00000400:00000200:22.0:1616513087.994889:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.17@tcp1(10.31.141.17@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513087.994895:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513087.994901:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513087.994902:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513087.994903:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513087.994905:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513087.994908:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513087.994909:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513087.994911:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513087.994914:0:50318:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513087.994916:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513087.994921:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e5d8 00000800:00000200:4.0:1616513087.994925:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513087.994927:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513087.994973:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513087.994979:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513087.994982:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513087.994983:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513087.994987:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513087.994991:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a110 00000800:00000200:26.0:1616513087.994994:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.018712:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.018767:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.018777:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.018786:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.204@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.018795:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.204@tcp1 of length 224 into portal 26 MB=0x5ffdbba6d1bc0 00000400:00000200:18.0:1616513088.018802:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.204@tcp1 of length 224/224 into md 0xf97cc579 [1] + 0 00000400:00000200:18.0:1616513088.018806:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb46e8 00000400:00000200:18.0:1616513088.018809:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.018812:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.018816:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb46e8 00000800:00000200:18.0:1616513088.018819:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.018822:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.018822:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.018826:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513088.018881:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694074186688 00010000:00000200:12.0:1616513088.018891:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688694074186688/t0(0) o400->df693f39-f0f1-4436-3f88-f80067af5171@10.31.141.204@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.018905:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694074186688, offset 192 00000400:00000200:12.0:1616513088.018912:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.204@tcp1 00000400:00000200:12.0:1616513088.018919:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.204@tcp1 routed destination 00000400:00000200:12.0:1616513088.018934:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.204@tcp1(10.31.141.204@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.018941:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.018946:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.018949:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.018952:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.018958:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513088.018964:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.018964:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.018977:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.018982:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.018985:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.018991:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.018996:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f5c8 00000800:00000200:8.0:1616513088.018999:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.019001:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.019058:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513088.019075:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513088.019113:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.019123:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513088.019126:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.019128:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.019130:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513088.019132:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513088.019139:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.019149:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.019157:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.204@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.019165:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.204@tcp1 of length 224 into portal 12 MB=0x5ffdbba6d27c0 00000400:00000200:14.0:1616513088.019172:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.204@tcp1 of length 224/224 into md 0xf964d519 [1] + 67856 00000400:00000200:14.0:1616513088.019177:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.019180:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513088.019187:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.019190:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.019192:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.019196:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.019262:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694074189760 00010000:00000200:22.0:1616513088.019274:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7de80 x1688694074189760/t0(0) o400->5696f2cb-cbb1-90a6-e177-1fda3438749b@10.31.141.204@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.019287:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694074189760, offset 192 00000400:00000200:22.0:1616513088.019293:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.204@tcp1 00000400:00000200:22.0:1616513088.019300:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.204@tcp1 routed destination 00000400:00000200:22.0:1616513088.019314:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.204@tcp1(10.31.141.204@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.019320:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.019326:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.019328:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.019330:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.019335:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.019339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.019348:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.019354:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.019356:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513088.019358:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513088.019363:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.019367:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a4c8 00000800:00000200:2.0:1616513088.019371:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.042930:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.042988:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.042997:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513088.043000:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513088.043006:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.204@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.043015:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.204@tcp1 of length 224 into portal 26 MB=0x5ffd4d85cd200 00000400:00000200:26.0:1616513088.043021:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.204@tcp1 of length 224/224 into md 0xf97cc5a9 [1] + 0 00000400:00000200:26.0:1616513088.043026:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5cb0 00000400:00000200:26.0:1616513088.043028:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.043031:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513088.043035:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5cb0 00000800:00000200:26.0:1616513088.043040:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.043042:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.043044:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513088.043100:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664511664640 00010000:00000200:12.0:1616513088.043111:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688664511664640/t0(0) o400->5001da39-0d6b-075d-5039-925a18990e9b@10.31.132.204@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.043125:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664511664640, offset 192 00000400:00000200:12.0:1616513088.043131:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.204@tcp1 00000400:00000200:12.0:1616513088.043139:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.204@tcp1 routed destination 00000400:00000200:12.0:1616513088.043154:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.204@tcp1(10.31.132.204@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.043160:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.043166:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.043169:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.043171:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.043175:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513088.043182:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.043219:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.043230:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.043235:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.043237:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.043242:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.043247:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f298 00000800:00000200:8.0:1616513088.043250:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.043252:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.043725:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.043778:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.043787:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.043795:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.204@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.043804:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.204@tcp1 of length 224 into portal 12 MB=0x5ffd4d85cd340 00000400:00000200:18.0:1616513088.043810:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.204@tcp1 of length 224/224 into md 0xf964d519 [1] + 68080 00000400:00000200:18.0:1616513088.043818:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.043821:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513088.043828:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.043831:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.043831:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.043836:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.043893:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664511664960 00010000:00000200:22.0:1616513088.043905:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b8480 x1688664511664960/t0(0) o400->f2ef465a-bcc6-fb3a-efba-cd4882207a9c@10.31.132.204@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.043919:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664511664960, offset 192 00000400:00000200:22.0:1616513088.043927:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.204@tcp1 00000400:00000200:22.0:1616513088.043935:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.204@tcp1 routed destination 00000400:00000200:22.0:1616513088.043950:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.204@tcp1(10.31.132.204@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.043956:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.043962:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.043965:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.043967:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.043972:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.043975:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.043984:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.043989:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.043991:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513088.043993:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513088.043997:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.044001:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a908 00000800:00000200:14.0:1616513088.044004:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:31.0:1616513088.059470:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927248, portal 4 00000100:00000200:31.0:1616513088.059476:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927248, offset 0 00000400:00000200:31.0:1616513088.059484:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:31.0:1616513088.059491:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:31.0:1616513088.059505:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513088.059512:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:31.0:1616513088.059520:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513088.059523:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:31.0:1616513088.059525:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:31.0:1616513088.059532:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513088.059578:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513088.059635:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513088.059643:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513088.059647:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513088.059653:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513088.059660:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded5580 x1693067725927248/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513088.059672:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2220 00000800:00000200:4.0:1616513088.059678:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513088.059681:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513088.059783:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513088.059792:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513088.059799:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513088.059806:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513088.059813:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627350 00000400:00000200:8.0:1616513088.059820:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d9521 [1] + 192 00000400:00000200:8.0:1616513088.059825:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.059829:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded5580 x1693067725927248/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513088.059843:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513088.059844:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513088.059846:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:8.0:1616513088.059848:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:31.0:1616513088.059853:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded5580 x1693067725927248/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513088.059879:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded5580 x1693067725927248/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513088.059889:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000800:00000200:7.2:1616513088.081543:0:50327:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513088.081599:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.081610:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513088.081619:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.73@tcp1 : PUT - for me 00000400:00000200:2.0:1616513088.081628:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.73@tcp1 of length 224 into portal 26 MB=0x5ffe936677f80 00000400:00000200:2.0:1616513088.081635:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.73@tcp1 of length 224/224 into md 0xf97cc5b9 [1] + 0 00000400:00000200:2.0:1616513088.081640:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4ff0 00000400:00000200:2.0:1616513088.081643:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.081645:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513088.081650:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4ff0 00000800:00000200:2.0:1616513088.081654:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.081655:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.081656:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.081662:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.081665:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.081666:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.081674:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.73@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.081683:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.73@tcp1 of length 224 into portal 12 MB=0x5ffe936678040 00000400:00000200:18.0:1616513088.081689:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.73@tcp1 of length 224/224 into md 0xf964d519 [1] + 68304 00000400:00000200:18.0:1616513088.081695:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.081698:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513088.081706:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.081709:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.081711:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513088.081717:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688751988768640 00010000:00000200:12.0:1616513088.081730:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688751988768640/t0(0) o400->1083b21f-3960-0307-5dec-a3e92066d67c@10.31.142.73@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.081745:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688751988768640, offset 192 00000400:00000200:12.0:1616513088.081751:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.73@tcp1 00000400:00000200:12.0:1616513088.081759:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.73@tcp1 routed destination 00000100:00000200:22.0:1616513088.081773:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688751988768832 00000400:00000200:12.0:1616513088.081774:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.73@tcp1(10.31.142.73@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.081780:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:22.0:1616513088.081787:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645be780 x1688751988768832/t0(0) o400->f1f67863-1c8d-1863-6fad-07ecc97600a9@10.31.142.73@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513088.081787:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.081789:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.081792:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.081797:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513088.081802:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688751988768832, offset 192 00000100:00000200:12.0:1616513088.081804:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.081806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513088.081808:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.73@tcp1 00000400:00000200:22.0:1616513088.081816:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.73@tcp1 routed destination 00000400:00000200:22.0:1616513088.081831:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.73@tcp1(10.31.142.73@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.081837:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.081844:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.081847:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.081849:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.081854:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.081858:0:50327:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513088.081861:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.081869:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513088.081871:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513088.081871:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.081878:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:4.0:1616513088.081878:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.081881:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513088.081882:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513088.081884:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e7f8 00000800:00000200:4.0:1616513088.081887:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513088.081888:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:4.0:1616513088.081890:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513088.081892:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a000 00000800:00000200:14.0:1616513088.081896:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.084690:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.084744:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.084753:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.084762:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.194@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.084771:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.194@tcp1 of length 224 into portal 26 MB=0x605a073113c80 00000400:00000200:18.0:1616513088.084778:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.194@tcp1 of length 224/224 into md 0xf97cc5d9 [1] + 0 00000400:00000200:18.0:1616513088.084782:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5078 00000400:00000200:18.0:1616513088.084785:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.084788:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.084792:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5078 00000800:00000200:18.0:1616513088.084795:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.084798:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513088.084798:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.084805:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.084808:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.084809:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.084817:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.194@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.084826:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.194@tcp1 of length 224 into portal 12 MB=0x605a073113d40 00000400:00000200:26.0:1616513088.084833:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.194@tcp1 of length 224/224 into md 0xf964d519 [1] + 68528 00000400:00000200:26.0:1616513088.084838:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.084840:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.084847:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.084850:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.084852:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513088.084855:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036543679616 00010000:00000200:12.0:1616513088.084865:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1695036543679616/t0(0) o400->f8e94b9f-814c-86fb-f6c1-020cfe963405@10.31.129.194@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.084881:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036543679616, offset 192 00000400:00000200:12.0:1616513088.084888:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.194@tcp1 00000400:00000200:12.0:1616513088.084895:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.194@tcp1 routed destination 00000400:00000200:12.0:1616513088.084910:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.194@tcp1(10.31.129.194@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513088.084911:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036543679808 00000800:00000200:12.0:1616513088.084916:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.084922:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513088.084923:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c696780 x1695036543679808/t0(0) o400->2755d25d-37b6-cfc7-ff3a-c9c3da867c71@10.31.129.194@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513088.084925:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.084927:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.084932:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513088.084937:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036543679808, offset 192 00000800:00000200:11.2:1616513088.084937:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513088.084938:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513088.084944:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.194@tcp1 00000400:00000200:22.0:1616513088.084951:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.194@tcp1 routed destination 00000400:00000200:22.0:1616513088.084966:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.194@tcp1(10.31.129.194@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.084973:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.084978:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.084983:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.084985:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.084990:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.084993:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513088.084994:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.085001:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513088.085004:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.085004:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513088.085006:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513088.085012:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513088.085013:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.085014:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513088.085016:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513088.085018:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f430 00000400:00000200:14.0:1616513088.085021:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513088.085022:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513088.085025:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba90 00000800:00000200:14.0:1616513088.085029:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.085662:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513088.085718:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.085737:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513088.085745:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.85@tcp1 : PUT - for me 00000400:00000200:4.0:1616513088.085754:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.85@tcp1 of length 224 into portal 26 MB=0x604a308c3ddc0 00000400:00000200:4.0:1616513088.085760:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.85@tcp1 of length 224/224 into md 0xf97cc5e9 [1] + 0 00000400:00000200:4.0:1616513088.085765:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4770 00000400:00000200:4.0:1616513088.085767:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513088.085770:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513088.085774:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4770 00000800:00000200:4.0:1616513088.085777:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.085779:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.085781:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.085784:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513088.085786:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693948133498304 00010000:00000200:12.0:1616513088.085792:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1693948133498304/t0(0) o400->03417b72-61bf-1c19-5ac2-af784305d62c@10.31.140.85@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.085804:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693948133498304, offset 192 00000400:00000200:12.0:1616513088.085809:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.85@tcp1 00000400:00000200:12.0:1616513088.085814:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.85@tcp1 routed destination 00000400:00000200:12.0:1616513088.085827:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.85@tcp1(10.31.140.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.085833:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.085838:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.085840:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.085842:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.085845:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513088.085849:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.085850:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.085859:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.085864:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.085866:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513088.085869:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.085871:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.085876:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fcb0 00000800:00000200:8.0:1616513088.085879:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.086170:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.086232:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.086241:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.086250:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.85@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.086259:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.85@tcp1 of length 224 into portal 12 MB=0x604a308c3e2c0 00000400:00000200:26.0:1616513088.086265:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.85@tcp1 of length 224/224 into md 0xf964d519 [1] + 68752 00000400:00000200:26.0:1616513088.086270:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.086272:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.086278:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.086281:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.086283:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.086286:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.086341:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693948133499584 00010000:00000200:22.0:1616513088.086353:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c692d00 x1693948133499584/t0(0) o400->34bc6316-798c-67a5-f454-66b9779c9378@10.31.140.85@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.086368:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693948133499584, offset 192 00000400:00000200:22.0:1616513088.086374:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.85@tcp1 00000400:00000200:22.0:1616513088.086382:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.85@tcp1 routed destination 00000400:00000200:22.0:1616513088.086397:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.85@tcp1(10.31.140.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.086403:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.086409:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.086411:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.086414:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.086418:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.086422:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.086475:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.086482:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.086485:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.086487:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.086491:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.086498:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a550 00000800:00000200:18.0:1616513088.086502:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.093115:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513088.093172:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.093181:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513088.093188:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513088.093189:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.91@tcp1 : PUT - for me 00000400:00000200:4.0:1616513088.093198:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.91@tcp1 of length 224 into portal 26 MB=0x5ffdd261fda80 00000400:00000200:4.0:1616513088.093204:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.91@tcp1 of length 224/224 into md 0xf97cc609 [1] + 0 00000400:00000200:4.0:1616513088.093208:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4f68 00000400:00000200:4.0:1616513088.093211:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513088.093214:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513088.093218:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4f68 00000800:00000200:4.0:1616513088.093221:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.093223:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513088.093225:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513088.093281:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700176030336 00010000:00000200:12.0:1616513088.093292:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688700176030336/t0(0) o400->38eeb541-2da5-c481-c425-5893ec79f73b@10.31.128.91@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.093306:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700176030336, offset 192 00000400:00000200:12.0:1616513088.093312:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.91@tcp1 00000400:00000200:12.0:1616513088.093320:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.91@tcp1 routed destination 00000400:00000200:12.0:1616513088.093336:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.91@tcp1(10.31.128.91@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.093342:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.093348:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.093351:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.093353:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.093358:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.093362:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513088.093364:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513088.093416:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.093423:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513088.093426:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513088.093426:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.093432:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513088.093450:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f078 00000800:00000200:20.0:1616513088.093453:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.094130:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.094141:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.094149:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513088.094151:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.094158:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.91@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.094168:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.91@tcp1 of length 224 into portal 12 MB=0x5ffdd261fdc00 00000400:00000200:28.0:1616513088.094176:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.91@tcp1 of length 224/224 into md 0x865bbec1 [1] + 95200 00000400:00000200:28.0:1616513088.094181:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.094184:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513088.094191:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.094194:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.094196:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513088.094207:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700176030720 00010000:00000200:12.0:1616513088.094215:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa6300 x1688700176030720/t0(0) o400->15e9541a-d3c7-80b2-8fdd-0340cf4c1993@10.31.128.91@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513088.094226:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700176030720, offset 192 00000400:00000200:12.0:1616513088.094231:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.91@tcp1 00000400:00000200:12.0:1616513088.094236:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.91@tcp1 routed destination 00000400:00000200:12.0:1616513088.094249:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.91@tcp1(10.31.128.91@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.094255:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513088.094260:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.094262:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513088.094264:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513088.094267:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.094272:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.094328:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.094335:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.094338:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513088.094345:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.094349:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eff0 00000800:00000200:2.0:1616513088.094353:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.094356:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.102261:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.102272:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.102279:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.102286:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.107.14@tcp2 : PUT - for me 00000400:00000200:8.0:1616513088.102294:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.107.14@tcp2 of length 224 into portal 26 MB=0x601c526eef4c0 00000400:00000200:8.0:1616513088.102299:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.107.14@tcp2 of length 224/224 into md 0xf97cc619 [1] + 0 00000400:00000200:8.0:1616513088.102303:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5320 00000400:00000200:8.0:1616513088.102306:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.102308:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513088.102312:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5320 00000800:00000200:8.0:1616513088.102314:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.102316:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.102318:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.102319:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.102324:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513088.102325:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.102330:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.107.14@tcp2 : PUT - for me 00000400:00000200:8.0:1616513088.102334:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.107.14@tcp2 of length 224 into portal 12 MB=0x601c526eef540 00000400:00000200:8.0:1616513088.102339:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.107.14@tcp2 of length 224/224 into md 0x865bbec1 [1] + 95424 00000400:00000200:8.0:1616513088.102342:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.102344:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513088.102349:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.102350:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.102352:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513088.102373:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690796133643456 00010000:00000200:12.0:1616513088.102383:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1690796133643456/t0(0) o400->dfd6149c-a0a4-ad2c-5bf6-cb58cc009b8e@10.242.107.14@tcp2:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.102396:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690796133643456, offset 192 00000400:00000200:12.0:1616513088.102402:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.107.14@tcp2 00000400:00000200:12.0:1616513088.102409:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.107.14@tcp2 routed destination 00000100:00000200:16.0:1616513088.102417:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690796133643584 00000400:00000200:12.0:1616513088.102424:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.107.14@tcp2(10.242.107.14@tcp2:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.102429:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:16.0:1616513088.102430:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89680 x1690796133643584/t0(0) o400->a46a966a-4ad7-e2ce-fcd3-ed153c26362c@10.242.107.14@tcp2:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513088.102447:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.102449:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513088.102453:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513088.102457:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:16.0:1616513088.102459:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690796133643584, offset 192 00000800:00000200:7.2:1616513088.102462:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513088.102464:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513088.102465:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.107.14@tcp2 00000400:00000200:16.0:1616513088.102472:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.107.14@tcp2 routed destination 00000400:00000200:16.0:1616513088.102487:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.107.14@tcp2(10.242.107.14@tcp2:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.102494:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513088.102499:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.102502:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.102504:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513088.102509:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.102514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513088.102519:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513088.102525:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513088.102527:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.102530:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513088.102532:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.102535:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513088.102536:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513088.102536:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.102541:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.102542:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa90 00000800:00000200:14.0:1616513088.102545:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513088.102546:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789a08 00000800:00000200:14.0:1616513088.102548:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513088.102549:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.103865:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.103873:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.103877:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.103883:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.188@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.103889:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.188@tcp1 of length 224 into portal 26 MB=0x5ffdaf099ec00 00000400:00000200:8.0:1616513088.103893:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.188@tcp1 of length 224/224 into md 0xf97cc639 [1] + 0 00000400:00000200:8.0:1616513088.103896:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5650 00000400:00000200:8.0:1616513088.103898:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.103900:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513088.103902:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5650 00000800:00000200:8.0:1616513088.103904:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.103906:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.103907:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.103909:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.103913:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.103918:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.188@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.103922:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.188@tcp1 of length 224 into portal 12 MB=0x5ffdaf099ec80 00000800:00000200:4.0:1616513088.103922:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.103926:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.188@tcp1 of length 224/224 into md 0x865bbec1 [1] + 95648 00000400:00000200:8.0:1616513088.103929:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.103931:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513088.103934:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.103936:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.103937:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513088.103980:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688690688125952 00010000:00000200:12.0:1616513088.103990:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688690688125952/t0(0) o400->023aea0d-0287-e864-f0b9-993d73436297@10.31.140.188@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513088.104003:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688690688126080 00000100:00000200:12.0:1616513088.104004:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688690688125952, offset 192 00000400:00000200:12.0:1616513088.104009:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.188@tcp1 00010000:00000200:16.0:1616513088.104016:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89f80 x1688690688126080/t0(0) o400->11456d94-e2a6-749a-8c09-36cff79d8671@10.31.140.188@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513088.104016:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.188@tcp1 routed destination 00000400:00000200:12.0:1616513088.104030:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.188@tcp1(10.31.140.188@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513088.104031:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688690688126080, offset 192 00000400:00000200:16.0:1616513088.104037:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.188@tcp1 00000800:00000200:12.0:1616513088.104037:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.104042:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513088.104044:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.188@tcp1 routed destination 00000800:00000200:12.0:1616513088.104044:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.104046:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.104051:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.104055:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513088.104057:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513088.104059:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.188@tcp1(10.31.140.188@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.104066:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.104072:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513088.104073:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513088.104074:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.104077:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513088.104080:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513088.104081:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513088.104082:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513088.104083:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513088.104085:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513088.104089:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.104095:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa90 00000800:00000200:28.0:1616513088.104098:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513088.104137:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.104143:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.104146:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513088.104152:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.104156:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789430 00000800:00000200:2.0:1616513088.104160:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.104162:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.104168:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513088.104170:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.104172:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.104173:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.104176:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.117022:0:50331:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.117076:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.117086:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.117094:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.123@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.117104:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.123@tcp1 of length 224 into portal 26 MB=0x60551a8ae9d40 00000400:00000200:18.0:1616513088.117111:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.123@tcp1 of length 224/224 into md 0xf97cc649 [1] + 0 00000400:00000200:18.0:1616513088.117116:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71dc0 00000400:00000200:18.0:1616513088.117119:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.117122:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.117126:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71dc0 00000800:00000200:18.0:1616513088.117130:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.117130:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.117133:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.117135:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.117150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.117162:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.117168:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.117176:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.123@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.117183:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.123@tcp1 of length 224 into portal 12 MB=0x60551a8ae9dc0 00000400:00000200:8.0:1616513088.117188:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.123@tcp1 of length 224/224 into md 0x865bbec1 [1] + 95872 00000100:00000200:12.0:1616513088.117190:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694698140769600 00000400:00000200:8.0:1616513088.117192:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.117194:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513088.117199:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:12.0:1616513088.117201:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1694698140769600/t0(0) o400->a98f2b0a-e1dd-007b-40a6-142cc0c1c1be@10.31.143.123@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:8.0:1616513088.117201:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.117203:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.117205:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513088.117215:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694698140769600, offset 192 00000400:00000200:12.0:1616513088.117221:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.123@tcp1 00000400:00000200:12.0:1616513088.117229:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.123@tcp1 routed destination 00000400:00000200:12.0:1616513088.117244:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.123@tcp1(10.31.143.123@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.117250:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.117256:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.117258:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.117260:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.117265:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513088.117268:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694698140769728 00000800:00000200:11.2:1616513088.117270:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513088.117272:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:16.0:1616513088.117281:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8e300 x1694698140769728/t0(0) o400->976566af-3c91-ff52-d919-87ea35e4f30d@10.31.143.123@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.117297:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694698140769728, offset 192 00000400:00000200:16.0:1616513088.117303:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.123@tcp1 00000400:00000200:16.0:1616513088.117311:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.123@tcp1 routed destination 00000400:00000200:16.0:1616513088.117326:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.123@tcp1(10.31.143.123@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.117333:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513088.117336:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513088.117339:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.117341:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513088.117343:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513088.117344:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513088.117346:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513088.117348:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513088.117349:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513088.117352:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513088.117353:0:50331:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513088.117358:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f078 00000800:00000200:4.0:1616513088.117362:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513088.117407:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.117414:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.117416:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.117423:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.117427:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788330 00000800:00000200:26.0:1616513088.117431:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.117449:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.118585:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513088.118639:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.118649:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513088.118654:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513088.118658:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.165@tcp1 : PUT - for me 00000400:00000200:2.0:1616513088.118667:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.165@tcp1 of length 224 into portal 26 MB=0x5ffd4e033ea00 00000400:00000200:2.0:1616513088.118673:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.165@tcp1 of length 224/224 into md 0xf97cc669 [1] + 0 00000400:00000200:2.0:1616513088.118678:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71188 00000400:00000200:2.0:1616513088.118680:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.118683:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513088.118687:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71188 00000800:00000200:2.0:1616513088.118690:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.118693:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.118696:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513088.118751:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664643201536 00010000:00000200:12.0:1616513088.118761:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688664643201536/t0(0) o400->2c1fa228-281b-b106-4156-c927abf7d7c3@10.31.132.165@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.118777:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664643201536, offset 192 00000400:00000200:12.0:1616513088.118783:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.165@tcp1 00000400:00000200:12.0:1616513088.118790:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.165@tcp1 routed destination 00000400:00000200:12.0:1616513088.118805:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.165@tcp1(10.31.132.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.118812:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.118817:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.118820:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.118822:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.118827:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.118832:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513088.118833:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513088.118842:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.118846:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.118848:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.118853:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.118857:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f430 00000800:00000200:8.0:1616513088.118859:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.118861:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.119933:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.119990:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.120002:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.120011:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.165@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.120019:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.165@tcp1 of length 224 into portal 12 MB=0x5ffd4e033eb00 00000400:00000200:26.0:1616513088.120026:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.165@tcp1 of length 224/224 into md 0xf964d519 [1] + 68976 00000400:00000200:26.0:1616513088.120031:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.120034:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.120040:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.120044:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.120046:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.120048:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.120116:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664643201792 00010000:00000200:22.0:1616513088.120127:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c694800 x1688664643201792/t0(0) o400->a70dac55-aa65-9576-614d-ae311c713a7a@10.31.132.165@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.120141:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664643201792, offset 192 00000400:00000200:22.0:1616513088.120146:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.165@tcp1 00000400:00000200:22.0:1616513088.120153:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.165@tcp1 routed destination 00000400:00000200:22.0:1616513088.120167:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.165@tcp1(10.31.132.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.120173:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.120178:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.120181:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.120183:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.120187:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.120191:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.120252:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.120259:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.120262:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513088.120268:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.120272:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b870 00000800:00000200:18.0:1616513088.120276:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.120279:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513088.122581:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927264, portal 4 00000100:00000200:28.0:1616513088.122586:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927264, offset 0 00000400:00000200:28.0:1616513088.122593:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:28.0:1616513088.122598:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:28.0:1616513088.122611:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513088.122618:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513088.122624:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513088.122626:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513088.122629:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513088.122633:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513088.122679:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513088.122694:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513088.122699:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513088.122701:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513088.122707:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513088.122713:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cb02e5100 x1693067725927264/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513088.122725:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc6e8 00000800:00000200:28.0:1616513088.122728:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513088.122730:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513088.122863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513088.122873:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513088.122879:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513088.122885:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513088.122891:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627360 00000400:00000200:8.0:1616513088.122896:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d95f1 [1] + 192 00000400:00000200:8.0:1616513088.122900:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.122904:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cb02e5100 x1693067725927264/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513088.122917:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513088.122919:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:8.0:1616513088.122920:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513088.122923:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:28.0:1616513088.122924:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cb02e5100 x1693067725927264/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513088.122935:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cb02e5100 x1693067725927264/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513088.122944:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc7f8 00000800:00000200:7.2:1616513088.129137:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.129190:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.129199:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.129208:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.170@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.129217:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.170@tcp1 of length 224 into portal 26 MB=0x6059f6c428f00 00000400:00000200:26.0:1616513088.129224:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.170@tcp1 of length 224/224 into md 0xf97cc699 [1] + 0 00000400:00000200:26.0:1616513088.129228:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70990 00000400:00000200:26.0:1616513088.129231:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.129234:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513088.129238:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70990 00000800:00000200:26.0:1616513088.129241:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.129244:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.129245:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.129249:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513088.129301:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695032134504192 00010000:00000200:12.0:1616513088.129312:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1695032134504192/t0(0) o400->b149c657-85db-7553-9b59-fcb00bc937ce@10.31.136.170@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.129326:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695032134504192, offset 192 00000400:00000200:12.0:1616513088.129332:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.170@tcp1 00000400:00000200:12.0:1616513088.129340:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.170@tcp1 routed destination 00000400:00000200:12.0:1616513088.129354:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.170@tcp1(10.31.136.170@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.129361:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.129367:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.129369:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.129371:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.129376:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513088.129383:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.129383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.129452:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.129459:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.129462:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513088.129468:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.129474:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f298 00000800:00000200:28.0:1616513088.129477:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.129480:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.130531:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513088.130590:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.130600:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513088.130601:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.130609:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.170@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.130617:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.170@tcp1 of length 224 into portal 12 MB=0x6059f6c429200 00000400:00000200:20.0:1616513088.130624:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.170@tcp1 of length 224/224 into md 0x865bbec1 [1] + 96096 00000400:00000200:20.0:1616513088.130628:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d85b0b73078 00000400:00000200:20.0:1616513088.130631:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.130634:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513088.130638:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d85b0b73078 00000800:00000200:20.0:1616513088.130641:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.130644:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.130646:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.130700:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695032134504960 00010000:00000200:16.0:1616513088.130711:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc2e80050 x1695032134504960/t0(0) o400->5f5fbcb6-57a7-5223-92af-30d9949d499d@10.31.136.170@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.130725:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695032134504960, offset 192 00000400:00000200:16.0:1616513088.130731:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.170@tcp1 00000400:00000200:16.0:1616513088.130737:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.170@tcp1 routed destination 00000400:00000200:16.0:1616513088.130751:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.170@tcp1(10.31.136.170@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.130757:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.130763:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.130765:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.130767:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.130772:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.130775:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.130826:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.130833:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.130835:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513088.130842:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.130847:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789a90 00000800:00000200:18.0:1616513088.130850:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.130853:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513088.131150:0:188393:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 12 00000800:00000200:7.2:1616513088.153986:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.154056:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.154066:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.154075:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.99@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.154086:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.99@tcp1 of length 224 into portal 12 MB=0x6001eb3079a40 00000400:00000200:14.0:1616513088.154092:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.99@tcp1 of length 224/224 into md 0xf964d519 [1] + 69200 00000800:00000200:11.2:1616513088.154094:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513088.154100:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.154102:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.154108:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.154110:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.154113:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.154115:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513088.154149:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.154158:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513088.154166:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.99@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.154175:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.99@tcp1 of length 224 into portal 26 MB=0x6001eb30796c0 00000400:00000200:28.0:1616513088.154181:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.99@tcp1 of length 224/224 into md 0xf97cc6a9 [1] + 0 00000400:00000200:28.0:1616513088.154186:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71a08 00000100:00000200:22.0:1616513088.154188:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688981712902720 00000400:00000200:28.0:1616513088.154189:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.154191:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513088.154196:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71a08 00000800:00000200:28.0:1616513088.154199:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:22.0:1616513088.154201:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7ec00 x1688981712902720/t0(0) o400->247f6fa0-930c-31cf-89f2-101bad6bb18d@10.31.142.99@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513088.154202:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.154203:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.154206:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513088.154226:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688981712902720, offset 192 00000400:00000200:22.0:1616513088.154233:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.99@tcp1 00000400:00000200:22.0:1616513088.154246:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.99@tcp1 routed destination 00000400:00000200:22.0:1616513088.154261:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.99@tcp1(10.31.142.99@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:12.0:1616513088.154264:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688981712901824 00000800:00000200:22.0:1616513088.154268:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513088.154274:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:12.0:1616513088.154275:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688981712901824/t0(0) o400->c3578b74-bc84-2a16-1e6e-1272e6efef82@10.31.142.99@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513088.154276:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513088.154279:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513088.154284:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513088.154288:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688981712901824, offset 192 00000800:00000200:11.2:1616513088.154288:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513088.154294:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.99@tcp1 00000800:00000200:8.0:1616513088.154297:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513088.154301:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.99@tcp1 routed destination 00000800:00000200:8.0:1616513088.154301:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.154303:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.154309:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.154312:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b980 00000800:00000200:8.0:1616513088.154315:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513088.154317:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.99@tcp1(10.31.142.99@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513088.154318:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513088.154323:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513088.154328:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.154331:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513088.154333:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513088.154338:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.154342:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513088.154344:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513088.154393:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.154399:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.154402:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513088.154408:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.154413:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e5d8 00000800:00000200:18.0:1616513088.154418:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.154421:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.155296:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.155349:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.155358:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.155367:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.27@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.155376:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.27@tcp1 of length 224 into portal 26 MB=0x60194ed03d380 00000400:00000200:26.0:1616513088.155382:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.27@tcp1 of length 224/224 into md 0xf97cc6b9 [1] + 0 00000400:00000200:26.0:1616513088.155387:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70220 00000400:00000200:26.0:1616513088.155389:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.155392:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513088.155396:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70220 00000800:00000200:26.0:1616513088.155399:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.155404:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.155406:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.155409:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513088.155409:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589003502464 00010000:00000200:12.0:1616513088.155415:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1690589003502464/t0(0) o400->d08d4d87-3d0f-e46e-4c4d-35ae85a9e553@10.31.133.27@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.155427:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690589003502464, offset 192 00000400:00000200:12.0:1616513088.155432:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.27@tcp1 00000400:00000200:12.0:1616513088.155449:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.27@tcp1 routed destination 00000400:00000200:12.0:1616513088.155462:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.27@tcp1(10.31.133.27@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.155468:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.155473:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.155476:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.155477:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.155481:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513088.155485:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.155486:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.155549:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.155555:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.155558:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513088.155567:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.155572:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:28.0:1616513088.155576:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.155579:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.155697:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.155704:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.155709:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.155716:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.27@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.155722:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.27@tcp1 of length 224 into portal 12 MB=0x60194ed03d440 00000400:00000200:8.0:1616513088.155727:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.27@tcp1 of length 224/224 into md 0x865be291 [1] + 0 00000400:00000200:8.0:1616513088.155731:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.155733:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513088.155738:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.155741:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.155742:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.155744:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.155803:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589003502656 00010000:00000200:16.0:1616513088.155815:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8de80 x1690589003502656/t0(0) o400->a8984c73-8c5f-675d-b9c7-184c59e6e693@10.31.133.27@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.155828:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690589003502656, offset 192 00000400:00000200:16.0:1616513088.155834:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.27@tcp1 00000400:00000200:16.0:1616513088.155841:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.27@tcp1 routed destination 00000400:00000200:16.0:1616513088.155855:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.27@tcp1(10.31.133.27@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.155862:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.155868:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.155870:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.155872:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.155877:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.155881:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.155933:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.155940:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.155942:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513088.155949:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.155953:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788088 00000800:00000200:18.0:1616513088.155957:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.155960:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.182895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513088.182897:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513088.182950:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.182960:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513088.182963:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513088.182970:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513088.182970:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.133@tcp1 : PUT - for me 00000800:00000200:28.0:1616513088.182979:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:14.0:1616513088.182980:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.133@tcp1 of length 224 into portal 26 MB=0x604aaef3423c0 00000400:00000200:14.0:1616513088.182987:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.133@tcp1 of length 224/224 into md 0xf97cc6e9 [1] + 0 00000400:00000200:28.0:1616513088.182988:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.133@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.182991:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f713a8 00000400:00000200:14.0:1616513088.182994:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.182997:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513088.182998:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.133@tcp1 of length 224 into portal 12 MB=0x604aaef342480 00000400:00000200:14.0:1616513088.183001:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f713a8 00000400:00000200:28.0:1616513088.183004:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.133@tcp1 of length 224/224 into md 0x865be291 [1] + 224 00000800:00000200:14.0:1616513088.183005:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.183007:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.183009:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513088.183010:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.183012:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513088.183019:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.183022:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.183023:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.183027:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513088.183070:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693982064387008 00010000:00000200:12.0:1616513088.183082:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1693982064387008/t0(0) o400->6bc156e1-53b1-fc93-3a38-869575553462@10.31.141.133@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513088.183091:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693982064387200 00000100:00000200:12.0:1616513088.183096:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693982064387008, offset 192 00000400:00000200:12.0:1616513088.183103:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.133@tcp1 00010000:00000200:16.0:1616513088.183105:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa2880 x1693982064387200/t0(0) o400->ce7f2878-41d0-3068-e237-f9594ba8ce01@10.31.141.133@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513088.183110:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.133@tcp1 routed destination 00000100:00000200:16.0:1616513088.183120:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693982064387200, offset 192 00000400:00000200:12.0:1616513088.183125:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.133@tcp1(10.31.141.133@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513088.183126:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.133@tcp1 00000800:00000200:12.0:1616513088.183132:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513088.183135:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.133@tcp1 routed destination 00000800:00000200:12.0:1616513088.183142:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.183147:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.183149:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513088.183150:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.133@tcp1(10.31.141.133@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.183154:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513088.183157:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513088.183159:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513088.183161:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513088.183163:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.183166:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513088.183168:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513088.183169:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513088.183172:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513088.183174:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513088.183174:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513088.183177:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513088.183179:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.183183:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e880 00000800:00000200:8.0:1616513088.183186:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.183188:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513088.183240:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.183248:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.183251:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513088.183257:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.183261:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788088 00000800:00000200:18.0:1616513088.183265:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.183268:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.226544:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.226599:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.226609:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.226619:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.191@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.226629:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.191@tcp1 of length 224 into portal 12 MB=0x605a0732268c0 00000400:00000200:26.0:1616513088.226636:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.191@tcp1 of length 224/224 into md 0xf964d519 [1] + 69424 00000400:00000200:26.0:1616513088.226641:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.226644:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.226652:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.226655:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.226656:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.226660:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.226721:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036544805056 00010000:00000200:22.0:1616513088.226735:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c696300 x1695036544805056/t0(0) o400->acc137ec-1152-0d8d-9145-fbfd68322a46@10.31.129.191@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.226751:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036544805056, offset 192 00000800:00000200:11.2:1616513088.226751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513088.226757:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.191@tcp1 00000400:00000200:22.0:1616513088.226765:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.191@tcp1 routed destination 00000800:00000200:4.0:1616513088.226765:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.226773:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513088.226781:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.191@tcp1(10.31.129.191@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.226788:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:4.0:1616513088.226790:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.191@tcp1 : PUT - for me 00000800:00000200:22.0:1616513088.226795:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.226797:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:4.0:1616513088.226798:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.191@tcp1 of length 224 into portal 26 MB=0x605a0732267c0 00000800:00000200:22.0:1616513088.226800:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:4.0:1616513088.226804:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.191@tcp1 of length 224/224 into md 0xf97cc6f9 [1] + 0 00000800:00000200:22.0:1616513088.226805:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:4.0:1616513088.226808:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71cb0 00000400:00000200:4.0:1616513088.226811:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513088.226813:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513088.226817:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71cb0 00000800:00000200:28.0:1616513088.226819:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513088.226820:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513088.226823:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:4.0:1616513088.226826:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513088.226827:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513088.226828:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.226830:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513088.226837:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.226841:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a880 00000800:00000200:28.0:1616513088.226845:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513088.226883:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036544804800 00010000:00000200:12.0:1616513088.226895:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1695036544804800/t0(0) o400->c40f25c3-69c7-e5f3-d755-8beb44824981@10.31.129.191@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.226909:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036544804800, offset 192 00000400:00000200:12.0:1616513088.226916:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.191@tcp1 00000400:00000200:12.0:1616513088.226923:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.191@tcp1 routed destination 00000400:00000200:12.0:1616513088.226939:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.191@tcp1(10.31.129.191@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.226946:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513088.226952:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.226955:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513088.226957:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513088.226962:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.226966:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513088.226969:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513088.227019:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.227026:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.227029:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513088.227035:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.227041:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ee58 00000800:00000200:18.0:1616513088.227045:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.227048:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.233430:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513088.233487:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.233496:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513088.233501:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.233505:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.233514:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x6039913407740 00000400:00000200:20.0:1616513088.233520:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0x865be291 [1] + 448 00000400:00000200:20.0:1616513088.233526:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.233528:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.233535:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.233538:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.233542:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.233599:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805848135488 00010000:00000200:16.0:1616513088.233673:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad3a80 x1692805848135488/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:714/0 lens 336/336 e 0 to 0 dl 1616513094 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513088.233688:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805848135488, offset 192 00000400:00000200:16.0:1616513088.233694:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:16.0:1616513088.233701:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:16.0:1616513088.233716:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.233723:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513088.233729:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.233731:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.233733:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513088.233738:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.233743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.233752:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.233757:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.233759:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513088.233763:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.233765:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.233768:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789a90 00000800:00000200:8.0:1616513088.233771:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.236684:0:50345:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.236738:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.236748:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.236757:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.207@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.236766:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.207@tcp1 of length 224 into portal 26 MB=0x5ffd4dbe23180 00000400:00000200:14.0:1616513088.236772:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.207@tcp1 of length 224/224 into md 0xf97cc719 [1] + 0 00000400:00000200:14.0:1616513088.236777:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70770 00000400:00000200:14.0:1616513088.236780:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.236782:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.236786:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70770 00000800:00000200:14.0:1616513088.236789:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.236792:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.236793:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513088.236800:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.236803:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.236804:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.236812:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.207@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.236821:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.207@tcp1 of length 224 into portal 12 MB=0x5ffd4dbe23200 00000400:00000200:26.0:1616513088.236827:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.207@tcp1 of length 224/224 into md 0xf964d519 [1] + 69648 00000400:00000200:26.0:1616513088.236833:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.236835:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.236842:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.236845:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.236847:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513088.236850:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664570737024 00010000:00000200:12.0:1616513088.236861:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688664570737024/t0(0) o400->18e1ac67-4171-4094-929f-bd239afa90d7@10.31.136.207@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.236875:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664570737024, offset 192 00000400:00000200:12.0:1616513088.236881:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.207@tcp1 00000400:00000200:12.0:1616513088.236889:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.207@tcp1 routed destination 00000400:00000200:12.0:1616513088.236904:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.207@tcp1(10.31.136.207@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513088.236905:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664570737152 00000800:00000200:12.0:1616513088.236910:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513088.236916:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513088.236918:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693f00 x1688664570737152/t0(0) o400->a8ede024-2d3d-39cf-ee13-241bb5be45e7@10.31.136.207@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513088.236919:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513088.236921:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513088.236926:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.236929:0:50345:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513088.236932:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664570737152, offset 192 00000100:00000200:12.0:1616513088.236932:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513088.236938:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.207@tcp1 00000400:00000200:22.0:1616513088.236946:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.207@tcp1 routed destination 00000400:00000200:22.0:1616513088.236961:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.207@tcp1(10.31.136.207@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.236969:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513088.236975:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.236978:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513088.236980:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513088.236985:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513088.236991:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.236999:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.237002:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513088.237002:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513088.237008:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.237014:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eb28 00000800:00000200:2.0:1616513088.237017:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.237029:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.237080:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.237085:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513088.237087:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513088.237093:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.237096:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4abb0 00000800:00000200:4.0:1616513088.237100:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513088.237102:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.238375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.238430:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.238444:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.238449:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513088.238455:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.238456:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.143@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.238464:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.143@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.238464:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.143@tcp1 of length 224 into portal 12 MB=0x603fd53d4fa80 00000400:00000200:8.0:1616513088.238469:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.143@tcp1 of length 224/224 into md 0x865be291 [1] + 784 00000400:00000200:28.0:1616513088.238473:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.143@tcp1 of length 224 into portal 26 MB=0x603fd53d4fa00 00000400:00000200:8.0:1616513088.238473:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.238475:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513088.238480:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.143@tcp1 of length 224/224 into md 0xf97cc739 [1] + 0 00000800:00000200:8.0:1616513088.238481:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513088.238483:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513088.238484:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70198 00000800:00000200:8.0:1616513088.238485:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.238487:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:8.0:1616513088.238487:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513088.238490:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513088.238495:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70198 00000800:00000200:28.0:1616513088.238498:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.238501:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.238503:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.238544:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693236428339840 00010000:00000200:16.0:1616513088.238557:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8c380 x1693236428339840/t0(0) o400->63b8c357-f11a-3b12-9627-65c5bd8cb9ae@10.31.138.143@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513088.238560:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693236428339712 00000100:00000200:16.0:1616513088.238571:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693236428339840, offset 192 00010000:00000200:12.0:1616513088.238571:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1693236428339712/t0(0) o400->90c25aee-060a-5941-225d-6606f7dbf090@10.31.138.143@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513088.238578:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.143@tcp1 00000400:00000200:16.0:1616513088.238585:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.143@tcp1 routed destination 00000100:00000200:12.0:1616513088.238586:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693236428339712, offset 192 00000400:00000200:12.0:1616513088.238592:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.143@tcp1 00000400:00000200:12.0:1616513088.238599:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.143@tcp1 routed destination 00000400:00000200:16.0:1616513088.238600:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.143@tcp1(10.31.138.143@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.238607:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513088.238614:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.143@tcp1(10.31.138.143@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.238621:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.238621:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513088.238624:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.238626:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513088.238627:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.238630:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.238631:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513088.238632:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.238636:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513088.238636:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513088.238641:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513088.238643:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513088.238688:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.238694:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.238697:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.238703:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.238707:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788330 00000800:00000200:26.0:1616513088.238710:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.238713:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513088.238713:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.238720:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513088.238722:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513088.238722:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.238729:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513088.238735:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e000 00000800:00000200:20.0:1616513088.238738:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.281974:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.281987:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.281998:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.282007:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.71@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.282018:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.71@tcp1 of length 224 into portal 26 MB=0x6019f06b4a7c0 00000400:00000200:14.0:1616513088.282025:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.71@tcp1 of length 224/224 into md 0xf97cc749 [1] + 0 00000400:00000200:14.0:1616513088.282029:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f704c8 00000400:00000200:14.0:1616513088.282032:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.282035:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.282040:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f704c8 00000800:00000200:14.0:1616513088.282043:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.282044:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.282046:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.282051:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.282054:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.282055:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513088.282064:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.71@tcp1 : PUT - for me 00000400:00000200:2.0:1616513088.282073:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.71@tcp1 of length 224 into portal 12 MB=0x6019f06b4a840 00000400:00000200:2.0:1616513088.282079:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.71@tcp1 of length 224/224 into md 0xf964d519 [1] + 69872 00000400:00000200:2.0:1616513088.282084:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.282087:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513088.282095:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.282098:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.282100:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513088.282105:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690632384194496 00010000:00000200:12.0:1616513088.282116:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1690632384194496/t0(0) o400->9444a01d-1682-db36-60c8-98dbea30d406@10.31.142.71@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.282131:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690632384194496, offset 192 00000400:00000200:12.0:1616513088.282137:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.71@tcp1 00000400:00000200:12.0:1616513088.282145:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.71@tcp1 routed destination 00000400:00000200:12.0:1616513088.282160:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.71@tcp1(10.31.142.71@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.282166:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513088.282172:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:22.0:1616513088.282175:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690632384194624 00000800:00000200:12.0:1616513088.282175:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513088.282177:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513088.282182:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.282186:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:22.0:1616513088.282188:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cde80 x1690632384194624/t0(0) o400->8d942ca8-a493-0d54-325d-e220ba0e3ad9@10.31.142.71@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513088.282189:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513088.282202:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690632384194624, offset 192 00000400:00000200:22.0:1616513088.282208:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.71@tcp1 00000400:00000200:22.0:1616513088.282216:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.71@tcp1 routed destination 00000400:00000200:22.0:1616513088.282231:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.71@tcp1(10.31.142.71@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.282237:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513088.282243:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.282245:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513088.282247:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513088.282248:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513088.282253:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513088.282254:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.282257:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.282257:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.282263:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.282269:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa08 00000800:00000200:18.0:1616513088.282272:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.282300:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.282358:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.282365:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513088.282367:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513088.282374:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.282380:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a2a8 00000800:00000200:4.0:1616513088.282384:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513088.282387:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.290551:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513088.290605:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.290614:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513088.290623:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.205@tcp1 : PUT - for me 00000400:00000200:2.0:1616513088.290632:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.205@tcp1 of length 224 into portal 26 MB=0x6058924abebc0 00000400:00000200:2.0:1616513088.290638:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.205@tcp1 of length 224/224 into md 0xf97cc759 [1] + 0 00000400:00000200:2.0:1616513088.290643:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71760 00000400:00000200:2.0:1616513088.290646:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.290649:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513088.290653:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71760 00000800:00000200:2.0:1616513088.290656:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.290657:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.290658:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.290662:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513088.290677:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694936444169152 00010000:00000200:12.0:1616513088.290688:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1694936444169152/t0(0) o400->4b8c23c4-4378-e24b-4b85-a71ddede7f48@10.31.132.205@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513088.290702:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694936444169152, offset 192 00000400:00000200:12.0:1616513088.290709:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.205@tcp1 00000400:00000200:12.0:1616513088.290717:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.205@tcp1 routed destination 00000400:00000200:12.0:1616513088.290732:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.205@tcp1(10.31.132.205@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.290739:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513088.290744:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513088.290747:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513088.290749:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513088.290754:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.290758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513088.290761:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513088.290766:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.290770:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.290772:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.290778:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.290782:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f7e8 00000800:00000200:14.0:1616513088.290785:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.290787:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.290858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.290868:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.290878:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513088.290880:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513088.290887:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.205@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.290896:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.205@tcp1 of length 224 into portal 12 MB=0x6058924abed80 00000400:00000200:26.0:1616513088.290902:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.205@tcp1 of length 224/224 into md 0xf964d519 [1] + 70096 00000400:00000200:26.0:1616513088.290907:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.290910:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.290917:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.290919:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.290921:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.290982:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694936444169600 00010000:00000200:22.0:1616513088.290995:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c692880 x1694936444169600/t0(0) o400->a86d452f-0213-93d7-15e6-491e624ae10f@10.31.132.205@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.291009:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694936444169600, offset 192 00000400:00000200:22.0:1616513088.291015:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.205@tcp1 00000400:00000200:22.0:1616513088.291023:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.205@tcp1 routed destination 00000400:00000200:22.0:1616513088.291038:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.205@tcp1(10.31.132.205@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.291044:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513088.291050:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.291052:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513088.291055:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513088.291059:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.291112:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.291125:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.291130:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.291133:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.291139:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.291143:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a5d8 00000800:00000200:8.0:1616513088.291147:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.291150:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.307305:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513088.307324:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.307333:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513088.307342:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.59@tcp1 : PUT - for me 00000400:00000200:4.0:1616513088.307352:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.59@tcp1 of length 224 into portal 26 MB=0x5f91b328c3940 00000400:00000200:4.0:1616513088.307359:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.59@tcp1 of length 224/224 into md 0xf97cc779 [1] + 0 00000400:00000200:4.0:1616513088.307364:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70ee0 00000400:00000200:4.0:1616513088.307366:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513088.307369:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513088.307374:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70ee0 00000800:00000200:28.0:1616513088.307377:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.307377:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.307379:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.307384:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.307387:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.307388:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513088.307392:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513088.307392:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513088.307397:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.87@tcp1 : PUT - for me 00000800:00000200:8.0:1616513088.307398:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513088.307399:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.59@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.307405:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.87@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.307408:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.87@tcp1 of length 224 into portal 26 MB=0x5ffde96563b00 00000400:00000200:4.0:1616513088.307408:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.59@tcp1 of length 224 into portal 12 MB=0x5f91b328c39c0 00000400:00000200:8.0:1616513088.307413:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.87@tcp1 of length 224 into portal 12 MB=0x5ffde96563b80 00000400:00000200:4.0:1616513088.307414:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.59@tcp1 of length 224/224 into md 0x865be291 [1] + 1008 00000400:00000200:28.0:1616513088.307415:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.87@tcp1 of length 224/224 into md 0xf97cc799 [1] + 0 00000400:00000200:4.0:1616513088.307418:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513088.307420:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70660 00000400:00000200:8.0:1616513088.307421:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.87@tcp1 of length 224/224 into md 0x865be291 [1] + 1232 00000400:00000200:28.0:1616513088.307423:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.307425:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513088.307425:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:28.0:1616513088.307426:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513088.307430:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70660 00000100:00000200:8.0:1616513088.307431:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513088.307431:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:4.0:1616513088.307445:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:4.0:1616513088.307447:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513088.307449:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.307450:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513088.307451:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.307453:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513088.307455:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.307455:0:6463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513088.307458:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.307459:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513088.307460:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681270091036992 00000800:00000200:11.2:1616513088.307463:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:12.0:1616513088.307470:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1681270091036992/t0(0) o400->9fbd6513-4181-e263-f25a-be8cc7666928@10.31.132.59@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.307483:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706353609472 00000100:00000200:4.0:1616513088.307484:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681270091037120 00000100:00000200:12.0:1616513088.307485:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681270091036992, offset 192 00000400:00000200:12.0:1616513088.307491:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.59@tcp1 00010000:00000200:4.0:1616513088.307494:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23f080 x1681270091037120/t0(0) o400->80c8e8ee-6e89-dcdc-bf43-1fc9064b3bcb@10.31.132.59@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513088.307495:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688706353609472/t0(0) o400->609eab61-f2c8-e573-7edc-2f5108aa6280@10.31.141.87@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513088.307499:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.59@tcp1 routed destination 00000100:00000200:16.0:1616513088.307503:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706353609600 00000100:00000200:4.0:1616513088.307508:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681270091037120, offset 192 00000100:00000200:24.0:1616513088.307510:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706353609472, offset 192 00000400:00000200:12.0:1616513088.307514:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.59@tcp1(10.31.132.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:4.0:1616513088.307514:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.59@tcp1 00010000:00000200:16.0:1616513088.307516:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8d100 x1688706353609600/t0(0) o400->f15d2c3c-f31e-d8f5-de80-31c7d429412d@10.31.141.87@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513088.307519:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.87@tcp1 00000800:00000200:12.0:1616513088.307520:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:4.0:1616513088.307521:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.59@tcp1 routed destination 00000100:00000200:16.0:1616513088.307530:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706353609600, offset 192 00000400:00000200:16.0:1616513088.307537:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.87@tcp1 00000400:00000200:4.0:1616513088.307537:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.59@tcp1(10.31.132.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513088.307544:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513088.307546:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.87@tcp1 routed destination 00000800:00000200:12.0:1616513088.307554:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513088.307556:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.87@tcp1 routed destination 00000800:00000200:12.0:1616513088.307557:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513088.307559:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513088.307561:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.87@tcp1(10.31.141.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513088.307563:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513088.307568:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:12.0:1616513088.307570:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513088.307571:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.87@tcp1(10.31.141.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.307578:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.307579:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513088.307579:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.307582:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513088.307582:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.307583:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513088.307584:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513088.307584:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.307587:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:4.0:1616513088.307587:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513088.307589:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513088.307589:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:16.0:1616513088.307593:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (25)-- 00000100:00000200:24.0:1616513088.307597:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513088.307599:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:7.2:1616513088.307603:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513088.307603:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:4.0:1616513088.307605:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513088.307607:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513088.307610:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:4.0:1616513088.307612:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513088.307615:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:4.0:1616513088.307617:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:28.0:1616513088.307620:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:4.0:1616513088.307620:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba5d8 00000400:00000200:28.0:1616513088.307622:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513088.307622:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.307624:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.307628:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.307631:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789430 00000800:00000200:28.0:1616513088.307634:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513088.307656:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513088.307663:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513088.307666:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.307673:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.307678:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f3a8 00000800:00000200:14.0:1616513088.307682:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.307684:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.307688:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.307689:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.307694:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.307697:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb42a8 00000800:00000200:14.0:1616513088.307699:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.307702:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.316306:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.316359:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.316368:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.316377:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.70@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.316387:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.70@tcp1 of length 224 into portal 26 MB=0x5ffd3f32da200 00000400:00000200:26.0:1616513088.316394:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.70@tcp1 of length 224/224 into md 0xf97cc7a9 [1] + 0 00000400:00000200:26.0:1616513088.316398:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70110 00000400:00000200:26.0:1616513088.316401:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.316404:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513088.316408:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70110 00000800:00000200:26.0:1616513088.316411:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.316414:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.316414:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513088.316423:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.316425:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.316426:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.316444:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.70@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.316453:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.70@tcp1 of length 224 into portal 12 MB=0x5ffd3f32da280 00000400:00000200:14.0:1616513088.316460:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.70@tcp1 of length 224/224 into md 0xf964d519 [1] + 70320 00000400:00000200:14.0:1616513088.316465:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.316468:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:24.0:1616513088.316471:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660666589696 00000800:00000200:14.0:1616513088.316474:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.316478:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.316479:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513088.316482:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688660666589696/t0(0) o400->38bb684c-5b45-6109-6253-aa9adfad8a03@10.31.133.70@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.316496:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660666589696, offset 192 00000400:00000200:24.0:1616513088.316502:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.70@tcp1 00000400:00000200:24.0:1616513088.316510:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.70@tcp1 routed destination 00000400:00000200:24.0:1616513088.316525:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.70@tcp1(10.31.133.70@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.316531:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.316537:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.316540:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.316542:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.316547:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.316550:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513088.316551:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660666589824 00000100:00000200:24.0:1616513088.316554:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:22.0:1616513088.316563:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a78480 x1688660666589824/t0(0) o400->ff5a5d4a-c486-bb45-d51e-a3cea2c66069@10.31.133.70@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.316577:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660666589824, offset 192 00000400:00000200:22.0:1616513088.316584:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.70@tcp1 00000400:00000200:22.0:1616513088.316591:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.70@tcp1 routed destination 00000800:00000200:2.0:1616513088.316602:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513088.316606:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.70@tcp1(10.31.133.70@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513088.316609:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.316612:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513088.316613:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513088.316613:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513088.316618:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.316621:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513088.316621:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513088.316623:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513088.316626:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5a90 00000800:00000200:22.0:1616513088.316628:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513088.316630:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.316677:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.316733:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.316740:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513088.316742:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513088.316749:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513088.316753:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b078 00000800:00000200:20.0:1616513088.316757:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.316760:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.325310:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.325322:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.325328:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.325335:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.242@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.325343:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.242@tcp1 of length 224 into portal 26 MB=0x5ffd43f0fac00 00000400:00000200:8.0:1616513088.325348:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.242@tcp1 of length 224/224 into md 0xf97cc7b9 [1] + 0 00000400:00000200:8.0:1616513088.325352:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70ff0 00000400:00000200:8.0:1616513088.325354:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.325357:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513088.325360:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70ff0 00000800:00000200:8.0:1616513088.325363:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.325365:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.325366:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.325368:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.325372:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513088.325374:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.325378:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.242@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.325382:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.242@tcp1 of length 224 into portal 12 MB=0x5ffd43f0fac80 00000400:00000200:8.0:1616513088.325386:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.242@tcp1 of length 224/224 into md 0x865be291 [1] + 1456 00000400:00000200:8.0:1616513088.325390:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.325391:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513088.325396:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.325398:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.325399:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.325421:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661939694592 00010000:00000200:24.0:1616513088.325431:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688661939694592/t0(0) o400->66afc219-eeae-2483-327e-4b8662e12217@10.31.132.242@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.325464:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661939694592, offset 192 00000100:00000200:16.0:1616513088.325467:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661939694720 00000400:00000200:24.0:1616513088.325470:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.242@tcp1 00000400:00000200:24.0:1616513088.325476:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.242@tcp1 routed destination 00010000:00000200:16.0:1616513088.325479:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89200 x1688661939694720/t0(0) o400->29c57c37-ddbc-fb7d-d615-a1ec950635f6@10.31.132.242@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513088.325490:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.242@tcp1(10.31.132.242@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513088.325493:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661939694720, offset 192 00000800:00000200:24.0:1616513088.325496:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513088.325500:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.242@tcp1 00000800:00000200:24.0:1616513088.325502:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.325504:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.325506:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513088.325507:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.242@tcp1 routed destination 00000800:00000200:24.0:1616513088.325511:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513088.325517:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513088.325517:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513088.325524:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.242@tcp1(10.31.132.242@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513088.325528:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513088.325530:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513088.325534:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513088.325536:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513088.325536:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513088.325539:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.325541:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513088.325542:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513088.325546:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513088.325548:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5a90 00000800:00000200:14.0:1616513088.325551:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513088.325551:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513088.325554:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513088.325563:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.325580:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513088.325583:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513088.325584:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513088.325589:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.325593:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789a08 00000800:00000200:4.0:1616513088.325596:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.330110:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.330120:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.330124:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.330130:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.142@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.330136:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.142@tcp1 of length 224 into portal 12 MB=0x5f919f3c94980 00000400:00000200:8.0:1616513088.330140:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.142@tcp1 of length 224/224 into md 0x865be291 [1] + 1680 00000400:00000200:8.0:1616513088.330144:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.330145:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513088.330148:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.330150:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.330151:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.330153:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.330220:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681264743106944 00010000:00000200:16.0:1616513088.330232:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8cc80 x1681264743106944/t0(0) o400->117df5f3-cefc-cc0d-c8bc-846c8e5b22fd@10.31.131.142@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.330246:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681264743106944, offset 192 00000400:00000200:16.0:1616513088.330253:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.142@tcp1 00000400:00000200:16.0:1616513088.330261:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.142@tcp1 routed destination 00000400:00000200:16.0:1616513088.330275:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.142@tcp1(10.31.131.142@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.330282:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.330288:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.330290:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.330292:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.330298:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.330303:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.330318:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.330324:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.330327:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.330334:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.330338:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788110 00000800:00000200:26.0:1616513088.330341:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.330344:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.330350:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513088.330354:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513088.330358:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.142@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.330366:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.142@tcp1 of length 224 into portal 26 MB=0x5f919f3c94900 00000400:00000200:26.0:1616513088.330374:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.142@tcp1 of length 224/224 into md 0xf97cc7d9 [1] + 0 00000400:00000200:26.0:1616513088.330378:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71540 00000400:00000200:26.0:1616513088.330381:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.330383:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513088.330387:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71540 00000800:00000200:26.0:1616513088.330389:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.330392:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.330393:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.330458:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681264743106816 00010000:00000200:24.0:1616513088.330468:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1681264743106816/t0(0) o400->5ac50e56-f726-1771-b4a4-37e4424e2969@10.31.131.142@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.330481:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681264743106816, offset 192 00000400:00000200:24.0:1616513088.330487:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.142@tcp1 00000400:00000200:24.0:1616513088.330494:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.142@tcp1 routed destination 00000400:00000200:24.0:1616513088.330508:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.142@tcp1(10.31.131.142@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.330514:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.330519:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.330522:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.330524:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.330528:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.330534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.330535:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513088.330549:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.330556:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.330559:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513088.330561:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.330565:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.330571:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5650 00000800:00000200:28.0:1616513088.330574:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.340546:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.340559:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.340565:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.340572:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.142@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.340579:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.142@tcp1 of length 224 into portal 26 MB=0x5ffd7c0940e80 00000400:00000200:8.0:1616513088.340584:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.142@tcp1 of length 224/224 into md 0xf97cc7e9 [1] + 0 00000400:00000200:8.0:1616513088.340588:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71a90 00000400:00000200:8.0:1616513088.340590:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.340592:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513088.340595:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71a90 00000800:00000200:8.0:1616513088.340598:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.340600:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.340601:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.340603:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.340657:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676997533312 00010000:00000200:24.0:1616513088.340667:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688676997533312/t0(0) o400->2cc2610d-ae5c-2c11-cfe8-c81b47dbcd79@10.31.129.142@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.340680:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688676997533312, offset 192 00000400:00000200:24.0:1616513088.340686:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.142@tcp1 00000400:00000200:24.0:1616513088.340693:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.142@tcp1 routed destination 00000400:00000200:24.0:1616513088.340708:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.142@tcp1(10.31.129.142@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.340715:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.340720:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.340722:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513088.340723:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.340724:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.340729:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.340734:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.340735:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513088.340776:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.340784:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513088.340792:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.142@tcp1 : PUT - for me 00000800:00000200:18.0:1616513088.340797:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513088.340800:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.142@tcp1 of length 224 into portal 12 MB=0x5ffd7c0941240 00000800:00000200:18.0:1616513088.340804:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.340806:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513088.340806:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.142@tcp1 of length 224/224 into md 0x865be291 [1] + 1904 00000400:00000200:4.0:1616513088.340811:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513088.340813:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:4.0:1616513088.340814:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513088.340820:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513088.340821:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4f68 00000800:00000200:4.0:1616513088.340823:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.340824:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513088.340825:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513088.340829:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513088.340830:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513088.340888:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676997534272 00010000:00000200:16.0:1616513088.340901:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb239200 x1688676997534272/t0(0) o400->d1dda560-ce65-c61d-fe9f-810bb438226e@10.31.129.142@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.340917:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688676997534272, offset 192 00000400:00000200:16.0:1616513088.340923:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.142@tcp1 00000400:00000200:16.0:1616513088.340930:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.142@tcp1 routed destination 00000400:00000200:16.0:1616513088.340945:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.142@tcp1(10.31.129.142@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.340952:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513088.340958:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.340960:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.340964:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513088.340969:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.340974:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.340981:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.340985:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.340986:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513088.340990:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.340991:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.340994:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788908 00000800:00000200:8.0:1616513088.340996:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.350769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.350826:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.350836:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513088.350844:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.8@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.350854:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.8@tcp1 of length 224 into portal 26 MB=0x5ffdc012ed340 00000400:00000200:28.0:1616513088.350860:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.8@tcp1 of length 224/224 into md 0xf97cc809 [1] + 0 00000400:00000200:28.0:1616513088.350865:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70908 00000400:00000200:28.0:1616513088.350867:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.350872:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513088.350876:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70908 00000800:00000200:28.0:1616513088.350879:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.350879:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.350882:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.350884:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.350933:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513088.350949:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695261287232 00010000:00000200:24.0:1616513088.350960:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688695261287232/t0(0) o400->fd355209-4db0-4a39-0bbc-da86dbb0c4a7@10.31.140.8@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.350974:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695261287232, offset 192 00000400:00000200:24.0:1616513088.350980:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.8@tcp1 00000800:00000200:2.0:1616513088.350986:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513088.350988:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.8@tcp1 routed destination 00000800:00000200:2.0:1616513088.350996:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513088.351003:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.8@tcp1(10.31.140.8@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.351009:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:2.0:1616513088.351012:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.8@tcp1 : PUT - for me 00000800:00000200:24.0:1616513088.351015:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.351018:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513088.351020:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:2.0:1616513088.351021:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.8@tcp1 of length 224 into portal 12 MB=0x5ffdc012ed3c0 00000800:00000200:24.0:1616513088.351025:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513088.351027:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.8@tcp1 of length 224/224 into md 0xf964d519 [1] + 70544 00000100:00000200:24.0:1616513088.351031:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513088.351032:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.351035:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.351041:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513088.351042:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513088.351045:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513088.351049:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513088.351050:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513088.351051:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.351052:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.351059:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.351065:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5078 00000800:00000200:26.0:1616513088.351068:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.351106:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695261287360 00010000:00000200:22.0:1616513088.351118:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cc800 x1688695261287360/t0(0) o400->bd3f5d93-130a-bcbb-ada7-afa462ee9a49@10.31.140.8@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.351132:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695261287360, offset 192 00000400:00000200:22.0:1616513088.351138:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.8@tcp1 00000400:00000200:22.0:1616513088.351146:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.8@tcp1 routed destination 00000400:00000200:22.0:1616513088.351161:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.8@tcp1(10.31.140.8@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.351167:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513088.351173:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.351176:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513088.351178:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513088.351182:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.351188:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.351261:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.351268:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513088.351271:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513088.351272:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.351278:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513088.351282:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bc28 00000800:00000200:20.0:1616513088.351286:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.352523:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.352573:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.352581:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.352588:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.59@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.352597:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.59@tcp1 of length 224 into portal 12 MB=0x603e1e329a240 00000400:00000200:14.0:1616513088.352602:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.59@tcp1 of length 224/224 into md 0xf964d519 [1] + 70768 00000400:00000200:14.0:1616513088.352607:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.352609:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513088.352615:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.352617:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.352619:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.352621:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.352678:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693118573945408 00010000:00000200:22.0:1616513088.352695:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7b600 x1693118573945408/t0(0) o400->be269075-b214-7a4b-17f8-be576a654b11@10.31.136.59@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.352715:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693118573945408, offset 192 00000400:00000200:22.0:1616513088.352721:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.59@tcp1 00000400:00000200:22.0:1616513088.352729:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.59@tcp1 routed destination 00000400:00000200:22.0:1616513088.352745:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.59@tcp1(10.31.136.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.352752:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.352758:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.352760:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.352763:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.352767:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.352771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513088.352777:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513088.352824:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.352830:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.352833:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.352840:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.352844:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b760 00000800:00000200:4.0:1616513088.352844:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513088.352847:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.352851:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513088.352854:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513088.352863:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.59@tcp1 : PUT - for me 00000400:00000200:4.0:1616513088.352872:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.59@tcp1 of length 224 into portal 26 MB=0x603e1e329a1c0 00000400:00000200:4.0:1616513088.352878:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.59@tcp1 of length 224/224 into md 0xf97cc829 [1] + 0 00000400:00000200:4.0:1616513088.352883:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70d48 00000400:00000200:4.0:1616513088.352886:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513088.352889:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513088.352893:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70d48 00000800:00000200:4.0:1616513088.352896:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.352899:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.352899:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.352903:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.352955:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693118573945280 00010000:00000200:24.0:1616513088.352966:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1693118573945280/t0(0) o400->24cca112-1a7f-e884-d2f2-5fc6f36cd22c@10.31.136.59@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.352980:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693118573945280, offset 192 00000400:00000200:24.0:1616513088.352986:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.59@tcp1 00000400:00000200:24.0:1616513088.352994:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.59@tcp1 routed destination 00000400:00000200:24.0:1616513088.353009:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.59@tcp1(10.31.136.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.353015:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.353021:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.353025:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.353027:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.353032:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.353037:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.353039:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513088.353045:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.353049:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.353051:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.353055:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.353059:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5cb0 00000800:00000200:8.0:1616513088.353062:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.353064:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.355365:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.355417:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.355426:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.355445:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.51@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.355455:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.51@tcp1 of length 224 into portal 26 MB=0x5ffd491204b80 00000400:00000200:14.0:1616513088.355462:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.51@tcp1 of length 224/224 into md 0xf97cc849 [1] + 0 00000400:00000200:14.0:1616513088.355474:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f702a8 00000800:00000200:2.0:1616513088.355474:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513088.355476:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.355479:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.355483:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f702a8 00000800:00000200:14.0:1616513088.355486:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.355489:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.355491:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.355546:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663316515712 00010000:00000200:24.0:1616513088.355556:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688663316515712/t0(0) o400->5175600a-d400-8340-77df-e37ed030491d@10.31.129.51@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.355571:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663316515712, offset 192 00000400:00000200:24.0:1616513088.355577:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.51@tcp1 00000400:00000200:24.0:1616513088.355586:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.51@tcp1 routed destination 00000400:00000200:24.0:1616513088.355600:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.51@tcp1(10.31.129.51@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.355607:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.355613:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.355615:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.355617:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.355622:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.355626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.355629:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513088.355678:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.355684:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.355687:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.355694:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.355699:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5210 00000800:00000200:26.0:1616513088.355703:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.355706:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.355707:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.355765:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.355775:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513088.355784:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.51@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.355793:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.51@tcp1 of length 224 into portal 12 MB=0x5ffd491204f80 00000400:00000200:28.0:1616513088.355799:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.51@tcp1 of length 224/224 into md 0x865be291 [1] + 2128 00000400:00000200:28.0:1616513088.355805:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.355807:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513088.355814:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.355817:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.355818:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.355823:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.355878:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663316516736 00010000:00000200:16.0:1616513088.355890:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa4c80 x1688663316516736/t0(0) o400->6ecd381c-2ff6-257d-7717-8663bb03b897@10.31.129.51@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.355905:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663316516736, offset 192 00000400:00000200:16.0:1616513088.355911:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.51@tcp1 00000400:00000200:16.0:1616513088.355931:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.51@tcp1 routed destination 00000400:00000200:16.0:1616513088.355945:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.51@tcp1(10.31.129.51@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.355951:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513088.355957:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.355959:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.355961:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513088.355966:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.355971:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.355979:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.355982:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.355983:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.355988:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.355990:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789540 00000800:00000200:8.0:1616513088.355992:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.355994:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.365520:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513088.365538:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.365547:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513088.365553:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513088.365557:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.62@tcp1 : PUT - for me 00000400:00000200:4.0:1616513088.365567:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.62@tcp1 of length 224 into portal 26 MB=0x60520032a99c0 00000400:00000200:4.0:1616513088.365573:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.62@tcp1 of length 224/224 into md 0xf97cc869 [1] + 0 00000400:00000200:4.0:1616513088.365578:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71430 00000400:00000200:4.0:1616513088.365581:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513088.365583:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513088.365588:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71430 00000800:00000200:4.0:1616513088.365591:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.365594:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513088.365596:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.365653:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694484910479808 00010000:00000200:24.0:1616513088.365664:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1694484910479808/t0(0) o400->c61f9078-c250-f3e4-361c-41b35a9d034a@10.31.129.62@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.365678:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694484910479808, offset 192 00000400:00000200:24.0:1616513088.365684:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.62@tcp1 00000400:00000200:24.0:1616513088.365692:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.62@tcp1 routed destination 00000400:00000200:24.0:1616513088.365707:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.62@tcp1(10.31.129.62@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.365714:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.365720:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.365723:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.365725:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.365730:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.365735:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.365737:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.365744:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.365753:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.365760:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.365767:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.62@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.365775:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.62@tcp1 of length 224 into portal 12 MB=0x60520032a9ec0 00000400:00000200:8.0:1616513088.365780:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.62@tcp1 of length 224/224 into md 0x865be291 [1] + 2352 00000400:00000200:8.0:1616513088.365785:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.365787:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513088.365789:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513088.365793:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.365802:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513088.365802:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:8.0:1616513088.365804:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513088.365805:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513088.365805:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513088.365806:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513088.365812:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.365817:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb57e8 00000800:00000200:2.0:1616513088.365821:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513088.365861:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694484910481088 00010000:00000200:16.0:1616513088.365874:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8b600 x1694484910481088/t0(0) o400->43fe633e-fca8-2e6e-bde8-f8f14439ec1d@10.31.129.62@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.365888:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694484910481088, offset 192 00000400:00000200:16.0:1616513088.365894:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.62@tcp1 00000400:00000200:16.0:1616513088.365902:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.62@tcp1 routed destination 00000400:00000200:16.0:1616513088.365917:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.62@tcp1(10.31.129.62@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.365923:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513088.365929:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.365932:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.365934:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513088.365938:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.365943:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.365953:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.365958:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.365960:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513088.365962:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.365966:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.365970:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789870 00000800:00000200:28.0:1616513088.365973:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513088.378520:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927280, portal 4 00000100:00000200:14.0:1616513088.378526:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927280, offset 0 00000400:00000200:14.0:1616513088.378533:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:14.0:1616513088.378538:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:14.0:1616513088.378551:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513088.378558:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:14.0:1616513088.378564:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513088.378567:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:14.0:1616513088.378569:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:14.0:1616513088.378574:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513088.378620:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:3.0:1616513088.378657:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927296, portal 4 00000100:00000200:3.0:1616513088.378663:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927296, offset 0 00000400:00000200:3.0:1616513088.378670:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:20.0:1616513088.378676:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:3.0:1616513088.378678:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:20.0:1616513088.378683:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513088.378686:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513088.378687:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:3.0:1616513088.378693:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513088.378694:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513088.378701:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf82400 x1693067725927280/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:3.0:1616513088.378701:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513088.378708:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513088.378712:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:20.0:1616513088.378713:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47565d8 00000800:00000200:3.0:1616513088.378714:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513088.378718:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:3.0:1616513088.378720:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513088.378725:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513088.378778:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513088.378786:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513088.378789:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513088.378808:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513088.378814:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456e8d80 x1693067725927296/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513088.378827:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2dd0 00000800:00000200:4.0:1616513088.378831:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513088.378834:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513088.378839:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:8.0:1616513088.378840:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:4.0:1616513088.378846:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:4.0:1616513088.378853:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627370 00000800:00000200:5.2:1616513088.378856:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:4.0:1616513088.378859:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d9811 [1] + 192 00000400:00000200:4.0:1616513088.378863:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513088.378866:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000100:00000200:4.0:1616513088.378867:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf82400 x1693067725927280/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513088.378872:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:28.0:1616513088.378875:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513088.378880:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:4.0:1616513088.378880:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513088.378882:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:4.0:1616513088.378884:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:20.0:1616513088.378887:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627380 00000100:00000200:14.0:1616513088.378889:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf82400 x1693067725927280/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513088.378902:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf82400 x1693067725927280/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513088.378912:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757980 00000400:00000200:20.0:1616513088.378917:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d9821 [1] + 192 00000400:00000200:20.0:1616513088.378923:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.378928:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456e8d80 x1693067725927296/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513088.378943:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513088.378945:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513088.378947:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:3.0:1616513088.378954:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456e8d80 x1693067725927296/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513088.378967:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456e8d80 x1693067725927296/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513088.378978:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:7.2:1616513088.380668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.380722:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.380732:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.380741:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.56@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.380750:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.56@tcp1 of length 224 into portal 26 MB=0x6052795c13d80 00000400:00000200:26.0:1616513088.380757:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.56@tcp1 of length 224/224 into md 0xf97cc889 [1] + 0 00000400:00000200:26.0:1616513088.380762:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71078 00000400:00000200:26.0:1616513088.380767:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.380769:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513088.380774:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71078 00000800:00000200:26.0:1616513088.380777:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.380777:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.380780:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.380781:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.380836:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694517434596736 00010000:00000200:24.0:1616513088.380846:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1694517434596736/t0(0) o400->71a8df32-a32a-4df7-ddbc-0e6d03e71c51@10.31.133.56@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.380859:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694517434596736, offset 192 00000400:00000200:24.0:1616513088.380865:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.56@tcp1 00000400:00000200:24.0:1616513088.380872:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.56@tcp1 routed destination 00000400:00000200:24.0:1616513088.380886:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.56@tcp1(10.31.133.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.380892:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.380898:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.380901:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.380903:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.380907:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.380911:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.380913:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513088.380964:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.380971:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.380974:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.380980:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.380986:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb43b8 00000800:00000200:14.0:1616513088.380990:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.380993:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.381018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.381028:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.381034:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.381041:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.56@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.381049:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.56@tcp1 of length 224 into portal 12 MB=0x6052795c13e40 00000400:00000200:8.0:1616513088.381054:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.56@tcp1 of length 224/224 into md 0x865be291 [1] + 2576 00000400:00000200:8.0:1616513088.381059:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.381061:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513088.381067:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.381069:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.381070:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.381073:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.381131:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694517434596928 00010000:00000200:16.0:1616513088.381144:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb88900 x1694517434596928/t0(0) o400->cecab1d8-eaf5-5fdd-0071-bb779398954a@10.31.133.56@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.381158:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694517434596928, offset 192 00000400:00000200:16.0:1616513088.381167:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.56@tcp1 00000400:00000200:16.0:1616513088.381174:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.56@tcp1 routed destination 00000400:00000200:16.0:1616513088.381189:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.56@tcp1(10.31.133.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.381196:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513088.381202:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.381204:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.381206:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513088.381211:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.381216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.381227:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.381233:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513088.381236:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513088.381242:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.381246:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788a18 00000800:00000200:4.0:1616513088.381250:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513088.381253:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.404166:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.404222:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.404232:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.404242:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.124@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.404253:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.124@tcp1 of length 224 into portal 26 MB=0x600853a144980 00000400:00000200:18.0:1616513088.404259:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.124@tcp1 of length 224/224 into md 0xf97cc8b9 [1] + 0 00000400:00000200:18.0:1616513088.404264:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70e58 00000400:00000200:18.0:1616513088.404267:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.404270:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.404274:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70e58 00000800:00000200:26.0:1616513088.404278:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.404278:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.404281:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.404282:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.404299:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.404311:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.404318:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.404327:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.124@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.404335:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.124@tcp1 of length 224 into portal 12 MB=0x600853a144c40 00000400:00000200:8.0:1616513088.404341:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.124@tcp1 of length 224/224 into md 0x865be291 [1] + 2800 00000400:00000200:8.0:1616513088.404346:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.404349:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:24.0:1616513088.404352:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689422065322368 00010000:00000200:24.0:1616513088.404364:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1689422065322368/t0(0) o400->c817e358-99aa-a55f-63a0-d39514bc75a8@10.31.141.124@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:8.0:1616513088.404366:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.404369:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.404371:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.404373:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.404379:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689422065322368, offset 192 00000400:00000200:24.0:1616513088.404385:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.124@tcp1 00000400:00000200:24.0:1616513088.404392:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.124@tcp1 routed destination 00000400:00000200:24.0:1616513088.404407:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.124@tcp1(10.31.141.124@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.404414:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.404420:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.404423:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.404425:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.404430:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.404434:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513088.404448:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689422065323072 00000100:00000200:24.0:1616513088.404455:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:16.0:1616513088.404460:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8c800 x1689422065323072/t0(0) o400->dc78b9ba-e1f6-8544-214d-286c43f8ba02@10.31.141.124@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.404474:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689422065323072, offset 192 00000400:00000200:16.0:1616513088.404479:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.124@tcp1 00000400:00000200:16.0:1616513088.404486:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.124@tcp1 routed destination 00000800:00000200:14.0:1616513088.404500:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513088.404501:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.124@tcp1(10.31.141.124@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.404507:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513088.404507:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.404510:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513088.404512:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.404515:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.404517:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513088.404519:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513088.404521:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513088.404524:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4dd0 00000800:00000200:11.2:1616513088.404525:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513088.404527:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.404530:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513088.404537:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.404543:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513088.404546:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513088.404552:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.404556:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788770 00000800:00000200:4.0:1616513088.404559:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513088.404562:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.416772:0:50367:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.416826:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.416837:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.416846:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.169@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.416856:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.169@tcp1 of length 224 into portal 12 MB=0x5ffde0b886200 00000400:00000200:26.0:1616513088.416862:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.169@tcp1 of length 224/224 into md 0xf964d519 [1] + 70992 00000400:00000200:26.0:1616513088.416867:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.416870:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.416878:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.416881:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.416882:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.416886:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.416902:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.416925:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.416931:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.416938:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.169@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.416945:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.169@tcp1 of length 224 into portal 26 MB=0x5ffde0b885e80 00000100:00000200:22.0:1616513088.416951:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704024863232 00000400:00000200:8.0:1616513088.416951:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.169@tcp1 of length 224/224 into md 0xf97cc8c9 [1] + 0 00000400:00000200:8.0:1616513088.416955:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71210 00000400:00000200:8.0:1616513088.416957:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.416959:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513088.416963:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71210 00010000:00000200:22.0:1616513088.416965:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c697980 x1688704024863232/t0(0) o400->74edb370-69ac-4927-68ab-8f3d9ebe7649@10.31.136.169@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:8.0:1616513088.416966:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.416968:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.416969:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.416972:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513088.416979:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704024863232, offset 192 00000400:00000200:22.0:1616513088.416985:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.169@tcp1 00000400:00000200:22.0:1616513088.416993:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.169@tcp1 routed destination 00000400:00000200:22.0:1616513088.417008:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.169@tcp1(10.31.136.169@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.417014:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.417020:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.417023:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.417025:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.417030:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704024862336 00000800:00000200:22.0:1616513088.417030:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.417035:0:50367:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:24.0:1616513088.417041:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688704024862336/t0(0) o400->378dfa33-f85d-f372-2754-1a34577e6fdf@10.31.136.169@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513088.417047:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.417054:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:24.0:1616513088.417055:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704024862336, offset 192 00000400:00000200:14.0:1616513088.417057:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513088.417063:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.169@tcp1 00000400:00000200:14.0:1616513088.417064:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.417067:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b5c8 00000400:00000200:24.0:1616513088.417071:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.169@tcp1 routed destination 00000800:00000200:14.0:1616513088.417072:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.417074:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513088.417086:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.169@tcp1(10.31.136.169@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.417092:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.417098:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.417101:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.417103:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.417108:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.417112:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.417114:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513088.417165:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.417172:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513088.417175:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513088.417181:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.417187:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5b18 00000800:00000200:4.0:1616513088.417192:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513088.417195:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.419719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.419773:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.419782:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.419791:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.419801:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d0322bb600 00000400:00000200:18.0:1616513088.419807:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0xf964d519 [1] + 71216 00000400:00000200:18.0:1616513088.419812:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.419815:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513088.419821:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.419824:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.419826:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.419830:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.419885:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042590070272 00010000:00000200:22.0:1616513088.419973:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b9200 x1693042590070272/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:714/0 lens 440/368 e 0 to 0 dl 1616513094 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513088.419987:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042590070272, offset 192 00000400:00000200:22.0:1616513088.419993:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:22.0:1616513088.420001:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:22.0:1616513088.420016:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.420023:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.420028:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.420034:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.420038:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.420043:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.420046:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.420057:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.420064:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.420067:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.420073:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.420077:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bba0 00000800:00000200:14.0:1616513088.420081:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.420084:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.430606:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.430661:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.430670:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.430679:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.196@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.430688:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.196@tcp1 of length 224 into portal 26 MB=0x5ffd43fc28000 00000400:00000200:26.0:1616513088.430695:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.196@tcp1 of length 224/224 into md 0xf97cc8e9 [1] + 0 00000400:00000200:26.0:1616513088.430699:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70cc0 00000400:00000200:26.0:1616513088.430702:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.430705:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513088.430709:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70cc0 00000800:00000200:26.0:1616513088.430712:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.430715:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.430715:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.430722:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.430725:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.430725:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.430734:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.196@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.430743:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.196@tcp1 of length 224 into portal 12 MB=0x5ffd43fc28180 00000400:00000200:18.0:1616513088.430749:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.196@tcp1 of length 224/224 into md 0xf964d519 [1] + 71656 00000400:00000200:18.0:1616513088.430754:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.430757:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513088.430765:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.430768:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.430770:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.430773:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661951414272 00010000:00000200:24.0:1616513088.430784:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688661951414272/t0(0) o400->2931c369-580f-0701-daee-af83ec0e579c@10.31.136.196@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.430798:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661951414272, offset 192 00000400:00000200:24.0:1616513088.430804:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.196@tcp1 00000400:00000200:24.0:1616513088.430811:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.196@tcp1 routed destination 00000400:00000200:24.0:1616513088.430826:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.196@tcp1(10.31.136.196@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513088.430830:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661951414656 00000800:00000200:24.0:1616513088.430833:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.430838:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.430841:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:22.0:1616513088.430842:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bf080 x1688661951414656/t0(0) o400->c817dfb7-da89-976e-52f1-649338b8902e@10.31.136.196@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513088.430843:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.430848:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.430854:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.430854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513088.430856:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661951414656, offset 192 00000400:00000200:22.0:1616513088.430863:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.196@tcp1 00000800:00000200:8.0:1616513088.430865:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513088.430870:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.196@tcp1 routed destination 00000800:00000200:8.0:1616513088.430870:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.430872:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.430878:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.430883:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4990 00000400:00000200:22.0:1616513088.430884:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.196@tcp1(10.31.136.196@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513088.430886:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.430889:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513088.430891:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.430897:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.430899:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.430901:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.430906:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.430910:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.430970:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.430977:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.430980:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513088.430981:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513088.430986:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.430990:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bd38 00000800:00000200:2.0:1616513088.430994:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.460806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.460859:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.460869:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.460879:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.157@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.460889:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.157@tcp1 of length 224 into portal 26 MB=0x603e0f43f0600 00000400:00000200:18.0:1616513088.460895:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.157@tcp1 of length 224/224 into md 0xf97cc8f9 [1] + 0 00000400:00000200:18.0:1616513088.460900:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f717e8 00000400:00000200:18.0:1616513088.460902:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.460905:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513088.460913:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.460921:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f717e8 00000800:00000200:18.0:1616513088.460924:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.460927:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.460929:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.460932:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513088.460985:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114565592576 00000800:00000200:4.0:1616513088.460990:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513088.460996:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1693114565592576/t0(0) o400->b6a2bcd4-66c8-f278-0ac3-9d296633c6bc@10.31.136.157@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:4.0:1616513088.461000:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513088.461009:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.157@tcp1 : PUT - for me 00000100:00000200:24.0:1616513088.461010:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114565592576, offset 192 00000400:00000200:24.0:1616513088.461016:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.157@tcp1 00000400:00000200:4.0:1616513088.461018:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.157@tcp1 of length 224 into portal 12 MB=0x603e0f43f0680 00000400:00000200:24.0:1616513088.461023:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.157@tcp1 routed destination 00000400:00000200:4.0:1616513088.461025:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.157@tcp1 of length 224/224 into md 0x865be291 [1] + 3024 00000400:00000200:4.0:1616513088.461030:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513088.461032:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513088.461038:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.157@tcp1(10.31.136.157@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513088.461047:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513088.461049:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513088.461049:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513088.461050:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.461053:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513088.461054:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.461057:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.461059:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.461064:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.461069:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.461071:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513088.461078:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.461084:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.461086:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.461092:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.461096:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5430 00000800:00000200:8.0:1616513088.461099:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.461102:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.461104:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114565592704 00010000:00000200:16.0:1616513088.461117:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23b180 x1693114565592704/t0(0) o400->ec6a53fa-d2f3-a78d-d0a4-416902e77a32@10.31.136.157@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.461132:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114565592704, offset 192 00000400:00000200:16.0:1616513088.461138:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.157@tcp1 00000400:00000200:16.0:1616513088.461146:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.157@tcp1 routed destination 00000400:00000200:16.0:1616513088.461161:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.157@tcp1(10.31.136.157@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.461167:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.461173:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.461178:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.461180:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.461185:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.461189:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.461259:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.461268:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.461270:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.461277:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.461281:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788220 00000800:00000200:26.0:1616513088.461285:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.461288:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.494808:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.494861:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.494870:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.494879:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.221@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.494889:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.221@tcp1 of length 224 into portal 26 MB=0x60182e0f2cc80 00000400:00000200:14.0:1616513088.494895:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.221@tcp1 of length 224/224 into md 0xf97cc919 [1] + 0 00000400:00000200:14.0:1616513088.494899:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71ed0 00000400:00000200:14.0:1616513088.494902:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.494904:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.494908:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71ed0 00000800:00000200:14.0:1616513088.494911:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.494914:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.494915:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.494919:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.494972:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511491648640 00010000:00000200:24.0:1616513088.494982:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1690511491648640/t0(0) o400->db95d389-9d94-26ae-d845-bb7249588b50@10.31.132.221@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.494997:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511491648640, offset 192 00000400:00000200:24.0:1616513088.495003:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.221@tcp1 00000400:00000200:24.0:1616513088.495011:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.221@tcp1 routed destination 00000400:00000200:24.0:1616513088.495026:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.221@tcp1(10.31.132.221@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.495032:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.495038:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.495041:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.495043:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.495048:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.495055:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.495088:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.495144:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.495154:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513088.495163:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.221@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.495172:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.221@tcp1 of length 224 into portal 12 MB=0x60182e0f2ce80 00000400:00000200:20.0:1616513088.495178:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.221@tcp1 of length 224/224 into md 0x865be291 [1] + 3248 00000400:00000200:20.0:1616513088.495184:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.495186:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.495193:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513088.495196:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513088.495197:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:20.0:1616513088.495204:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513088.495205:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513088.495207:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513088.495208:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513088.495214:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.495220:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4c38 00000800:00000200:4.0:1616513088.495224:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.495257:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511491649152 00010000:00000200:16.0:1616513088.495269:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad1200 x1690511491649152/t0(0) o400->9801eb6d-eda6-268d-f52a-aef4b5573157@10.31.132.221@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.495284:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511491649152, offset 192 00000400:00000200:16.0:1616513088.495290:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.221@tcp1 00000400:00000200:16.0:1616513088.495298:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.221@tcp1 routed destination 00000400:00000200:16.0:1616513088.495313:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.221@tcp1(10.31.132.221@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.495319:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.495325:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.495328:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.495330:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.495335:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.495339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.495392:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.495399:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.495402:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.495409:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.495413:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789650 00000800:00000200:26.0:1616513088.495416:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.495419:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.502137:0:50377:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.502192:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.502201:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.502210:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.163@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.502220:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.163@tcp1 of length 224 into portal 26 MB=0x5ffdbf9870180 00000400:00000200:18.0:1616513088.502227:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.163@tcp1 of length 224/224 into md 0xf97cc929 [1] + 0 00000400:00000200:18.0:1616513088.502231:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71b18 00000400:00000200:18.0:1616513088.502234:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.502237:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.502241:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71b18 00000800:00000200:18.0:1616513088.502244:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.502246:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.502247:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.502250:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.502262:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513088.502305:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695132848512 00010000:00000200:24.0:1616513088.502316:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688695132848512/t0(0) o400->37565a5f-be40-a2cb-bffd-a6f1783601d1@10.31.140.163@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513088.502326:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.502331:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695132848512, offset 192 00000400:00000200:24.0:1616513088.502337:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.163@tcp1 00000400:00000200:24.0:1616513088.502345:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.163@tcp1 routed destination 00000800:00000200:28.0:1616513088.502348:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513088.502350:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513088.502360:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.163@tcp1(10.31.140.163@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.502367:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513088.502369:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.163@tcp1 : PUT - for me 00000800:00000200:24.0:1616513088.502372:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.502375:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513088.502379:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.163@tcp1 of length 224 into portal 12 MB=0x5ffdbf9870540 00000800:00000200:24.0:1616513088.502379:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513088.502384:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.502385:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.163@tcp1 of length 224/224 into md 0x865be291 [1] + 3472 00000800:00000200:11.2:1616513088.502388:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513088.502390:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513088.502390:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513088.502392:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513088.502399:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513088.502404:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513088.502406:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.502409:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.502413:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.502415:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513088.502420:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.502425:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4c38 00000800:00000200:28.0:1616513088.502427:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.502430:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.502478:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695132849472 00010000:00000200:16.0:1616513088.502490:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa1200 x1688695132849472/t0(0) o400->fd768d79-092e-3ee3-67bf-e7d49c7dab7e@10.31.140.163@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.502504:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695132849472, offset 192 00000400:00000200:16.0:1616513088.502511:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.163@tcp1 00000400:00000200:16.0:1616513088.502518:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.163@tcp1 routed destination 00000400:00000200:16.0:1616513088.502533:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.163@tcp1(10.31.140.163@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.502540:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.502546:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.502548:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.502551:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.502556:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.502560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.502614:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.502621:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.502624:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.502630:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.502634:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7883b8 00000800:00000200:26.0:1616513088.502638:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.502641:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.538721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.538775:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.538785:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.538796:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.107.15@tcp2 : PUT - for me 00000400:00000200:14.0:1616513088.538805:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.107.15@tcp2 of length 224 into portal 26 MB=0x601c5d8ca2bc0 00000400:00000200:14.0:1616513088.538811:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.107.15@tcp2 of length 224/224 into md 0xf97cc959 [1] + 0 00000400:00000200:14.0:1616513088.538816:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70550 00000400:00000200:14.0:1616513088.538819:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.538822:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.538826:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70550 00000800:00000200:14.0:1616513088.538829:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.538830:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.538844:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513088.538846:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.538849:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513088.538849:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.538858:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.107.15@tcp2 : PUT - for me 00000400:00000200:18.0:1616513088.538866:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.107.15@tcp2 of length 224 into portal 12 MB=0x601c5d8ca2c40 00000400:00000200:18.0:1616513088.538872:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.107.15@tcp2 of length 224/224 into md 0xf964d519 [1] + 71880 00000400:00000200:18.0:1616513088.538879:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.538882:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513088.538889:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.538892:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.538895:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.538911:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690799117577152 00010000:00000200:24.0:1616513088.538922:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1690799117577152/t0(0) o400->322fb9a7-6665-5951-5234-23ddb7b46e72@10.242.107.15@tcp2:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.538937:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690799117577152, offset 192 00000400:00000200:24.0:1616513088.538943:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.107.15@tcp2 00000400:00000200:24.0:1616513088.538951:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.107.15@tcp2 routed destination 00000100:00000200:22.0:1616513088.538955:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690799117577280 00000400:00000200:24.0:1616513088.538966:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.107.15@tcp2(10.242.107.15@tcp2:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:22.0:1616513088.538967:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b9b00 x1690799117577280/t0(0) o400->747eee4c-2b49-8f08-8c38-4c795987842c@10.242.107.15@tcp2:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513088.538973:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.538979:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.538981:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:22.0:1616513088.538982:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690799117577280, offset 192 00000800:00000200:24.0:1616513088.538983:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.538988:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513088.538988:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.107.15@tcp2 00000800:00000200:7.2:1616513088.538993:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.538995:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513088.538995:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.107.15@tcp2 routed destination 00000400:00000200:22.0:1616513088.539010:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.107.15@tcp2(10.242.107.15@tcp2:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.539017:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513088.539023:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.539025:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513088.539028:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513088.539032:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513088.539045:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.539053:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.539055:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.539056:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513088.539062:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.539068:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4990 00000800:00000200:2.0:1616513088.539071:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.539074:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.539085:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.539091:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.539093:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.539100:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.539103:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4acc0 00000800:00000200:8.0:1616513088.539107:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.539109:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.550398:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513088.550428:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513088.550464:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.550474:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.550483:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.23@tcp1 : PUT - for me 00000800:00000200:20.0:1616513088.550486:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513088.550492:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.23@tcp1 of length 224 into portal 12 MB=0x60108770b4480 00000800:00000200:20.0:1616513088.550495:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513088.550498:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.23@tcp1 of length 224/224 into md 0xf964d519 [1] + 72104 00000400:00000200:26.0:1616513088.550504:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513088.550505:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.23@tcp1 : PUT - for me 00000100:00000200:26.0:1616513088.550506:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.550513:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513088.550513:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.23@tcp1 of length 224 into portal 26 MB=0x60108770b4400 00000800:00000200:26.0:1616513088.550515:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.550517:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513088.550520:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.23@tcp1 of length 224/224 into md 0xf97cc969 [1] + 0 00000800:00000200:18.0:1616513088.550520:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513088.550525:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71100 00000400:00000200:20.0:1616513088.550527:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.550530:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513088.550535:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71100 00000800:00000200:20.0:1616513088.550538:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.550541:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.550542:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513088.550546:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513088.550579:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689985728857216 00010000:00000200:22.0:1616513088.550592:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c692400 x1689985728857216/t0(0) o400->c2a4252c-816f-7638-ef13-2b6d8b4f0b30@10.31.132.23@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513088.550601:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689985728857088 00000100:00000200:22.0:1616513088.550606:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689985728857216, offset 192 00010000:00000200:24.0:1616513088.550612:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1689985728857088/t0(0) o400->422bca50-9051-df7b-8c64-1c659bb00863@10.31.132.23@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513088.550618:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.23@tcp1 00000400:00000200:22.0:1616513088.550625:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.23@tcp1 routed destination 00000100:00000200:24.0:1616513088.550626:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689985728857088, offset 192 00000400:00000200:24.0:1616513088.550632:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.23@tcp1 00000400:00000200:22.0:1616513088.550640:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.23@tcp1(10.31.132.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.550647:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513088.550649:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.23@tcp1 routed destination 00000800:00000200:22.0:1616513088.550652:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.550655:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513088.550657:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513088.550662:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513088.550664:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.23@tcp1(10.31.132.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513088.550668:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.550671:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.550676:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513088.550677:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513088.550679:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.550681:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513088.550682:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.550684:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513088.550686:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513088.550690:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513088.550690:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.550692:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:8.0:1616513088.550693:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4add0 00000800:00000200:8.0:1616513088.550696:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.550699:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513088.550701:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.550708:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.550711:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.550717:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.550723:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4dd0 00000800:00000200:14.0:1616513088.550726:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.550729:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.551632:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.551641:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.551647:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.551654:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.244@tcp1 : PUT - for me 00000800:00000200:18.0:1616513088.551655:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513088.551661:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.244@tcp1 of length 224 into portal 26 MB=0x6000bd13c2340 00000800:00000200:18.0:1616513088.551664:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513088.551665:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.551671:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.244@tcp1 of length 224/224 into md 0xf97cc979 [1] + 0 00000400:00000200:18.0:1616513088.551673:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.244@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.551676:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71c28 00000400:00000200:26.0:1616513088.551678:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.551680:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.551681:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.244@tcp1 of length 224 into portal 12 MB=0x6000bd13c23c0 00000400:00000200:26.0:1616513088.551684:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71c28 00000800:00000200:26.0:1616513088.551686:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.551688:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.551689:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.244@tcp1 of length 224/224 into md 0xf964d519 [1] + 72328 00000800:00000200:26.0:1616513088.551690:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.551694:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513088.551696:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688900615283520 00000100:00000200:18.0:1616513088.551696:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:24.0:1616513088.551702:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688900615283520/t0(0) o400->a2a719e5-a051-9b08-66e0-6972de168ab3@10.31.141.244@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513088.551705:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.551708:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.551710:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.551715:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688900615283520, offset 192 00000100:00000200:22.0:1616513088.551716:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688900615283648 00000400:00000200:24.0:1616513088.551719:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.244@tcp1 00010000:00000200:22.0:1616513088.551722:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120bf00 x1688900615283648/t0(0) o400->f02b4954-fee0-c5d6-5d2b-8a5b392c76a3@10.31.141.244@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513088.551725:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.244@tcp1 routed destination 00000100:00000200:22.0:1616513088.551733:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688900615283648, offset 192 00000400:00000200:24.0:1616513088.551737:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.244@tcp1(10.31.141.244@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513088.551737:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.244@tcp1 00000800:00000200:24.0:1616513088.551743:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513088.551743:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.244@tcp1 routed destination 00000800:00000200:24.0:1616513088.551747:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.551749:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.551751:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.551754:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513088.551755:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.244@tcp1(10.31.141.244@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513088.551758:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.551759:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513088.551761:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.551765:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.551768:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.551769:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.551772:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.551776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.551787:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.551796:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.551799:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.551800:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513088.551805:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.551809:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a3b8 00000800:00000200:4.0:1616513088.551812:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513088.551813:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513088.551818:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513088.551821:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513088.551828:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.551834:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb57e8 00000800:00000200:4.0:1616513088.551837:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513088.551840:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.553216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.553274:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.553290:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.553299:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.200@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.553308:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.200@tcp1 of length 224 into portal 26 MB=0x5ffd3ff7c1ec0 00000400:00000200:26.0:1616513088.553314:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.200@tcp1 of length 224/224 into md 0xf97cc999 [1] + 0 00000400:00000200:26.0:1616513088.553319:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71e48 00000400:00000200:26.0:1616513088.553321:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.553324:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513088.553328:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71e48 00000800:00000200:26.0:1616513088.553331:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.553334:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.553335:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.553339:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.553375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513088.553389:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660873060032 00010000:00000200:24.0:1616513088.553399:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688660873060032/t0(0) o400->5b0da800-74ad-556a-73be-efbb7d67f5cf@10.31.128.200@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.553411:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660873060032, offset 192 00000400:00000200:24.0:1616513088.553417:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.200@tcp1 00000400:00000200:24.0:1616513088.553424:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.200@tcp1 routed destination 00000800:00000200:14.0:1616513088.553428:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513088.553448:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.200@tcp1(10.31.128.200@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513088.553449:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.553455:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513088.553457:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.200@tcp1 : PUT - for me 00000800:00000200:24.0:1616513088.553460:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.553462:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.553464:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513088.553466:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.200@tcp1 of length 224 into portal 12 MB=0x5ffd3ff7c2340 00000800:00000200:24.0:1616513088.553469:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513088.553472:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.200@tcp1 of length 224/224 into md 0xf964d519 [1] + 72552 00000100:00000200:24.0:1616513088.553475:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513088.553477:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.553480:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513088.553487:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.553490:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.553492:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.553495:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.553513:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.553524:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.553528:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.553529:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.553534:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:28.0:1616513088.553537:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.553538:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5cb0 00000800:00000200:8.0:1616513088.553541:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513088.553547:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660873061184 00010000:00000200:22.0:1616513088.553558:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7da00 x1688660873061184/t0(0) o400->052b5b88-8e05-9bdf-ccd2-2b8a860cc89f@10.31.128.200@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.553572:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660873061184, offset 192 00000400:00000200:22.0:1616513088.553577:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.200@tcp1 00000400:00000200:22.0:1616513088.553584:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.200@tcp1 routed destination 00000400:00000200:22.0:1616513088.553597:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.200@tcp1(10.31.128.200@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.553604:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.553609:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.553613:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.553615:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.553619:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.553623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.553633:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.553638:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.553640:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513088.553642:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.553647:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.553651:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a990 00000800:00000200:18.0:1616513088.553655:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513088.570497:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927312, portal 4 00000100:00000200:10.0:1616513088.570502:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927312, offset 0 00000400:00000200:10.0:1616513088.570507:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:10.0:1616513088.570513:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:10.0:1616513088.570523:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513088.570528:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:10.0:1616513088.570534:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513088.570536:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:10.0:1616513088.570538:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:10.0:1616513088.570542:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513088.570590:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.570609:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513088.570616:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513088.570619:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513088.570621:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513088.570627:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513088.570635:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf84380 x1693067725927312/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513088.570648:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e833a8 00000800:00000200:1.0:1616513088.570655:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513088.570763:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513088.570773:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.570780:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513088.570781:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513088.570789:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513088.570798:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627390 00000400:00000200:1.0:1616513088.570805:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffe95 [1] + 192 00000400:00000200:1.0:1616513088.570811:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513088.570816:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf84380 x1693067725927312/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513088.570831:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.570834:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513088.570835:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:10.0:1616513088.570836:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf84380 x1693067725927312/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513088.570847:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf84380 x1693067725927312/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513088.570855:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83540 00000800:00000200:11.2:1616513088.571045:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513088.571107:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.571116:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513088.571126:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.97@tcp1 : PUT - for me 00000400:00000200:4.0:1616513088.571135:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.97@tcp1 of length 224 into portal 26 MB=0x5ffeffdd14200 00000400:00000200:4.0:1616513088.571142:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.97@tcp1 of length 224/224 into md 0xf97cc9a9 [1] + 0 00000400:00000200:4.0:1616513088.571147:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70440 00000400:00000200:4.0:1616513088.571149:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513088.571153:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513088.571158:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70440 00000800:00000200:20.0:1616513088.571160:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.571161:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.571164:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513088.571168:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513088.571171:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513088.571171:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.571182:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.97@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.571190:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.97@tcp1 of length 224 into portal 12 MB=0x5ffeffdd14280 00000400:00000200:20.0:1616513088.571196:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.97@tcp1 of length 224/224 into md 0x865be291 [1] + 3696 00000400:00000200:20.0:1616513088.571202:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.571204:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.571211:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.571214:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.571216:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.571220:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688781104169472 00010000:00000200:24.0:1616513088.571230:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688781104169472/t0(0) o400->f35416d8-3f91-0a88-7b2d-d595bab5cc7c@10.31.141.97@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.571243:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688781104169472, offset 192 00000400:00000200:24.0:1616513088.571249:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.97@tcp1 00000400:00000200:24.0:1616513088.571255:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.97@tcp1 routed destination 00000400:00000200:24.0:1616513088.571270:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.97@tcp1(10.31.141.97@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.571276:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:16.0:1616513088.571278:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688781104169600 00000800:00000200:24.0:1616513088.571282:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.571284:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.571286:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.571290:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:16.0:1616513088.571291:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad4380 x1688781104169600/t0(0) o400->0c7028df-bb00-32ee-2ac9-01adb3897854@10.31.141.97@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513088.571295:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.571297:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513088.571304:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.571310:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.571312:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513088.571314:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513088.571316:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688781104169600, offset 192 00000400:00000200:8.0:1616513088.571318:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513088.571322:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.97@tcp1 00000400:00000200:8.0:1616513088.571326:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4f68 00000400:00000200:16.0:1616513088.571329:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.97@tcp1 routed destination 00000800:00000200:8.0:1616513088.571335:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:16.0:1616513088.571344:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.97@tcp1(10.31.141.97@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.571351:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.571356:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.571359:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.571361:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.571366:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.571372:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.571425:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.571447:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.571450:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.571453:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513088.571460:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.571464:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789b18 00000800:00000200:2.0:1616513088.571468:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.571504:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513088.571513:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.571518:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513088.571523:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.571526:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.11@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.571532:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.11@tcp1 of length 224 into portal 26 MB=0x603e25714a180 00000400:00000200:20.0:1616513088.571538:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.11@tcp1 of length 224/224 into md 0xf97cc9b9 [1] + 0 00000400:00000200:20.0:1616513088.571542:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f706e8 00000400:00000200:20.0:1616513088.571544:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.571547:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513088.571551:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f706e8 00000800:00000200:20.0:1616513088.571553:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.571555:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.571557:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.571562:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693120518726016 00010000:00000200:24.0:1616513088.571567:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1693120518726016/t0(0) o400->2e60a351-eabd-2e2b-0ac5-31b15ef779d2@10.31.130.11@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.571578:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693120518726016, offset 192 00000400:00000200:24.0:1616513088.571583:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.11@tcp1 00000400:00000200:24.0:1616513088.571588:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.11@tcp1 routed destination 00000400:00000200:24.0:1616513088.571600:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.11@tcp1(10.31.130.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.571605:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.571610:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.571612:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.571614:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.571617:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.571621:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.571621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513088.571630:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.571634:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513088.571636:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513088.571637:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513088.571641:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.571646:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5a90 00000800:00000200:4.0:1616513088.571649:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.571850:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.571861:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.571868:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513088.571870:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.571877:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.11@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.571886:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.11@tcp1 of length 224 into portal 12 MB=0x603e25714a400 00000400:00000200:28.0:1616513088.571892:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.11@tcp1 of length 224/224 into md 0x865be291 [1] + 3920 00000400:00000200:28.0:1616513088.571898:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.571900:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513088.571907:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.571910:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.571912:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.571919:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693120518726656 00010000:00000200:16.0:1616513088.571925:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa4380 x1693120518726656/t0(0) o400->35faee61-b666-a136-8691-3caf5debef83@10.31.130.11@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.571936:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693120518726656, offset 192 00000400:00000200:16.0:1616513088.571941:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.11@tcp1 00000400:00000200:16.0:1616513088.571947:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.11@tcp1 routed destination 00000400:00000200:16.0:1616513088.571960:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.11@tcp1(10.31.130.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.571965:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.571970:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.571972:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.571974:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.571977:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.571981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.572032:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.572039:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.572041:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.572048:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.572052:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788b28 00000800:00000200:26.0:1616513088.572056:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.572059:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.580621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.580674:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.580683:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.580691:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.126@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.580699:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.126@tcp1 of length 224 into portal 26 MB=0x5ffdeb985ebc0 00000400:00000200:14.0:1616513088.580705:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.126@tcp1 of length 224/224 into md 0xf97cc9d9 [1] + 0 00000400:00000200:14.0:1616513088.580710:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71320 00000400:00000200:14.0:1616513088.580712:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.580715:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.580718:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71320 00000800:00000200:14.0:1616513088.580721:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.580724:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.580725:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.580728:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.580738:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513088.580747:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.126@tcp1 : PUT - for me 00000400:00000200:2.0:1616513088.580755:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.126@tcp1 of length 224 into portal 12 MB=0x5ffdeb985ec40 00000400:00000200:2.0:1616513088.580762:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.126@tcp1 of length 224/224 into md 0xf964d519 [1] + 72776 00000400:00000200:2.0:1616513088.580767:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.580770:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513088.580777:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:24.0:1616513088.580779:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706943937472 00000800:00000200:2.0:1616513088.580780:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.580782:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.580785:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513088.580788:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688706943937472/t0(0) o400->892c72b9-0fc6-e1a4-0159-060e57735941@10.31.142.126@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.580801:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706943937472, offset 192 00000400:00000200:24.0:1616513088.580806:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.126@tcp1 00000400:00000200:24.0:1616513088.580812:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.126@tcp1 routed destination 00000400:00000200:24.0:1616513088.580825:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.126@tcp1(10.31.142.126@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.580830:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.580835:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.580839:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513088.580839:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706943937600 00000800:00000200:24.0:1616513088.580841:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.580845:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.580851:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:22.0:1616513088.580851:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cb180 x1688706943937600/t0(0) o400->d64c94b6-7441-2322-0334-cca7b9de8eff@10.31.142.126@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.580864:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706943937600, offset 192 00000400:00000200:22.0:1616513088.580870:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.126@tcp1 00000400:00000200:22.0:1616513088.580876:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.126@tcp1 routed destination 00000400:00000200:22.0:1616513088.580890:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.126@tcp1(10.31.142.126@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513088.580895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513088.580896:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.580901:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.580903:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.580905:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513088.580906:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.580909:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513088.580910:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513088.580911:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513088.580913:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513088.580916:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.580920:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5cb0 00000800:00000200:8.0:1616513088.580922:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.580924:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513088.580969:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.580976:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.580979:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.580980:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.580986:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.580989:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b188 00000800:00000200:18.0:1616513088.580994:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.618802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513088.618830:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513088.618869:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.618877:0:6484:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927328, portal 4 00000800:00000200:20.0:1616513088.618879:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:24.0:1616513088.618881:0:6484:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927328, offset 0 00000400:00000200:24.0:1616513088.618887:0:6484:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:20.0:1616513088.618888:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.187@tcp1 : PUT - for me 00000800:00000200:26.0:1616513088.618891:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513088.618892:0:6484:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:26.0:1616513088.618901:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513088.618901:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.187@tcp1 of length 224 into portal 12 MB=0x5ffeaf6e7d2c0 00000400:00000200:24.0:1616513088.618903:0:6484:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513088.618908:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.187@tcp1 of length 224/224 into md 0x865be291 [1] + 4144 00000800:00000200:24.0:1616513088.618909:0:6484:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:26.0:1616513088.618912:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.187@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.618913:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513088.618914:0:6484:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000100:00000200:20.0:1616513088.618916:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513088.618917:0:6484:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:24.0:1616513088.618919:0:6484:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000400:00000200:26.0:1616513088.618922:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.187@tcp1 of length 224 into portal 26 MB=0x5ffeaf6e7d240 00000800:00000200:20.0:1616513088.618922:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.618923:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513088.618924:0:6484:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:20.0:1616513088.618925:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513088.618927:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513088.618928:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.187@tcp1 of length 224/224 into md 0xf97cc9e9 [1] + 0 00000400:00000200:26.0:1616513088.618934:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71298 00000400:00000200:26.0:1616513088.618937:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.618940:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513088.618944:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71298 00000800:00000200:26.0:1616513088.618947:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.618950:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.618952:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.618954:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.618963:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688759513371200 00010000:00000200:24.0:1616513088.618971:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688759513371200/t0(0) o400->a6d00dda-911e-e683-a986-47951c3ad89b@10.31.141.187@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:25.2:1616513088.618974:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000100:00000200:24.0:1616513088.618982:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688759513371200, offset 192 00000400:00000200:24.0:1616513088.618987:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.187@tcp1 00000100:00000200:16.0:1616513088.618991:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688759513371328 00000400:00000200:24.0:1616513088.618992:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.187@tcp1 routed destination 00000400:00000200:24.0:1616513088.619004:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.187@tcp1(10.31.141.187@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:16.0:1616513088.619004:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad7500 x1688759513371328/t0(0) o400->4dddf652-99da-82dc-bf2e-05e6d710cbdc@10.31.141.187@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513088.619009:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.619013:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.619015:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.619017:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513088.619019:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688759513371328, offset 192 00000800:00000200:24.0:1616513088.619020:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.619024:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513088.619025:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.187@tcp1 00000100:00000200:24.0:1616513088.619026:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513088.619032:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.187@tcp1 routed destination 00000800:00000200:14.0:1616513088.619032:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:8.0:1616513088.619033:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513088.619038:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000800:00000200:8.0:1616513088.619038:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.619040:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.619041:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513088.619044:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.619046:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.619047:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:16.0:1616513088.619048:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.187@tcp1(10.31.141.187@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513088.619050:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4c38 00000800:00000200:8.0:1616513088.619053:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513088.619055:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cde86a880 x1693067725927328/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513088.619056:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.619062:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.619065:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.619067:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:14.0:1616513088.619067:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4990 00000800:00000200:14.0:1616513088.619071:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:16.0:1616513088.619072:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.619073:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:7.2:1616513088.619076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.619087:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.619095:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.619098:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513088.619100:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.619104:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.619108:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789980 00000800:00000200:18.0:1616513088.619117:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:25.2:1616513088.619148:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513088.619158:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513088.619164:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:26.0:1616513088.619166:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513088.619171:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513088.619178:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6273a0 00000400:00000200:14.0:1616513088.619184:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97ccff9 [1] + 192 00000400:00000200:14.0:1616513088.619188:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.619192:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cde86a880 x1693067725927328/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513088.619205:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513088.619207:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513088.619211:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:24.0:1616513088.619212:0:6484:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cde86a880 x1693067725927328/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:24.0:1616513088.619222:0:6484:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cde86a880 x1693067725927328/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513088.619239:0:6484:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4dd0 00000100:00000200:4.0:1616513088.634619:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927344, portal 4 00000100:00000200:4.0:1616513088.634624:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927344, offset 0 00000400:00000200:4.0:1616513088.634631:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:4.0:1616513088.634637:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:4.0:1616513088.634650:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513088.634657:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:4.0:1616513088.634663:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513088.634666:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:4.0:1616513088.634668:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:4.0:1616513088.634674:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513088.634684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513088.634700:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513088.634706:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513088.634709:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513088.634716:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513088.634723:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5772fa3600 x1693067725927344/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513088.634735:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb188 00000800:00000200:27.0:1616513088.634739:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513088.634742:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513088.634903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513088.634913:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513088.634919:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513088.634923:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513088.634927:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513088.634935:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6273b0 00000400:00000200:27.0:1616513088.634942:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166edd [1] + 192 00000400:00000200:27.0:1616513088.634947:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513088.634951:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5772fa3600 x1693067725927344/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513088.634967:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513088.634970:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513088.634972:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:4.0:1616513088.634977:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5772fa3600 x1693067725927344/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513088.634990:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5772fa3600 x1693067725927344/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513088.635001:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba5d8 00000100:00000200:25.0:1616513088.650645:0:6489:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927360, portal 4 00000100:00000200:25.0:1616513088.650651:0:6489:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927360, offset 0 00000400:00000200:25.0:1616513088.650657:0:6489:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:25.0:1616513088.650664:0:6489:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:25.0:1616513088.650677:0:6489:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:25.0:1616513088.650684:0:6489:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:25.0:1616513088.650691:0:6489:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:25.0:1616513088.650693:0:6489:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:25.0:1616513088.650696:0:6489:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:25.0:1616513088.650701:0:6489:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513088.650750:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.650770:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513088.650776:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513088.650779:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513088.650781:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513088.650786:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513088.650793:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843f7ba880 x1693067725927360/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513088.650805:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492673100 00000800:00000200:1.0:1616513088.650809:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513088.650918:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513088.650927:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.650933:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513088.650934:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513088.650941:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513088.650950:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6273c0 00000400:00000200:1.0:1616513088.650960:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffea5 [1] + 192 00000400:00000200:1.0:1616513088.650965:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513088.650970:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843f7ba880 x1693067725927360/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513088.650984:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.650987:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513088.650989:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:25.0:1616513088.650991:0:6489:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843f7ba880 x1693067725927360/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:25.0:1616513088.651004:0:6489:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843f7ba880 x1693067725927360/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:25.0:1616513088.651014:0:6489:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492673650 00000100:00000200:23.0:1616513088.666507:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927376, portal 4 00000100:00000200:23.0:1616513088.666514:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927376, offset 0 00000400:00000200:23.0:1616513088.666521:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:23.0:1616513088.666526:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:23.0:1616513088.666539:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513088.666546:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:23.0:1616513088.666551:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513088.666554:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:23.0:1616513088.666557:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:23.0:1616513088.666561:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513088.666607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.666625:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513088.666630:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513088.666632:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513088.666639:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513088.666646:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8f30a8ad00 x1693067725927376/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513088.666658:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fdc0 00000800:00000200:1.0:1616513088.666662:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513088.666665:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513088.666780:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513088.666788:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.666794:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513088.666797:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513088.666802:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513088.666809:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6273d0 00000400:00000200:1.0:1616513088.666816:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffec5 [1] + 192 00000400:00000200:1.0:1616513088.666821:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513088.666825:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8f30a8ad00 x1693067725927376/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513088.666838:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.666841:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513088.666843:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:23.0:1616513088.666847:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8f30a8ad00 x1693067725927376/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513088.666870:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8f30a8ad00 x1693067725927376/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513088.666879:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f4b8 00000800:00000200:11.2:1616513088.670872:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513088.670889:0:50397:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513088.670933:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.670943:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513088.670951:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513088.670953:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.13@tcp1 : PUT - for me 00000800:00000200:2.0:1616513088.670961:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513088.670962:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.13@tcp1 of length 224 into portal 12 MB=0x603e3323b7c80 00000400:00000200:28.0:1616513088.670969:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.13@tcp1 of length 224/224 into md 0x865be291 [1] + 4368 00000400:00000200:2.0:1616513088.670970:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.13@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.670974:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.670977:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:2.0:1616513088.670980:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.13@tcp1 of length 224 into portal 26 MB=0x603e3323b7b00 00000800:00000200:28.0:1616513088.670985:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.670987:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513088.670987:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.13@tcp1 of length 224/224 into md 0xf97cca09 [1] + 0 00000800:00000200:28.0:1616513088.670988:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513088.670992:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f716d8 00000800:00000200:28.0:1616513088.670994:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513088.670995:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.670998:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513088.671004:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f716d8 00000800:00000200:18.0:1616513088.671007:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.671008:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.671012:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.671014:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513088.671054:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693124195482752 00010000:00000200:16.0:1616513088.671067:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa7980 x1693124195482752/t0(0) o400->26230f90-974c-c969-b394-54ceebe4cb44@10.31.130.13@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513088.671075:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693124195482368 00000100:00000200:16.0:1616513088.671083:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693124195482752, offset 192 00010000:00000200:24.0:1616513088.671087:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1693124195482368/t0(0) o400->e3621267-4fc2-049d-34e7-c4fef866c150@10.31.130.13@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513088.671089:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.13@tcp1 00000400:00000200:16.0:1616513088.671097:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.13@tcp1 routed destination 00000100:00000200:24.0:1616513088.671102:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693124195482368, offset 192 00000400:00000200:24.0:1616513088.671109:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.13@tcp1 00000400:00000200:16.0:1616513088.671112:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.13@tcp1(10.31.130.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.671119:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513088.671121:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.13@tcp1 routed destination 00000800:00000200:16.0:1616513088.671129:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.671132:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.671134:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513088.671136:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.13@tcp1(10.31.130.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.671139:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513088.671143:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513088.671145:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.671149:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.671152:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.671154:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.671159:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.671163:0:50397:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.671166:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513088.671199:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513088.671206:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513088.671209:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513088.671211:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513088.671226:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513088.671226:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.671231:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788f68 00000800:00000200:26.0:1616513088.671233:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:4.0:1616513088.671235:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513088.671236:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.671243:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.671248:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4dd0 00000800:00000200:26.0:1616513088.671252:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.671255:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.673123:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.673140:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.673149:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.673158:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.27@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.673167:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.27@tcp1 of length 224 into portal 26 MB=0x5fff523527480 00000400:00000200:18.0:1616513088.673174:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.27@tcp1 of length 224/224 into md 0xf97cca19 [1] + 0 00000400:00000200:18.0:1616513088.673178:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f703b8 00000400:00000200:18.0:1616513088.673181:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.673183:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.673187:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f703b8 00000800:00000200:18.0:1616513088.673190:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513088.673190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513088.673193:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.673193:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.673197:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.673250:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688803208230016 00000800:00000200:20.0:1616513088.673257:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513088.673261:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688803208230016/t0(0) o400->5a5b6a5d-c71b-921e-2868-b9a562710848@10.31.142.27@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513088.673266:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513088.673272:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.673275:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688803208230016, offset 192 00000400:00000200:20.0:1616513088.673275:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.27@tcp1 : PUT - for me 00000400:00000200:24.0:1616513088.673282:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.27@tcp1 00000400:00000200:20.0:1616513088.673284:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.27@tcp1 of length 224 into portal 12 MB=0x5fff523527540 00000400:00000200:24.0:1616513088.673289:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.27@tcp1 routed destination 00000400:00000200:20.0:1616513088.673290:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.27@tcp1 of length 224/224 into md 0x865be291 [1] + 4592 00000400:00000200:20.0:1616513088.673296:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.673298:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513088.673304:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.27@tcp1(10.31.142.27@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513088.673309:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.673310:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513088.673312:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.673314:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513088.673316:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.673319:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.673321:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.673325:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.673331:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.673332:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513088.673339:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.673344:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.673346:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.673352:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:4.0:1616513088.673352:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.673356:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4dd0 00000800:00000200:8.0:1616513088.673360:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.673369:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688803208230208 00010000:00000200:16.0:1616513088.673381:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad1b00 x1688803208230208/t0(0) o400->6b4b09a8-1066-709e-005f-6c9824207199@10.31.142.27@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.673396:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688803208230208, offset 192 00000400:00000200:16.0:1616513088.673402:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.27@tcp1 00000400:00000200:16.0:1616513088.673410:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.27@tcp1 routed destination 00000400:00000200:16.0:1616513088.673425:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.27@tcp1(10.31.142.27@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.673431:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.673450:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.673453:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.673455:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.673460:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.673468:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.673520:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.673526:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.673529:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.673536:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.673540:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788bb0 00000800:00000200:26.0:1616513088.673543:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.673546:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.675249:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.675304:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.675313:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513088.675322:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.203@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.675331:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.203@tcp1 of length 224 into portal 26 MB=0x6000e9fff3c00 00000400:00000200:28.0:1616513088.675337:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.203@tcp1 of length 224/224 into md 0xf97cca29 [1] + 0 00000400:00000200:28.0:1616513088.675342:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71d38 00000400:00000200:28.0:1616513088.675344:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.675347:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513088.675351:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71d38 00000800:00000200:28.0:1616513088.675355:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.675356:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.675357:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.675361:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.675415:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688912674110464 00010000:00000200:24.0:1616513088.675425:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688912674110464/t0(0) o400->248920f1-3349-1b29-d6c6-2bb2172a3b5c@10.31.141.203@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.675453:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688912674110464, offset 192 00000400:00000200:24.0:1616513088.675459:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.203@tcp1 00000800:00000200:7.2:1616513088.675465:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513088.675467:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.203@tcp1 routed destination 00000400:00000200:24.0:1616513088.675482:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.203@tcp1(10.31.141.203@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.675488:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.675494:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.675497:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.675499:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.675504:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.675508:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.675511:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513088.675517:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.675526:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513088.675535:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.203@tcp1 : PUT - for me 00000400:00000200:2.0:1616513088.675543:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.203@tcp1 of length 224 into portal 12 MB=0x6000e9fff3c80 00000400:00000200:2.0:1616513088.675550:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.203@tcp1 of length 224/224 into md 0xf964d519 [1] + 73000 00000400:00000200:2.0:1616513088.675555:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.675558:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513088.675562:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513088.675565:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.675568:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:4.0:1616513088.675569:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513088.675570:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513088.675571:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513088.675572:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513088.675573:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513088.675578:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513088.675584:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4c38 00000800:00000200:4.0:1616513088.675587:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513088.675631:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688912674110592 00010000:00000200:22.0:1616513088.675643:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cf080 x1688912674110592/t0(0) o400->d92923a7-9186-d868-ac8f-5c4ee046beb1@10.31.141.203@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.675657:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688912674110592, offset 192 00000400:00000200:22.0:1616513088.675664:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.203@tcp1 00000400:00000200:22.0:1616513088.675671:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.203@tcp1 routed destination 00000400:00000200:22.0:1616513088.675686:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.203@tcp1(10.31.141.203@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.675692:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.675698:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.675701:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.675703:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.675708:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.675712:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.675765:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.675772:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.675775:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.675778:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513088.675782:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.675786:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a330 00000800:00000200:26.0:1616513088.675790:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.688988:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513088.689043:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.689053:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513088.689062:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.180@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.689071:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.180@tcp1 of length 224 into portal 12 MB=0x5ffe384e84d80 00000400:00000200:20.0:1616513088.689077:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.180@tcp1 of length 224/224 into md 0x865be291 [1] + 4816 00000400:00000200:20.0:1616513088.689082:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.689085:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.689091:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.689094:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513088.689096:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.689098:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.689144:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513088.689155:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727536029056 00010000:00000200:16.0:1616513088.689168:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0000 x1688727536029056/t0(0) o400->c2f682e7-0ee0-c6d1-5648-411bd6d47738@10.31.141.180@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.689182:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688727536029056, offset 192 00000400:00000200:16.0:1616513088.689188:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.180@tcp1 00000400:00000200:16.0:1616513088.689196:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.180@tcp1 routed destination 00000800:00000200:2.0:1616513088.689201:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513088.689210:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.180@tcp1(10.31.141.180@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513088.689210:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.689217:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513088.689219:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.180@tcp1 : PUT - for me 00000800:00000200:16.0:1616513088.689223:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.689226:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.689228:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513088.689228:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.180@tcp1 of length 224 into portal 26 MB=0x5ffe384e84cc0 00000800:00000200:16.0:1616513088.689233:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513088.689234:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.180@tcp1 of length 224/224 into md 0xf97cca49 [1] + 0 00000800:00000200:11.2:1616513088.689238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513088.689239:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70aa0 00000400:00000200:2.0:1616513088.689242:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.689245:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:8.0:1616513088.689247:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513088.689249:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70aa0 00000800:00000200:2.0:1616513088.689252:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513088.689253:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513088.689254:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513088.689255:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513088.689255:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.689258:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513088.689259:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.689260:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.689263:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788198 00000800:00000200:8.0:1616513088.689267:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.689315:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727536028864 00010000:00000200:24.0:1616513088.689326:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688727536028864/t0(0) o400->343d6e8c-6800-f240-22cd-834afcac8ffc@10.31.141.180@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.689341:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688727536028864, offset 192 00000400:00000200:24.0:1616513088.689347:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.180@tcp1 00000400:00000200:24.0:1616513088.689354:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.180@tcp1 routed destination 00000400:00000200:24.0:1616513088.689369:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.180@tcp1(10.31.141.180@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.689376:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.689382:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.689384:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.689387:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.689391:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.689395:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.689398:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513088.689449:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.689456:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.689459:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.689466:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.689471:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4c38 00000800:00000200:14.0:1616513088.689475:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.689478:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.691066:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513088.691120:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.691139:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513088.691147:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.183@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.691156:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.183@tcp1 of length 224 into portal 12 MB=0x5ffd4624d2800 00000400:00000200:20.0:1616513088.691162:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.183@tcp1 of length 224/224 into md 0x865be291 [1] + 5040 00000400:00000200:20.0:1616513088.691167:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.691170:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.691175:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.691178:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.691178:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513088.691182:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.691238:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662530926592 00010000:00000200:16.0:1616513088.691249:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad6c00 x1688662530926592/t0(0) o400->6fdbb5c2-15a9-b85b-f864-1f8bba0bb9dd@10.31.132.183@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.691263:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662530926592, offset 192 00000400:00000200:16.0:1616513088.691268:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.183@tcp1 00000400:00000200:16.0:1616513088.691275:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.183@tcp1 routed destination 00000400:00000200:16.0:1616513088.691289:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.183@tcp1(10.31.132.183@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.691295:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513088.691301:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.691303:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513088.691305:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513088.691305:0:50399:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.691310:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.691314:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.691326:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513088.691332:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513088.691335:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513088.691335:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513088.691342:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513088.691345:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7898f8 00000800:00000200:12.0:1616513088.691350:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513088.691376:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.691385:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.691393:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.183@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.691401:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.183@tcp1 of length 224 into portal 26 MB=0x5ffd4624d2780 00000400:00000200:18.0:1616513088.691407:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.183@tcp1 of length 224/224 into md 0xf97cca59 [1] + 0 00000400:00000200:18.0:1616513088.691412:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71870 00000400:00000200:18.0:1616513088.691415:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.691417:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.691421:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71870 00000800:00000200:18.0:1616513088.691423:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.691426:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.691428:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.691430:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.691482:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662530926464 00010000:00000200:24.0:1616513088.691492:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688662530926464/t0(0) o400->a0e90a79-5ec5-8c34-35c3-5dcf069cbd68@10.31.132.183@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.691505:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662530926464, offset 192 00000400:00000200:24.0:1616513088.691511:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.183@tcp1 00000400:00000200:24.0:1616513088.691518:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.183@tcp1 routed destination 00000400:00000200:24.0:1616513088.691534:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.183@tcp1(10.31.132.183@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.691540:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.691545:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.691548:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.691550:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.691554:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.691558:0:50399:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.691560:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513088.691611:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.691618:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.691620:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.691627:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.691633:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfb650 00000800:00000200:14.0:1616513088.691636:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.691639:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.696308:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.696363:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.696372:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513088.696380:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.150@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.696389:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.150@tcp1 of length 224 into portal 26 MB=0x5ffe566afbec0 00000400:00000200:28.0:1616513088.696395:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.150@tcp1 of length 224/224 into md 0xf97cca69 [1] + 0 00000400:00000200:28.0:1616513088.696399:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f715c8 00000400:00000200:28.0:1616513088.696402:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.696404:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513088.696408:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f715c8 00000800:00000200:28.0:1616513088.696411:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.696413:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.696415:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.696418:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.696430:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513088.696430:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.696450:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.150@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.696459:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.150@tcp1 of length 224 into portal 12 MB=0x5ffe566afbfc0 00000400:00000200:20.0:1616513088.696465:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.150@tcp1 of length 224/224 into md 0x865be291 [1] + 5264 00000100:00000200:24.0:1616513088.696469:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688735618940608 00000400:00000200:20.0:1616513088.696470:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.696472:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.696478:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513088.696479:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688735618940608/t0(0) o400->2120c017-059f-64f3-cc90-e22a35bfc426@10.31.141.150@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513088.696481:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.696483:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.696492:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688735618940608, offset 192 00000400:00000200:24.0:1616513088.696498:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.150@tcp1 00000400:00000200:24.0:1616513088.696504:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.150@tcp1 routed destination 00000400:00000200:24.0:1616513088.696518:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.150@tcp1(10.31.141.150@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.696524:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.696530:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.696532:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.696534:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.696538:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.696543:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.696545:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:16.0:1616513088.696546:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688735618940864 00000800:00000200:12.0:1616513088.696556:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:16.0:1616513088.696558:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad3f00 x1688735618940864/t0(0) o400->b487aec5-f34f-f7d1-df21-d72bfcc01c1c@10.31.141.150@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513088.696563:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.696567:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513088.696568:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:16.0:1616513088.696572:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688735618940864, offset 192 00000400:00000200:12.0:1616513088.696575:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513088.696577:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.150@tcp1 00000400:00000200:12.0:1616513088.696582:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfbdc0 00000400:00000200:16.0:1616513088.696584:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.150@tcp1 routed destination 00000800:00000200:12.0:1616513088.696591:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:16.0:1616513088.696600:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.150@tcp1(10.31.141.150@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.696606:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.696612:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.696614:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.696616:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.696621:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.696625:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.696676:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.696682:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.696684:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513088.696691:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.696695:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789ba0 00000800:00000200:18.0:1616513088.696698:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.696703:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513088.698529:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927392, portal 4 00000100:00000200:4.0:1616513088.698534:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927392, offset 0 00000400:00000200:4.0:1616513088.698541:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:4.0:1616513088.698546:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:4.0:1616513088.698558:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513088.698564:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000100:00000200:24.0:1616513088.698568:0:6487:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927408, portal 4 00000800:00000200:4.0:1616513088.698570:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000100:00000200:24.0:1616513088.698573:0:6487:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927408, offset 0 00000800:00000200:4.0:1616513088.698573:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:4.0:1616513088.698576:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:24.0:1616513088.698579:0:6487:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:4.0:1616513088.698581:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:24.0:1616513088.698585:0:6487:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:24.0:1616513088.698597:0:6487:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.698603:0:6487:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:24.0:1616513088.698608:0:6487:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.698611:0:6487:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:24.0:1616513088.698613:0:6487:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:24.0:1616513088.698620:0:6487:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.2:1616513088.698628:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513088.698648:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:27.0:1616513088.698654:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:27.0:1616513088.698657:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513088.698663:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513088.698670:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5772fa0480 x1693067725927392/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513088.698682:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba7f8 00000800:00000200:27.0:1616513088.698686:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513088.698689:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513088.698694:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513088.698695:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513088.698700:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513088.698701:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513088.698706:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affd4800 x1693067725927408/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513088.698715:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfa198 00000800:00000200:27.0:1616513088.698717:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513088.698770:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513088.698779:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513088.698785:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513088.698788:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513088.698792:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:31.0:1616513088.698799:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513088.698800:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6273e0 00000800:00000200:3.0:1616513088.698801:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:31.0:1616513088.698807:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513088.698807:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166efd [1] + 192 00000400:00000200:27.0:1616513088.698812:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:31.0:1616513088.698815:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6273f0 00000100:00000200:27.0:1616513088.698816:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5772fa0480 x1693067725927392/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513088.698830:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:31.0:1616513088.698832:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166f0d [1] + 192 00000800:00000200:27.0:1616513088.698833:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513088.698835:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:31.0:1616513088.698838:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513088.698840:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5772fa0480 x1693067725927392/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:4.0:1616513088.698851:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5772fa0480 x1693067725927392/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513088.698861:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba5d8 00000100:00000200:31.0:1616513088.698866:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affd4800 x1693067725927408/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513088.698882:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513088.698885:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:31.0:1616513088.698887:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:24.0:1616513088.698891:0:6487:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affd4800 x1693067725927408/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:24.0:1616513088.698903:0:6487:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affd4800 x1693067725927408/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513088.698912:0:6487:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfb320 00000100:00000200:6.0:1616513088.714551:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927424, portal 4 00000100:00000200:6.0:1616513088.714557:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927424, offset 0 00000400:00000200:6.0:1616513088.714564:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513088.714569:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:6.0:1616513088.714582:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513088.714589:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:6.0:1616513088.714595:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513088.714598:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:6.0:1616513088.714600:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:6.0:1616513088.714607:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513088.714653:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513088.714712:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513088.714720:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513088.714731:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.714737:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:26.0:1616513088.714738:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000100:00000200:14.0:1616513088.714744:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d747c697080 x1693067725927424/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513088.714756:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01320 00000800:00000200:14.0:1616513088.714760:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513088.714828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513088.714881:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513088.714890:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:2.0:1616513088.714893:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513088.714899:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513088.714906:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627400 00000400:00000200:18.0:1616513088.714912:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cd089 [1] + 192 00000400:00000200:18.0:1616513088.714917:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.714922:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d747c697080 x1693067725927424/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513088.714936:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513088.714939:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:6.0:1616513088.714940:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d747c697080 x1693067725927424/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513088.714941:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:6.0:1616513088.714949:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d747c697080 x1693067725927424/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513088.714956:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:11.2:1616513088.716284:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.716297:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.716305:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.716314:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.242@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.716324:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.242@tcp1 of length 224 into portal 26 MB=0x60183654add40 00000400:00000200:8.0:1616513088.716331:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.242@tcp1 of length 224/224 into md 0xf97cca79 [1] + 0 00000400:00000200:8.0:1616513088.716336:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f707f8 00000400:00000200:8.0:1616513088.716339:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.716341:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513088.716346:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f707f8 00000800:00000200:8.0:1616513088.716349:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.716350:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.716351:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.716355:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.716411:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513712012608 00010000:00000200:24.0:1616513088.716422:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1690513712012608/t0(0) o400->a9c86a7c-a722-5cdb-cc01-a783147a71eb@10.31.131.242@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.716449:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513712012608, offset 192 00000400:00000200:24.0:1616513088.716455:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.242@tcp1 00000400:00000200:24.0:1616513088.716463:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.242@tcp1 routed destination 00000400:00000200:24.0:1616513088.716479:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.242@tcp1(10.31.131.242@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.716485:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.716491:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.716494:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.716496:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.716501:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.716505:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.716508:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513088.716558:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.716565:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.716568:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513088.716571:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.716575:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.716580:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfb320 00000800:00000200:28.0:1616513088.716584:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.717033:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513088.717045:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.717053:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513088.717053:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.717062:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.242@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.717071:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.242@tcp1 of length 224 into portal 12 MB=0x60183654adf40 00000400:00000200:20.0:1616513088.717077:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.242@tcp1 of length 224/224 into md 0x865be291 [1] + 5488 00000400:00000200:20.0:1616513088.717083:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.717085:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.717093:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.717095:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.717097:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.717159:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513712013120 00010000:00000200:16.0:1616513088.717172:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad7080 x1690513712013120/t0(0) o400->6a8ad5df-42b2-0246-0e8e-71e8134d22cc@10.31.131.242@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.717186:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513712013120, offset 192 00000400:00000200:16.0:1616513088.717193:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.242@tcp1 00000400:00000200:16.0:1616513088.717200:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.242@tcp1 routed destination 00000400:00000200:16.0:1616513088.717215:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.242@tcp1(10.31.131.242@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.717222:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.717228:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.717230:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.717233:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.717237:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.717243:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.717308:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.717314:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.717316:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.717322:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.717326:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789210 00000800:00000200:26.0:1616513088.717331:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.717334:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.722447:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.722501:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.722511:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513088.722519:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.68@tcp1 : PUT - for me 00000400:00000200:12.0:1616513088.722528:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.68@tcp1 of length 224 into portal 26 MB=0x5ffd492ff0740 00000400:00000200:12.0:1616513088.722535:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.68@tcp1 of length 224/224 into md 0xf97cca89 [1] + 0 00000400:00000200:12.0:1616513088.722540:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71650 00000400:00000200:12.0:1616513088.722542:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513088.722545:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513088.722549:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71650 00000800:00000200:12.0:1616513088.722552:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.722554:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.722555:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513088.722558:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.722611:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663347889984 00010000:00000200:24.0:1616513088.722621:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688663347889984/t0(0) o400->9af65f94-715c-0963-614c-0b000a173cc8@10.31.138.68@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.722634:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663347889984, offset 192 00000400:00000200:24.0:1616513088.722640:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.68@tcp1 00000400:00000200:24.0:1616513088.722646:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.68@tcp1 routed destination 00000400:00000200:24.0:1616513088.722660:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.68@tcp1(10.31.138.68@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.722666:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.722672:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.722674:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.722676:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.722682:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.722688:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.722688:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.722697:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.722701:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.722702:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.722707:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.722711:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfb5c8 00000800:00000200:8.0:1616513088.722713:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.722715:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.722727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.722738:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.722746:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513088.722749:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.722754:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.68@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.722762:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.68@tcp1 of length 224 into portal 12 MB=0x5ffd492ff0880 00000400:00000200:28.0:1616513088.722768:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.68@tcp1 of length 224/224 into md 0x865be291 [1] + 5712 00000400:00000200:28.0:1616513088.722773:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.722775:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513088.722781:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.722784:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.722786:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.722842:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663347890304 00010000:00000200:16.0:1616513088.722854:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa6780 x1688663347890304/t0(0) o400->641e2c40-b723-6aeb-59aa-c4846a5074d0@10.31.138.68@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.722868:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663347890304, offset 192 00000400:00000200:16.0:1616513088.722873:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.68@tcp1 00000400:00000200:16.0:1616513088.722880:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.68@tcp1 routed destination 00000400:00000200:16.0:1616513088.722894:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.68@tcp1(10.31.138.68@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.722900:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.722905:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.722908:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.722912:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.722916:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.722924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.722982:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.722989:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.722992:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513088.722998:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.723003:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789760 00000800:00000200:2.0:1616513088.723009:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.723012:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.723017:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513088.723020:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.723022:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.723023:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.723026:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.751999:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.752010:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.752018:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.752027:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.228@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.752036:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.228@tcp1 of length 224 into portal 26 MB=0x600046313a540 00000400:00000200:8.0:1616513088.752043:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.228@tcp1 of length 224/224 into md 0xf97cca99 [1] + 0 00000400:00000200:8.0:1616513088.752048:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70dd0 00000400:00000200:8.0:1616513088.752051:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.752053:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513088.752058:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70dd0 00000800:00000200:8.0:1616513088.752061:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.752063:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513088.752064:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513088.752071:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513088.752074:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.752075:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513088.752084:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.228@tcp1 : PUT - for me 00000400:00000200:12.0:1616513088.752093:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.228@tcp1 of length 224 into portal 12 MB=0x600046313a5c0 00000400:00000200:12.0:1616513088.752099:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.228@tcp1 of length 224/224 into md 0x865be291 [1] + 5936 00000400:00000200:12.0:1616513088.752105:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513088.752107:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513088.752114:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.752117:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.752119:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.752123:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688868702364992 00010000:00000200:24.0:1616513088.752134:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688868702364992/t0(0) o400->9852e9c3-b2f5-1458-7745-abf60456526e@10.31.142.228@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.752148:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688868702364992, offset 192 00000400:00000200:24.0:1616513088.752154:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.228@tcp1 00000400:00000200:24.0:1616513088.752162:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.228@tcp1 routed destination 00000400:00000200:24.0:1616513088.752177:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.228@tcp1(10.31.142.228@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513088.752181:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688868702365120 00000800:00000200:24.0:1616513088.752184:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.752189:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.752192:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.752194:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:16.0:1616513088.752194:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d207980 x1688868702365120/t0(0) o400->53740457-bd07-1c9b-b5ae-41c0f05b685f@10.31.142.228@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513088.752199:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.752206:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:16.0:1616513088.752209:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688868702365120, offset 192 00000800:00000200:11.2:1616513088.752209:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513088.752215:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.228@tcp1 00000800:00000200:28.0:1616513088.752221:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513088.752223:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.228@tcp1 routed destination 00000800:00000200:28.0:1616513088.752229:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.752231:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513088.752231:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.752238:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513088.752239:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.228@tcp1(10.31.142.228@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513088.752244:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfa198 00000800:00000200:16.0:1616513088.752245:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513088.752247:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513088.752252:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.752267:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.752269:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.752273:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.752279:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.752333:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.752341:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.752343:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.752350:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.752354:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7886e8 00000800:00000200:14.0:1616513088.752358:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.752361:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.778552:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.778607:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.778617:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513088.778623:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.778626:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.21@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.778636:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.21@tcp1 of length 224 into portal 26 MB=0x601855eded880 00000400:00000200:18.0:1616513088.778643:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.21@tcp1 of length 224/224 into md 0xf97ccaa9 [1] + 0 00000400:00000200:18.0:1616513088.778647:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71f58 00000400:00000200:18.0:1616513088.778650:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.778653:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.778657:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71f58 00000800:00000200:18.0:1616513088.778660:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.778663:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.778665:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.778724:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690522194204800 00010000:00000200:24.0:1616513088.778735:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1690522194204800/t0(0) o400->a1b26c10-0bef-1683-bbf4-4172012743dc@10.31.141.21@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.778750:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690522194204800, offset 192 00000800:00000200:7.2:1616513088.778755:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513088.778756:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.21@tcp1 00000400:00000200:24.0:1616513088.778763:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.21@tcp1 routed destination 00000400:00000200:24.0:1616513088.778778:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.21@tcp1(10.31.141.21@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.778784:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.778790:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.778793:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.778795:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.778800:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513088.778806:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.778807:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513088.778815:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.778822:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.21@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.778830:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.21@tcp1 of length 224 into portal 12 MB=0x601855eded9c0 00000400:00000200:14.0:1616513088.778838:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.21@tcp1 of length 224/224 into md 0xf964d519 [1] + 73224 00000400:00000200:14.0:1616513088.778843:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:11.2:1616513088.778843:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:14.0:1616513088.778845:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513088.778853:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.778856:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.778858:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.778861:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513088.778900:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513088.778906:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513088.778909:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513088.778916:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513088.778921:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfb650 00000800:00000200:12.0:1616513088.778925:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.778930:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513088.778938:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690522194205120 00010000:00000200:22.0:1616513088.778951:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7d100 x1690522194205120/t0(0) o400->96be1653-5999-f4e0-c758-d2246ade5908@10.31.141.21@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.778976:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690522194205120, offset 192 00000400:00000200:22.0:1616513088.778983:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.21@tcp1 00000400:00000200:22.0:1616513088.778991:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.21@tcp1 routed destination 00000400:00000200:22.0:1616513088.779005:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.21@tcp1(10.31.141.21@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.779012:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.779018:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.779020:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.779023:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.779027:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.779031:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.779041:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.779046:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.779048:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513088.779050:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513088.779054:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.779058:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b540 00000800:00000200:26.0:1616513088.779062:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.789143:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513088.789158:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.789166:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.789175:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.162@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.789184:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.162@tcp1 of length 224 into portal 26 MB=0x603e0f37ca480 00000400:00000200:8.0:1616513088.789191:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.162@tcp1 of length 224/224 into md 0xf97ccab9 [1] + 0 00000400:00000200:8.0:1616513088.789195:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71ba0 00000400:00000200:8.0:1616513088.789197:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.789200:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513088.789204:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71ba0 00000800:00000200:8.0:1616513088.789207:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.789209:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.789211:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.789215:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.789268:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114552853632 00010000:00000200:24.0:1616513088.789279:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1693114552853632/t0(0) o400->20fea48e-3e34-9bce-f9d1-1b186a0a712a@10.31.136.162@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.789303:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114552853632, offset 192 00000400:00000200:24.0:1616513088.789309:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.162@tcp1 00000400:00000200:24.0:1616513088.789317:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.162@tcp1 routed destination 00000400:00000200:24.0:1616513088.789332:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.162@tcp1(10.31.136.162@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.789338:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.789344:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.789347:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.789349:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.789354:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.789359:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.789360:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513088.789411:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513088.789417:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513088.789420:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513088.789427:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513088.789432:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfb5c8 00000800:00000200:12.0:1616513088.789448:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.789452:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.789569:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.789580:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.789587:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513088.789588:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.789597:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.11@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.789605:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.11@tcp1 of length 224 into portal 26 MB=0x5ffe22a4b73c0 00000400:00000200:28.0:1616513088.789612:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.11@tcp1 of length 224/224 into md 0xf97ccac9 [1] + 0 00000400:00000200:28.0:1616513088.789617:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f70a18 00000400:00000200:28.0:1616513088.789619:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.789622:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513088.789627:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f70a18 00000800:00000200:28.0:1616513088.789630:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.789633:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.789635:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.789640:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688721720832960 00010000:00000200:24.0:1616513088.789646:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688721720832960/t0(0) o400->5f5206ef-8765-a0eb-291e-91431c25d79b@10.31.143.11@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.789657:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688721720832960, offset 192 00000400:00000200:24.0:1616513088.789663:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.11@tcp1 00000800:00000200:7.2:1616513088.789667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513088.789668:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.11@tcp1 routed destination 00000400:00000200:24.0:1616513088.789681:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.11@tcp1(10.31.143.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.789687:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.789692:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.789695:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.789696:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513088.789700:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:24.0:1616513088.789704:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513088.789722:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.789732:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513088.789734:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513088.789740:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513088.789741:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.11@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.789743:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.789749:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.789750:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.11@tcp1 of length 224 into portal 12 MB=0x5ffe22a4b7680 00000400:00000200:14.0:1616513088.789754:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfb5c8 00000400:00000200:2.0:1616513088.789756:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.11@tcp1 of length 224/224 into md 0xf964d519 [1] + 73448 00000800:00000200:14.0:1616513088.789758:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.789761:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513088.789761:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.789764:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513088.789766:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513088.789771:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513088.789773:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.162@tcp1 : PUT - for me 00000800:00000200:2.0:1616513088.789773:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513088.789775:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.789778:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513088.789781:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.162@tcp1 of length 224 into portal 12 MB=0x603e0f37ca680 00000400:00000200:14.0:1616513088.789787:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.162@tcp1 of length 224/224 into md 0xf964d519 [1] + 73672 00000400:00000200:14.0:1616513088.789792:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.789794:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513088.789802:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.789804:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.789806:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513088.789839:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688721720833664 00000100:00000200:18.0:1616513088.789841:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114552854144 00010000:00000200:30.0:1616513088.789851:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cbf00 x1688721720833664/t0(0) o400->0abb512e-f387-042f-a76f-43170798d71b@10.31.143.11@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:18.0:1616513088.789854:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05100 x1693114552854144/t0(0) o400->910f9e79-4053-a48d-e3f1-64edd9ceed53@10.31.136.162@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513088.789865:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688721720833664, offset 192 00000100:00000200:18.0:1616513088.789869:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114552854144, offset 192 00000400:00000200:30.0:1616513088.789872:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.11@tcp1 00000400:00000200:18.0:1616513088.789878:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.162@tcp1 00000400:00000200:30.0:1616513088.789880:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.11@tcp1 routed destination 00000400:00000200:18.0:1616513088.789889:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.162@tcp1 routed destination 00000400:00000200:30.0:1616513088.789894:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.11@tcp1(10.31.143.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513088.789901:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513088.789904:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.162@tcp1(10.31.136.162@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513088.789913:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513088.789914:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513088.789916:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513088.789918:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513088.789919:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513088.789921:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513088.789923:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513088.789924:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.789928:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513088.789928:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513088.789933:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513088.789937:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513088.789941:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513088.789941:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513088.789942:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513088.789945:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513088.789946:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513088.789947:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513088.789948:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513088.789950:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513088.789951:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eecc0 00000800:00000200:12.0:1616513088.789953:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513088.789953:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.789956:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc330 00000800:00000200:2.0:1616513088.789960:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513088.794585:0:6480:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927440, portal 4 00000100:00000200:0.0:1616513088.794590:0:6480:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927440, offset 0 00000400:00000200:0.0:1616513088.794599:0:6480:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:0.0:1616513088.794605:0:6480:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:0.0:1616513088.794617:0:6480:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513088.794624:0:6480:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:0.0:1616513088.794630:0:6480:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513088.794633:0:6480:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:0.0:1616513088.794635:0:6480:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:0.0:1616513088.794640:0:6480:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513088.794687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513088.794740:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513088.794746:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513088.794748:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.794754:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513088.794759:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56cbad1680 x1693067725927440/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513088.794770:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43078 00000800:00000200:14.0:1616513088.794773:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513088.794776:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513088.794860:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513088.794914:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513088.794923:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513088.794932:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513088.794940:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627410 00000400:00000200:18.0:1616513088.794946:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cd139 [1] + 192 00000400:00000200:18.0:1616513088.794951:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.794956:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56cbad1680 x1693067725927440/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513088.794967:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513088.794971:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513088.794974:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513088.794976:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:0.0:1616513088.794980:0:6480:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56cbad1680 x1693067725927440/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:0.0:1616513088.794993:0:6480:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56cbad1680 x1693067725927440/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513088.795002:0:6480:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42cc0 00000800:00000200:7.2:1616513088.799559:0:50411:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.799611:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.799621:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513088.799623:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513088.799628:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.251@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.799637:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.251@tcp1 of length 224 into portal 26 MB=0x603ff5c93c340 00000400:00000200:14.0:1616513088.799642:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.251@tcp1 of length 224/224 into md 0xf97ccad9 [1] + 0 00000400:00000200:14.0:1616513088.799646:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8481f71980 00000400:00000200:14.0:1616513088.799649:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.799651:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.799655:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8481f71980 00000800:00000200:14.0:1616513088.799657:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.799660:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.799661:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.799716:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245164995392 00010000:00000200:24.0:1616513088.799726:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1693245164995392/t0(0) o400->e0cf9265-6661-2b8e-47df-572beb28eff8@10.31.132.251@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.799739:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245164995392, offset 192 00000400:00000200:24.0:1616513088.799745:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.251@tcp1 00000400:00000200:24.0:1616513088.799753:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.251@tcp1 routed destination 00000400:00000200:24.0:1616513088.799768:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.251@tcp1(10.31.132.251@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.799774:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.799779:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.799781:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.799784:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.799788:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.799794:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.799834:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.799846:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.799850:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.799853:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.799859:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.799863:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2dd0 00000800:00000200:8.0:1616513088.799866:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.799869:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.799989:0:50411:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513088.800045:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.800054:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513088.800062:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.251@tcp1 : PUT - for me 00000400:00000200:2.0:1616513088.800071:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.251@tcp1 of length 224 into portal 12 MB=0x603ff5c93c480 00000400:00000200:2.0:1616513088.800077:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.251@tcp1 of length 224/224 into md 0xf964d519 [1] + 73896 00000400:00000200:2.0:1616513088.800082:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.800085:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513088.800092:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.800097:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.800098:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.800102:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.800156:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245164995712 00010000:00000200:22.0:1616513088.800168:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c8900 x1693245164995712/t0(0) o400->eae8cb9b-5084-20c7-b27c-c907a0576783@10.31.132.251@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.800183:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245164995712, offset 192 00000400:00000200:22.0:1616513088.800189:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.251@tcp1 00000400:00000200:22.0:1616513088.800197:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.251@tcp1 routed destination 00000400:00000200:22.0:1616513088.800212:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.251@tcp1(10.31.132.251@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.800218:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.800224:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.800227:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.800229:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.800234:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.800237:0:50411:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.800246:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.800250:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.800252:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.800254:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513088.800258:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.800261:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a198 00000800:00000200:26.0:1616513088.800265:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513088.810660:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927456, portal 4 00000100:00000200:19.0:1616513088.810666:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927456, offset 0 00000400:00000200:19.0:1616513088.810673:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:19.0:1616513088.810682:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:19.0:1616513088.810695:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513088.810702:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:19.0:1616513088.810709:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513088.810712:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:19.0:1616513088.810714:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:19.0:1616513088.810720:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513088.810770:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513088.810790:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513088.810796:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513088.810800:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513088.810807:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513088.810813:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63e300 x1693067725927456/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513088.810826:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706110 00000800:00000200:27.0:1616513088.810830:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513088.810833:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513088.810936:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513088.810945:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513088.810951:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513088.810959:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513088.810967:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627420 00000400:00000200:27.0:1616513088.810974:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166f1d [1] + 192 00000400:00000200:27.0:1616513088.810979:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513088.810983:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63e300 x1693067725927456/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513088.810997:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513088.810997:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513088.811000:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513088.811002:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:19.0:1616513088.811004:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63e300 x1693067725927456/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513088.811017:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63e300 x1693067725927456/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513088.811027:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000800:00000200:7.2:1616513088.816868:0:50413:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.816922:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.816932:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.816941:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.156@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.816950:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.156@tcp1 of length 224 into portal 26 MB=0x5ffd839a15e80 00000400:00000200:18.0:1616513088.816957:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.156@tcp1 of length 224/224 into md 0xf97ccaf9 [1] + 0 00000400:00000200:18.0:1616513088.816962:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2660 00000400:00000200:18.0:1616513088.816964:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.816967:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.816971:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:18.0:1616513088.816975:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.816976:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.816977:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.816980:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.817056:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688679028448896 00010000:00000200:24.0:1616513088.817066:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688679028448896/t0(0) o400->89bdf234-0756-6da1-06de-9e7786fd111c@10.31.129.156@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.817079:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688679028448896, offset 192 00000400:00000200:24.0:1616513088.817085:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.156@tcp1 00000400:00000200:24.0:1616513088.817093:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.156@tcp1 routed destination 00000400:00000200:24.0:1616513088.817106:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.156@tcp1(10.31.129.156@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.817113:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.817118:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.817120:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.817122:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.817127:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.817133:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513088.817136:0:50413:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513088.817174:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513088.817187:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.817196:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.817204:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.156@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.817211:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.156@tcp1 of length 224 into portal 12 MB=0x5ffd839a162c0 00000400:00000200:14.0:1616513088.817217:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.156@tcp1 of length 224/224 into md 0xf964d519 [1] + 74120 00000400:00000200:14.0:1616513088.817222:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.817224:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.817231:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513088.817231:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.817234:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.817235:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513088.817238:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513088.817238:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513088.817241:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513088.817247:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513088.817253:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2880 00000800:00000200:20.0:1616513088.817256:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.817259:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513088.817291:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688679028449984 00010000:00000200:22.0:1616513088.817303:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e02880 x1688679028449984/t0(0) o400->2eb67d5e-b760-3508-f0b9-a16cd56254c9@10.31.129.156@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.817316:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688679028449984, offset 192 00000400:00000200:22.0:1616513088.817322:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.156@tcp1 00000400:00000200:22.0:1616513088.817329:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.156@tcp1 routed destination 00000400:00000200:22.0:1616513088.817343:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.156@tcp1(10.31.129.156@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.817350:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.817356:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.817358:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.817360:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.817365:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.817369:0:50413:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.817380:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.817384:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.817387:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513088.817389:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513088.817393:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.817396:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b7e8 00000800:00000200:2.0:1616513088.817400:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.839202:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.839254:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.839263:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.839271:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.99@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.839280:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.99@tcp1 of length 224 into portal 26 MB=0x5ffe267987440 00000400:00000200:14.0:1616513088.839286:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.99@tcp1 of length 224/224 into md 0xf97ccb19 [1] + 0 00000400:00000200:14.0:1616513088.839291:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2220 00000400:00000200:14.0:1616513088.839293:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.839296:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.839300:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:14.0:1616513088.839304:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.839307:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.839307:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.839311:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.839363:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688722749289536 00010000:00000200:24.0:1616513088.839374:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688722749289536/t0(0) o400->e0d2b83c-f2ef-b114-ea57-fe19e0da75d9@10.31.141.99@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.839388:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688722749289536, offset 192 00000400:00000200:24.0:1616513088.839394:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.99@tcp1 00000400:00000200:24.0:1616513088.839402:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.99@tcp1 routed destination 00000400:00000200:24.0:1616513088.839416:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.99@tcp1(10.31.141.99@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.839423:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.839428:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.839432:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.839447:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.839452:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.839459:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.839498:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.839511:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.839517:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.839519:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.839525:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.839530:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f36d8 00000800:00000200:8.0:1616513088.839533:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.839535:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.839541:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513088.839548:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.99@tcp1 : PUT - for me 00000400:00000200:8.0:1616513088.839556:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.99@tcp1 of length 224 into portal 12 MB=0x5ffe2679877c0 00000400:00000200:8.0:1616513088.839561:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.99@tcp1 of length 224/224 into md 0x865be291 [1] + 6160 00000800:00000200:28.0:1616513088.839562:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.839566:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.839569:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513088.839574:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.839576:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.839579:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.839639:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688722749290432 00010000:00000200:16.0:1616513088.839651:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8f500 x1688722749290432/t0(0) o400->e9ac84b5-64a7-3522-3495-2431dd787d68@10.31.141.99@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.839666:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688722749290432, offset 192 00000400:00000200:16.0:1616513088.839672:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.99@tcp1 00000400:00000200:16.0:1616513088.839680:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.99@tcp1 routed destination 00000400:00000200:16.0:1616513088.839695:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.99@tcp1(10.31.141.99@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.839701:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.839707:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.839710:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.839712:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.839717:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.839721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.839773:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.839780:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.839783:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513088.839789:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.839793:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788000 00000800:00000200:18.0:1616513088.839797:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.839800:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513088.842635:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927488, portal 4 00000100:00000200:8.0:1616513088.842639:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927488, offset 0 00000100:00000200:27.0:1616513088.842643:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927472, portal 4 00000400:00000200:8.0:1616513088.842645:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000100:00000200:27.0:1616513088.842649:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927472, offset 0 00000400:00000200:8.0:1616513088.842650:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:27.0:1616513088.842656:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:27.0:1616513088.842663:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:8.0:1616513088.842663:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513088.842669:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:27.0:1616513088.842677:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513088.842684:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:8.0:1616513088.842684:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513088.842686:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:8.0:1616513088.842688:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:27.0:1616513088.842691:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513088.842691:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:27.0:1616513088.842694:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:27.0:1616513088.842696:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513088.842702:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513088.842710:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:25.2:1616513088.842738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:12.0:1616513088.842766:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513088.842773:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513088.842776:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513088.842783:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:12.0:1616513088.842790:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd604c80 x1693067725927472/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513088.842796:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000400:00000200:12.0:1616513088.842802:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248550 00000800:00000200:26.0:1616513088.842803:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513088.842806:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513088.842806:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513088.842809:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000400:00000200:26.0:1616513088.842812:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513088.842818:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb23ec00 x1693067725927488/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513088.842830:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2f68 00000800:00000200:26.0:1616513088.842834:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513088.842837:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:5.2:1616513088.842886:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513088.842895:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513088.842900:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:25.2:1616513088.842906:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000400:00000200:8.0:1616513088.842906:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513088.842912:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627430 00000400:00000200:8.0:1616513088.842917:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d9b81 [1] + 192 00000400:00000200:8.0:1616513088.842921:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513088.842926:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd604c80 x1693067725927472/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513088.842939:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513088.842941:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:8.0:1616513088.842942:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513088.842944:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:27.0:1616513088.842950:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd604c80 x1693067725927472/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513088.842963:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd604c80 x1693067725927472/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513088.842968:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000400:00000200:27.0:1616513088.842976:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248660 00000800:00000200:18.0:1616513088.842978:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513088.842986:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513088.842994:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627440 00000400:00000200:18.0:1616513088.843001:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cd199 [1] + 192 00000400:00000200:18.0:1616513088.843006:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.843011:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb23ec00 x1693067725927488/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513088.843022:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513088.843026:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513088.843029:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513088.843030:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:8.0:1616513088.843030:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb23ec00 x1693067725927488/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:8.0:1616513088.843041:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb23ec00 x1693067725927488/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513088.843049:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:11.2:1616513088.849631:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.849688:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.849697:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513088.849706:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.113@tcp1 : PUT - for me 00000400:00000200:12.0:1616513088.849715:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.113@tcp1 of length 224 into portal 26 MB=0x5f91af07b22c0 00000400:00000200:12.0:1616513088.849722:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.113@tcp1 of length 224/224 into md 0xf97ccb39 [1] + 0 00000400:00000200:12.0:1616513088.849727:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3a90 00000400:00000200:12.0:1616513088.849729:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513088.849732:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513088.849736:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:12.0:1616513088.849739:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.849741:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513088.849742:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513088.849748:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513088.849751:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.849752:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513088.849762:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.113@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.849771:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.113@tcp1 of length 224 into portal 12 MB=0x5f91af07b2340 00000400:00000200:28.0:1616513088.849778:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.113@tcp1 of length 224/224 into md 0x865be291 [1] + 6384 00000400:00000200:28.0:1616513088.849783:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.849786:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513088.849792:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.849795:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513088.849797:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.849800:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681268982620864 00010000:00000200:24.0:1616513088.849811:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1681268982620864/t0(0) o400->2b323c1f-4d8d-9062-ec13-65888427d569@10.31.143.113@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.849826:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681268982620864, offset 192 00000400:00000200:24.0:1616513088.849832:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.113@tcp1 00000400:00000200:24.0:1616513088.849839:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.113@tcp1 routed destination 00000400:00000200:24.0:1616513088.849854:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.113@tcp1(10.31.143.113@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513088.849855:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681268982620992 00000800:00000200:24.0:1616513088.849861:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.849867:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:16.0:1616513088.849867:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa7500 x1681268982620992/t0(0) o400->ce4b2284-f36e-131e-d0f3-f96011d9c933@10.31.143.113@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513088.849869:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.849872:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.849876:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513088.849882:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681268982620992, offset 192 00000800:00000200:11.2:1616513088.849882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.849883:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513088.849890:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.113@tcp1 00000800:00000200:8.0:1616513088.849892:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513088.849897:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.113@tcp1 routed destination 00000800:00000200:8.0:1616513088.849897:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.849900:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513088.849901:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.849906:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513088.849912:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.113@tcp1(10.31.143.113@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513088.849923:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2d48 00000800:00000200:16.0:1616513088.849925:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513088.849926:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513088.849930:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.849933:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.849935:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.849940:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.849945:0:50417:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.849997:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.850003:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.850006:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.850008:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513088.850012:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.850015:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788c38 00000800:00000200:26.0:1616513088.850019:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.856543:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.856598:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.856606:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513088.856614:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.228@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.856623:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.228@tcp1 of length 224 into portal 26 MB=0x603e0e67b94c0 00000400:00000200:28.0:1616513088.856629:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.228@tcp1 of length 224/224 into md 0xf97ccb59 [1] + 0 00000400:00000200:28.0:1616513088.856634:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3210 00000400:00000200:28.0:1616513088.856636:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.856638:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513088.856642:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3210 00000800:00000200:28.0:1616513088.856645:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.856648:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.856649:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.856652:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.856663:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513088.856671:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.228@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.856680:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.228@tcp1 of length 224 into portal 12 MB=0x603e0e67b9540 00000400:00000200:20.0:1616513088.856688:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.228@tcp1 of length 224/224 into md 0x865be291 [1] + 6608 00000400:00000200:20.0:1616513088.856694:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.856696:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.856702:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513088.856703:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114334680256 00000800:00000200:20.0:1616513088.856706:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513088.856708:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.856710:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:24.0:1616513088.856713:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1693114334680256/t0(0) o400->175437e7-bd9c-bda4-aa72-df7d5af562d8@10.31.136.228@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.856726:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114334680256, offset 192 00000400:00000200:24.0:1616513088.856732:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.228@tcp1 00000400:00000200:24.0:1616513088.856739:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.228@tcp1 routed destination 00000400:00000200:24.0:1616513088.856752:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.228@tcp1(10.31.136.228@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.856759:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.856764:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.856766:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.856768:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.856773:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.856778:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.856779:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:16.0:1616513088.856779:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114334680384 00010000:00000200:16.0:1616513088.856791:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5a00 x1693114334680384/t0(0) o400->c67a0590-2f41-f737-8b80-e5f967966f23@10.31.136.228@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:8.0:1616513088.856791:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.856796:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.856798:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513088.856801:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513088.856804:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114334680384, offset 192 00000400:00000200:8.0:1616513088.856804:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.856808:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3430 00000800:00000200:8.0:1616513088.856811:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:16.0:1616513088.856813:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.228@tcp1 00000400:00000200:16.0:1616513088.856820:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.228@tcp1 routed destination 00000400:00000200:16.0:1616513088.856834:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.228@tcp1(10.31.136.228@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.856840:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.856846:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.856848:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.856850:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.856855:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.856859:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.856911:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.856918:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513088.856921:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513088.856928:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513088.856932:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789078 00000800:00000200:2.0:1616513088.856936:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.856939:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.859045:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513088.859101:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.859111:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513088.859120:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.128@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.859130:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.128@tcp1 of length 224 into portal 26 MB=0x603fc0f4b8b00 00000400:00000200:20.0:1616513088.859137:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.128@tcp1 of length 224/224 into md 0xf97ccb79 [1] + 0 00000400:00000200:20.0:1616513088.859142:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f8f8 00000400:00000200:20.0:1616513088.859144:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.859147:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513088.859152:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f8f8 00000800:00000200:20.0:1616513088.859158:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.859160:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513088.859162:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.859164:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.859169:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513088.859170:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.859176:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.128@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.859182:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.128@tcp1 of length 224 into portal 12 MB=0x603fc0f4b8b80 00000400:00000200:20.0:1616513088.859187:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.128@tcp1 of length 224/224 into md 0x865be291 [1] + 6832 00000400:00000200:20.0:1616513088.859191:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.859193:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.859197:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.859199:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.859201:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.859216:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230983514880 00010000:00000200:24.0:1616513088.859226:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1693230983514880/t0(0) o400->0da8023f-2f4d-af91-a42b-87c76c097843@10.31.138.128@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.859239:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230983514880, offset 192 00000400:00000200:24.0:1616513088.859244:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.128@tcp1 00000400:00000200:24.0:1616513088.859252:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.128@tcp1 routed destination 00000400:00000200:24.0:1616513088.859266:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.128@tcp1(10.31.138.128@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.859272:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.859277:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:16.0:1616513088.859278:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230983515008 00000800:00000200:24.0:1616513088.859279:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.859281:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.859286:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:16.0:1616513088.859289:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad3600 x1693230983515008/t0(0) o400->50b29ec9-e3b5-9bd0-0f76-d015b62f27c9@10.31.138.128@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513088.859291:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.859292:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513088.859301:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:16.0:1616513088.859303:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230983515008, offset 192 00000800:00000200:8.0:1616513088.859306:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.859308:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513088.859309:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.128@tcp1 00000800:00000200:12.0:1616513088.859309:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513088.859313:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513088.859316:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.128@tcp1 routed destination 00000400:00000200:8.0:1616513088.859317:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3188 00000800:00000200:8.0:1616513088.859321:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:16.0:1616513088.859329:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.128@tcp1(10.31.138.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.859336:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.859341:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.859345:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.859347:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.859352:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.859357:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.859410:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.859416:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.859419:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.859425:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.859429:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7896d8 00000800:00000200:14.0:1616513088.859432:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.859444:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.859450:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513088.859452:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.859454:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.859455:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.859458:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.867642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.867696:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.867705:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.867714:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.6@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.867723:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.6@tcp1 of length 224 into portal 26 MB=0x5ffe8006b5980 00000400:00000200:18.0:1616513088.867729:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.6@tcp1 of length 224/224 into md 0xf97ccb89 [1] + 0 00000400:00000200:18.0:1616513088.867734:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e550 00000400:00000200:18.0:1616513088.867736:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.867739:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.867743:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e550 00000800:00000200:18.0:1616513088.867746:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.867747:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.867749:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.867751:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.867804:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688746788084096 00010000:00000200:24.0:1616513088.867814:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688746788084096/t0(0) o400->9c29ab34-4ecf-ff7c-20b9-69da63e04278@10.31.142.6@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513088.867814:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513088.867827:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688746788084096, offset 192 00000400:00000200:24.0:1616513088.867833:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.6@tcp1 00000400:00000200:24.0:1616513088.867840:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.6@tcp1 routed destination 00000400:00000200:24.0:1616513088.867853:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.6@tcp1(10.31.142.6@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.867859:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.867865:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.867867:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.867869:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.867873:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513088.867874:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:24.0:1616513088.867880:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513088.867883:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513088.867892:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.6@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.867900:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.6@tcp1 of length 224 into portal 12 MB=0x5ffe8006b5c40 00000400:00000200:20.0:1616513088.867908:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.6@tcp1 of length 224/224 into md 0x865be291 [1] + 7056 00000400:00000200:20.0:1616513088.867913:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.867916:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.867922:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513088.867924:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513088.867925:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513088.867931:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513088.867932:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513088.867934:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.867935:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513088.867941:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.867946:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2088 00000800:00000200:28.0:1616513088.867949:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513088.867984:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688746788084800 00010000:00000200:16.0:1616513088.868002:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad7980 x1688746788084800/t0(0) o400->c0910577-7ab2-42c5-75a4-25733a39f9f5@10.31.142.6@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513088.868016:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688746788084800, offset 192 00000400:00000200:16.0:1616513088.868022:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.6@tcp1 00000400:00000200:16.0:1616513088.868029:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.6@tcp1 routed destination 00000400:00000200:16.0:1616513088.868042:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.6@tcp1(10.31.142.6@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513088.868049:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513088.868054:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.868056:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.868058:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513088.868063:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.868067:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.868118:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.868125:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.868128:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.868136:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.868140:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788d48 00000800:00000200:26.0:1616513088.868144:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.868147:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:31.0:1616513088.871548:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513088.871556:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000100:00000200:15.0:1616513088.874640:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927504, portal 4 00000100:00000200:15.0:1616513088.874645:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927504, offset 0 00000400:00000200:15.0:1616513088.874652:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:15.0:1616513088.874658:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:15.0:1616513088.874670:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513088.874676:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:15.0:1616513088.874682:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513088.874685:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:15.0:1616513088.874687:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:15.0:1616513088.874691:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513088.874737:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.874755:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513088.874761:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513088.874763:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513088.874767:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513088.874770:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513088.874776:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bfda00 x1693067725927504/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513088.874789:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0abb0 00000800:00000200:1.0:1616513088.874794:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513088.874911:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513088.874920:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.874926:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513088.874928:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513088.874934:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513088.874942:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627450 00000400:00000200:1.0:1616513088.874948:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffed5 [1] + 192 00000400:00000200:1.0:1616513088.874953:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513088.874957:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bfda00 x1693067725927504/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513088.874969:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.874972:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513088.874973:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:15.0:1616513088.874977:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bfda00 x1693067725927504/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513088.874990:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bfda00 x1693067725927504/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513088.875011:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000800:00000200:7.2:1616513088.887499:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.887551:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.887559:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.887567:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.81@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.887576:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.81@tcp1 of length 224 into portal 26 MB=0x60183067c7480 00000400:00000200:14.0:1616513088.887582:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.81@tcp1 of length 224/224 into md 0xf97ccba9 [1] + 0 00000400:00000200:14.0:1616513088.887586:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e770 00000400:00000200:14.0:1616513088.887589:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.887591:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.887594:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e770 00000800:00000200:14.0:1616513088.887597:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.887600:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.887601:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.887604:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.887655:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512121427072 00010000:00000200:24.0:1616513088.887664:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1690512121427072/t0(0) o400->cb598134-f280-2176-b039-e87c015c2374@10.31.138.81@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.887676:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512121427072, offset 192 00000400:00000200:24.0:1616513088.887683:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.81@tcp1 00000400:00000200:24.0:1616513088.887690:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.81@tcp1 routed destination 00000400:00000200:24.0:1616513088.887705:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.81@tcp1(10.31.138.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.887710:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.887715:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.887718:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.887720:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.887724:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.887730:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.887770:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513088.887783:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513088.887787:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513088.887789:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513088.887795:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513088.887799:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f33a8 00000800:00000200:8.0:1616513088.887802:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513088.887804:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.887852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.887905:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.887914:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.887923:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.81@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.887932:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.81@tcp1 of length 224 into portal 12 MB=0x60183067c75c0 00000400:00000200:26.0:1616513088.887938:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.81@tcp1 of length 224/224 into md 0xf964d519 [1] + 74344 00000400:00000200:26.0:1616513088.887943:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.887946:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.887953:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.887956:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.887958:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.887961:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.888019:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512121427392 00010000:00000200:22.0:1616513088.888031:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693a80 x1690512121427392/t0(0) o400->bf74dbbe-6a98-c42d-b93f-bf7a33a93a6a@10.31.138.81@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.888046:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512121427392, offset 192 00000400:00000200:22.0:1616513088.888052:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.81@tcp1 00000400:00000200:22.0:1616513088.888060:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.81@tcp1 routed destination 00000400:00000200:22.0:1616513088.888075:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.81@tcp1(10.31.138.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.888081:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.888087:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.888090:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.888092:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.888097:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.888102:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.888113:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.888121:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.888123:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.888125:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.888132:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.888136:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b3a8 00000800:00000200:18.0:1616513088.888140:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.911942:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513088.911998:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513088.912008:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513088.912017:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.194@tcp1 : PUT - for me 00000400:00000200:28.0:1616513088.912027:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.194@tcp1 of length 224 into portal 26 MB=0x5ffdb51d2c0c0 00000400:00000200:28.0:1616513088.912033:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.194@tcp1 of length 224/224 into md 0xf97ccbc9 [1] + 0 00000400:00000200:28.0:1616513088.912039:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f100 00000400:00000200:28.0:1616513088.912042:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513088.912045:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513088.912049:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f100 00000800:00000200:28.0:1616513088.912052:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513088.912052:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.912055:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.912059:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513088.912062:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.912063:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513088.912071:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.194@tcp1 : PUT - for me 00000400:00000200:12.0:1616513088.912080:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.194@tcp1 of length 224 into portal 12 MB=0x5ffdb51d2c2c0 00000400:00000200:12.0:1616513088.912086:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.194@tcp1 of length 224/224 into md 0x865be291 [1] + 7280 00000400:00000200:12.0:1616513088.912092:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513088.912094:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513088.912101:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.912104:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.912106:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.912113:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692319240384 00010000:00000200:24.0:1616513088.912124:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688692319240384/t0(0) o400->2818b46d-daa9-4ee8-152f-5532c15064fc@10.31.141.194@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.912138:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692319240384, offset 192 00000400:00000200:24.0:1616513088.912144:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.194@tcp1 00000400:00000200:24.0:1616513088.912152:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.194@tcp1 routed destination 00000400:00000200:24.0:1616513088.912166:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.194@tcp1(10.31.141.194@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513088.912166:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692319240896 00000800:00000200:24.0:1616513088.912173:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:16.0:1616513088.912178:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d205e80 x1688692319240896/t0(0) o400->744c5a1e-c11f-480e-81b2-1ddd4b9938e2@10.31.141.194@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513088.912179:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.912182:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.912184:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.912189:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513088.912192:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692319240896, offset 192 00000800:00000200:11.2:1616513088.912194:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.912195:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513088.912199:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.194@tcp1 00000400:00000200:16.0:1616513088.912207:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.194@tcp1 routed destination 00000800:00000200:20.0:1616513088.912216:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513088.912222:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.194@tcp1(10.31.141.194@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513088.912225:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513088.912227:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513088.912228:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513088.912229:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.912234:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:16.0:1616513088.912234:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513088.912237:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513088.912239:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513088.912240:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3ba0 00000800:00000200:20.0:1616513088.912243:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513088.912244:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.912252:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.912309:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.912316:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.912319:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.912326:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.912330:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789ed0 00000800:00000200:26.0:1616513088.912334:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.912337:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.922042:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513088.922097:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.922107:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513088.922116:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.225@tcp1 : PUT - for me 00000400:00000200:14.0:1616513088.922125:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.225@tcp1 of length 224 into portal 26 MB=0x602172c8f0500 00000400:00000200:14.0:1616513088.922134:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.225@tcp1 of length 224/224 into md 0xf97ccbe9 [1] + 0 00000400:00000200:14.0:1616513088.922138:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f188 00000400:00000200:14.0:1616513088.922141:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.922144:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.922148:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f188 00000800:00000200:18.0:1616513088.922151:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.922151:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.922154:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.922155:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.922211:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691148415337728 00010000:00000200:24.0:1616513088.922222:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1691148415337728/t0(0) o400->8951672c-b621-ba8f-93a2-da42a387ec77@10.31.140.225@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513088.922231:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513088.922235:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691148415337728, offset 192 00000400:00000200:24.0:1616513088.922242:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.225@tcp1 00000400:00000200:24.0:1616513088.922250:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.225@tcp1 routed destination 00000400:00000200:24.0:1616513088.922264:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.225@tcp1(10.31.140.225@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.922275:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.922281:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.922283:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.922285:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513088.922288:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.922290:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.922297:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513088.922298:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.922307:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.225@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.922315:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.225@tcp1 of length 224 into portal 12 MB=0x602172c8f0780 00000400:00000200:26.0:1616513088.922322:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.225@tcp1 of length 224/224 into md 0xf964d519 [1] + 74568 00000400:00000200:26.0:1616513088.922327:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.922330:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.922336:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.922339:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513088.922339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513088.922341:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.922344:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.922412:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691148415338368 00000800:00000200:28.0:1616513088.922422:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:22.0:1616513088.922425:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693600 x1691148415338368/t0(0) o400->868fd706-4967-9f4e-5a19-8f70a834d1bb@10.31.140.225@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513088.922429:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.922432:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513088.922453:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:22.0:1616513088.922457:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691148415338368, offset 192 00000400:00000200:28.0:1616513088.922458:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3650 00000800:00000200:28.0:1616513088.922462:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513088.922464:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.225@tcp1 00000800:00000200:28.0:1616513088.922465:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513088.922471:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.225@tcp1 routed destination 00000400:00000200:22.0:1616513088.922486:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.225@tcp1(10.31.140.225@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.922493:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.922499:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.922501:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.922504:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.922508:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.922514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.922524:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.922530:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.922532:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513088.922534:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513088.922539:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.922543:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ab28 00000800:00000200:18.0:1616513088.922547:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513088.938637:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927520, portal 4 00000100:00000200:23.0:1616513088.938643:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927520, offset 0 00000400:00000200:23.0:1616513088.938651:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:23.0:1616513088.938657:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:23.0:1616513088.938670:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513088.938677:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:23.0:1616513088.938684:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513088.938687:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:23.0:1616513088.938689:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:23.0:1616513088.938695:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:15.0:1616513088.938699:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927536, portal 4 00000100:00000200:15.0:1616513088.938704:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927536, offset 0 00000400:00000200:15.0:1616513088.938711:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:15.0:1616513088.938717:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:15.0:1616513088.938730:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513088.938737:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:1.2:1616513088.938741:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:15.0:1616513088.938744:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513088.938747:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:15.0:1616513088.938749:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:15.0:1616513088.938755:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:1.0:1616513088.938759:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513088.938765:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513088.938768:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513088.938775:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513088.938781:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bfbf00 x1693067725927520/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513088.938793:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e4c8 00000800:00000200:1.0:1616513088.938798:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513088.938800:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:25.2:1616513088.938812:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513088.938874:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513088.938881:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513088.938884:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.2:1616513088.938889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:2.0:1616513088.938891:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:1.0:1616513088.938897:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:2.0:1616513088.938898:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc566780 x1693067725927536/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513088.938903:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513088.938906:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:2.0:1616513088.938910:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0bb18 00000400:00000200:1.0:1616513088.938912:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:2.0:1616513088.938915:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513088.938918:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000400:00000200:1.0:1616513088.938920:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627460 00000400:00000200:1.0:1616513088.938926:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436ffee5 [1] + 192 00000800:00000200:25.2:1616513088.938927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000400:00000200:1.0:1616513088.938931:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513088.938936:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bfbf00 x1693067725927520/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513088.938949:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513088.938955:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513088.938956:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:23.0:1616513088.938957:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bfbf00 x1693067725927520/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513088.938970:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bfbf00 x1693067725927520/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513088.938980:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f4b8 00000800:00000200:14.0:1616513088.938982:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513088.938991:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513088.939001:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513088.939009:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627470 00000400:00000200:14.0:1616513088.939016:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cd239 [1] + 192 00000400:00000200:14.0:1616513088.939021:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513088.939026:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc566780 x1693067725927536/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513088.939036:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513088.939041:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513088.939044:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513088.939046:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:15.0:1616513088.939051:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc566780 x1693067725927536/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513088.939064:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc566780 x1693067725927536/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513088.939075:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000800:00000200:11.2:1616513088.947807:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513088.947864:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.947873:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513088.947882:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.120@tcp1 : PUT - for me 00000400:00000200:12.0:1616513088.947891:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.120@tcp1 of length 224 into portal 26 MB=0x5ffd6526b7c80 00000400:00000200:12.0:1616513088.947898:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.120@tcp1 of length 224/224 into md 0xf97ccc19 [1] + 0 00000400:00000200:12.0:1616513088.947903:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fc28 00000400:00000200:12.0:1616513088.947906:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513088.947909:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513088.947913:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fc28 00000800:00000200:12.0:1616513088.947916:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.947917:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.947919:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513088.947921:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.947976:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688670854413440 00010000:00000200:24.0:1616513088.947986:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688670854413440/t0(0) o400->8cf6507a-4e60-7e32-55de-4f93cf923582@10.31.132.120@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.947999:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688670854413440, offset 192 00000400:00000200:24.0:1616513088.948005:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.120@tcp1 00000400:00000200:24.0:1616513088.948012:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.120@tcp1 routed destination 00000400:00000200:24.0:1616513088.948026:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.120@tcp1(10.31.132.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.948032:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.948038:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.948040:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.948042:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.948047:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513088.948052:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.948053:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513088.948105:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.948112:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.948115:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513088.948116:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513088.948121:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.948127:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2440 00000800:00000200:28.0:1616513088.948130:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.948557:0:50429:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513088.948613:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513088.948623:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513088.948631:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.120@tcp1 : PUT - for me 00000400:00000200:2.0:1616513088.948640:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.120@tcp1 of length 224 into portal 12 MB=0x5ffd6526b7d80 00000400:00000200:2.0:1616513088.948647:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.120@tcp1 of length 224/224 into md 0xf964d519 [1] + 74792 00000400:00000200:2.0:1616513088.948652:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513088.948654:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513088.948661:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513088.948666:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.948667:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513088.948671:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.948733:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688670854413696 00010000:00000200:22.0:1616513088.948745:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cda00 x1688670854413696/t0(0) o400->d8534b5f-471a-e704-aa57-0b02eab2f29a@10.31.132.120@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.948760:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688670854413696, offset 192 00000400:00000200:22.0:1616513088.948766:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.120@tcp1 00000400:00000200:22.0:1616513088.948774:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.120@tcp1 routed destination 00000400:00000200:22.0:1616513088.948789:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.120@tcp1(10.31.132.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.948795:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.948801:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.948804:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.948806:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.948811:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.948814:0:50429:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.948867:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.948874:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513088.948877:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513088.948883:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513088.948887:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a088 00000800:00000200:18.0:1616513088.948891:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.948894:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.957722:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513088.957779:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.957789:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513088.957798:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.78@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.957807:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.78@tcp1 of length 224 into portal 26 MB=0x5ffd4f19e2b00 00000400:00000200:20.0:1616513088.957813:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.78@tcp1 of length 224/224 into md 0xf97ccc39 [1] + 0 00000400:00000200:20.0:1616513088.957818:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ec38 00000400:00000200:20.0:1616513088.957821:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.957823:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513088.957828:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ec38 00000800:00000200:20.0:1616513088.957831:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.957833:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513088.957833:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513088.957840:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513088.957843:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.957844:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513088.957852:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.78@tcp1 : PUT - for me 00000400:00000200:12.0:1616513088.957863:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.78@tcp1 of length 224 into portal 12 MB=0x5ffd4f19e2b80 00000400:00000200:12.0:1616513088.957869:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.78@tcp1 of length 224/224 into md 0x865be291 [1] + 7504 00000400:00000200:12.0:1616513088.957875:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513088.957877:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513088.957884:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.957887:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.957889:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513088.957905:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664935377792 00010000:00000200:20.0:1616513088.957917:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d203180 x1688664935377792/t0(0) o400->b6be00b8-c082-e2cb-5dae-bfe7a2566a7d@10.31.131.78@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513088.957918:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664935377664 00010000:00000200:24.0:1616513088.957929:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688664935377664/t0(0) o400->0125589e-09b3-b36c-6e59-32b6a719c1ad@10.31.131.78@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513088.957930:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664935377792, offset 192 00000400:00000200:20.0:1616513088.957936:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.78@tcp1 00000400:00000200:20.0:1616513088.957942:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.78@tcp1 routed destination 00000100:00000200:24.0:1616513088.957943:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664935377664, offset 192 00000400:00000200:24.0:1616513088.957950:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.78@tcp1 00000400:00000200:20.0:1616513088.957958:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.78@tcp1(10.31.131.78@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513088.957964:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513088.957966:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.78@tcp1 routed destination 00000800:00000200:20.0:1616513088.957974:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513088.957977:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513088.957978:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513088.957981:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.78@tcp1(10.31.131.78@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513088.957982:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513088.957988:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513088.957988:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.957994:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.957997:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513088.958000:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513088.958001:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.958005:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513088.958007:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.958009:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513088.958011:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513088.958011:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513088.958012:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513088.958016:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513088.958020:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bba0 00000800:00000200:28.0:1616513088.958024:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513088.958063:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.958070:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.958073:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.958078:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.958084:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3870 00000800:00000200:26.0:1616513088.958087:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.958090:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.977549:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.977605:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.977614:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.977623:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.161@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.977633:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.161@tcp1 of length 224 into portal 26 MB=0x60419163e2c40 00000400:00000200:18.0:1616513088.977640:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.161@tcp1 of length 224/224 into md 0xf97ccc49 [1] + 0 00000400:00000200:18.0:1616513088.977644:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e440 00000400:00000200:18.0:1616513088.977647:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.977650:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.977654:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e440 00000800:00000200:18.0:1616513088.977657:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.977659:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.977660:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513088.977664:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.977718:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693355654130752 00010000:00000200:24.0:1616513088.977729:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1693355654130752/t0(0) o400->73047f8d-4c04-5724-c6e0-5a7bf21cdac7@10.31.129.161@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.977744:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693355654130752, offset 192 00000400:00000200:24.0:1616513088.977750:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.161@tcp1 00000400:00000200:24.0:1616513088.977758:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.161@tcp1 routed destination 00000400:00000200:24.0:1616513088.977773:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.161@tcp1(10.31.129.161@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.977780:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.977785:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.977788:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.977790:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.977795:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.977801:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.977840:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.977897:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513088.977903:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513088.977906:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513088.977913:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513088.977918:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f25d8 00000800:00000200:12.0:1616513088.977921:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.977924:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513088.978015:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513088.978069:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.978078:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513088.978084:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513088.978087:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.161@tcp1 : PUT - for me 00000400:00000200:20.0:1616513088.978096:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.161@tcp1 of length 224 into portal 12 MB=0x60419163e3080 00000400:00000200:20.0:1616513088.978103:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.161@tcp1 of length 224/224 into md 0x865be291 [1] + 7728 00000400:00000200:20.0:1616513088.978108:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513088.978111:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513088.978119:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513088.978122:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513088.978124:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513088.978133:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693355654131840 00010000:00000200:24.0:1616513088.978140:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad2d00 x1693355654131840/t0(0) o400->d7eb5453-3396-d0ec-cb97-f2a093415b0b@10.31.129.161@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513088.978152:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693355654131840, offset 192 00000400:00000200:24.0:1616513088.978157:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.161@tcp1 00000400:00000200:24.0:1616513088.978162:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.161@tcp1 routed destination 00000400:00000200:24.0:1616513088.978176:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.161@tcp1(10.31.129.161@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.978182:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.978186:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.978189:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513088.978191:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513088.978194:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.978200:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.978253:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.978260:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513088.978262:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.978269:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.978273:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2b28 00000800:00000200:26.0:1616513088.978277:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.978280:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513088.981887:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513088.981892:0:50433:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513088.981943:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.981952:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513088.981960:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.77@tcp1 : PUT - for me 00000800:00000200:14.0:1616513088.981961:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513088.981969:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.77@tcp1 of length 224 into portal 26 MB=0x5ffd434996100 00000800:00000200:14.0:1616513088.981971:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513088.981976:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.77@tcp1 of length 224/224 into md 0xf97ccc69 [1] + 0 00000400:00000200:14.0:1616513088.981980:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.77@tcp1 : PUT - for me 00000400:00000200:12.0:1616513088.981981:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e990 00000400:00000200:12.0:1616513088.981983:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513088.981986:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513088.981989:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.77@tcp1 of length 224 into portal 12 MB=0x5ffd434996480 00000400:00000200:12.0:1616513088.981990:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e990 00000800:00000200:12.0:1616513088.981993:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513088.981995:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.77@tcp1 of length 224/224 into md 0xf964d519 [1] + 75016 00000800:00000200:12.0:1616513088.981996:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513088.981998:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513088.982001:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513088.982001:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513088.982003:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513088.982009:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.982012:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513088.982014:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.982017:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.982055:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661764169984 00010000:00000200:24.0:1616513088.982066:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688661764169984/t0(0) o400->aa298490-0155-8d6b-23a8-adfe2abad6d4@10.31.133.77@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513088.982075:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661764170880 00010000:00000200:22.0:1616513088.982087:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01680 x1688661764170880/t0(0) o400->6171f751-7df6-13e8-a51b-af68471b80c8@10.31.133.77@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513088.982091:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661764169984, offset 192 00000400:00000200:24.0:1616513088.982097:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.77@tcp1 00000100:00000200:22.0:1616513088.982102:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661764170880, offset 192 00000400:00000200:24.0:1616513088.982105:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.77@tcp1 routed destination 00000400:00000200:22.0:1616513088.982108:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.77@tcp1 00000400:00000200:22.0:1616513088.982116:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.77@tcp1 routed destination 00000400:00000200:24.0:1616513088.982120:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.77@tcp1(10.31.133.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.982127:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513088.982131:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.77@tcp1(10.31.133.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.982133:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.982136:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.982138:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513088.982138:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513088.982143:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513088.982144:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.982146:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.982148:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513088.982148:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513088.982149:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513088.982153:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.982157:0:50433:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513088.982202:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513088.982209:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513088.982212:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513088.982218:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513088.982222:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513088.982224:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3cb0 00000800:00000200:28.0:1616513088.982227:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513088.982229:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513088.982230:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513088.982232:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513088.982238:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513088.982243:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a6e8 00000800:00000200:26.0:1616513088.982247:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513088.982249:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513088.992485:0:50434:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513088.992541:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513088.992550:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513088.992559:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.214@tcp1 : PUT - for me 00000400:00000200:18.0:1616513088.992569:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.214@tcp1 of length 224 into portal 26 MB=0x5ffd50de21740 00000400:00000200:18.0:1616513088.992578:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.214@tcp1 of length 224/224 into md 0xf97ccc79 [1] + 0 00000400:00000200:18.0:1616513088.992582:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e6e8 00000400:00000200:18.0:1616513088.992585:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513088.992588:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513088.992592:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:14.0:1616513088.992594:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.992595:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513088.992598:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513088.992600:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513088.992657:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665409591104 00010000:00000200:24.0:1616513088.992668:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688665409591104/t0(0) o400->99675499-149b-23b3-ec77-02f8364b99c9@10.31.132.214@tcp1:45/0 lens 224/224 e 0 to 0 dl 1616513180 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513088.992682:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665409591104, offset 192 00000400:00000200:24.0:1616513088.992689:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.214@tcp1 00000400:00000200:24.0:1616513088.992703:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.214@tcp1 routed destination 00000400:00000200:24.0:1616513088.992718:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.214@tcp1(10.31.132.214@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513088.992725:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513088.992730:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513088.992733:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513088.992735:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513088.992740:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513088.992746:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513088.992787:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513088.992842:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513088.992850:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513088.992852:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513088.992859:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513088.992864:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2770 00000800:00000200:12.0:1616513088.992868:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513088.992873:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513088.994584:0:50434:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513088.994640:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513088.994649:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513088.994658:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.214@tcp1 : PUT - for me 00000400:00000200:26.0:1616513088.994666:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.214@tcp1 of length 224 into portal 12 MB=0x5ffd50de21880 00000400:00000200:26.0:1616513088.994672:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.214@tcp1 of length 224/224 into md 0xf964d519 [1] + 75240 00000400:00000200:26.0:1616513088.994677:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513088.994680:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513088.994687:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513088.994690:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513088.994692:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513088.994695:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513088.994751:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665409591424 00010000:00000200:22.0:1616513088.994763:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691f80 x1688665409591424/t0(0) o400->93b50f0c-a293-1a8e-5976-cb51f0707421@10.31.132.214@tcp1:714/0 lens 224/224 e 0 to 0 dl 1616513094 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513088.994777:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665409591424, offset 192 00000400:00000200:22.0:1616513088.994784:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.214@tcp1 00000400:00000200:22.0:1616513088.994791:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.214@tcp1 routed destination 00000400:00000200:22.0:1616513088.994807:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.214@tcp1(10.31.132.214@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513088.994814:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513088.994820:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513088.994822:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513088.994824:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513088.994829:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513088.994833:0:50434:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513088.994889:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513088.994895:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513088.994898:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513088.994905:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513088.994908:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a440 00000800:00000200:14.0:1616513088.994912:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513088.994916:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.002667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.002722:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.002730:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.002741:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.209@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.002750:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.209@tcp1 of length 224 into portal 26 MB=0x6018781eebd00 00000400:00000200:26.0:1616513089.002756:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.209@tcp1 of length 224/224 into md 0xf97ccc89 [1] + 0 00000400:00000200:26.0:1616513089.002760:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777eff0 00000400:00000200:26.0:1616513089.002762:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.002765:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513089.002769:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eff0 00000800:00000200:26.0:1616513089.002772:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.002774:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.002776:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.002780:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.002831:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690531372383488 00010000:00000200:24.0:1616513089.002841:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1690531372383488/t0(0) o400->6f05dbd9-3ae3-78b3-d1d1-f6ef05d34e00@10.31.142.209@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.002855:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690531372383488, offset 192 00000400:00000200:24.0:1616513089.002860:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.209@tcp1 00000800:00000200:11.2:1616513089.002864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:24.0:1616513089.002867:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.209@tcp1 routed destination 00000400:00000200:24.0:1616513089.002883:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.209@tcp1(10.31.142.209@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.002889:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.002894:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.002897:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.002899:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.002903:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:24.0:1616513089.002910:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513089.002929:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.002936:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513089.002942:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.209@tcp1 : PUT - for me 00000400:00000200:24.0:1616513089.002949:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.209@tcp1 of length 224 into portal 12 MB=0x6018781eec880 00000400:00000200:24.0:1616513089.002954:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.209@tcp1 of length 224/224 into md 0x865be291 [1] + 7952 00000400:00000200:24.0:1616513089.002959:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513089.002961:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513089.002967:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513089.002969:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513089.002971:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513089.002973:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.002976:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513089.002978:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513089.002982:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513089.002983:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513089.002985:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3b18 00000800:00000200:24.0:1616513089.002988:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.002990:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690531372386432 00010000:00000200:0.0:1616513089.003002:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e3f00 x1690531372386432/t0(0) o400->26d28d76-9633-b37d-21bb-001454b5bf21@10.31.142.209@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.003015:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690531372386432, offset 192 00000400:00000200:0.0:1616513089.003021:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.209@tcp1 00000400:00000200:0.0:1616513089.003028:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.209@tcp1 routed destination 00000400:00000200:0.0:1616513089.003042:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.209@tcp1(10.31.142.209@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.003049:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.003054:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.003056:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.003058:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.003063:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.003068:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.003123:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.003130:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.003133:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513089.003140:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.003144:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42cc0 00000800:00000200:14.0:1616513089.003148:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.003151:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.023416:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.023472:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.023481:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.023491:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.129@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.023502:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.129@tcp1 of length 224 into portal 26 MB=0x605212442e2c0 00000400:00000200:16.0:1616513089.023509:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.129@tcp1 of length 224/224 into md 0xf97ccc99 [1] + 0 00000400:00000200:16.0:1616513089.023514:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fcb0 00000400:00000200:16.0:1616513089.023516:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.023519:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513089.023524:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fcb0 00000800:00000200:20.0:1616513089.023525:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.023527:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.023530:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.023532:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.023592:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694489760686784 00010000:00000200:24.0:1616513089.023602:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1694489760686784/t0(0) o400->09a2da09-179f-02c6-9e9a-f1d93c1aafb7@10.31.143.129@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.023615:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694489760686784, offset 192 00000400:00000200:24.0:1616513089.023621:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.129@tcp1 00000400:00000200:24.0:1616513089.023629:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.129@tcp1 routed destination 00000400:00000200:24.0:1616513089.023643:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.129@tcp1(10.31.143.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513089.023647:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.023649:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.023662:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.023665:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.023667:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.023672:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:24.0:1616513089.023679:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513089.023691:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.023696:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513089.023703:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.129@tcp1 : PUT - for me 00000400:00000200:24.0:1616513089.023710:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.129@tcp1 of length 224 into portal 12 MB=0x605212442ec00 00000400:00000200:24.0:1616513089.023715:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.129@tcp1 of length 224/224 into md 0x865be291 [1] + 8176 00000400:00000200:24.0:1616513089.023720:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513089.023722:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513089.023728:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513089.023730:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513089.023732:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513089.023734:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.023738:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513089.023739:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513089.023743:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513089.023744:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513089.023747:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3b18 00000800:00000200:24.0:1616513089.023750:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.023751:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694489760689152 00010000:00000200:0.0:1616513089.023763:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e2400 x1694489760689152/t0(0) o400->22e5f6d3-9e36-819b-b9cb-65e7ed795d4f@10.31.143.129@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.023778:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694489760689152, offset 192 00000400:00000200:0.0:1616513089.023784:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.129@tcp1 00000400:00000200:0.0:1616513089.023793:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.129@tcp1 routed destination 00000400:00000200:0.0:1616513089.023808:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.129@tcp1(10.31.143.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.023815:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.023821:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.023824:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.023826:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.023831:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.023837:0:50437:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.023893:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.023901:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.023903:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.023910:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.023914:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43540 00000800:00000200:2.0:1616513089.023918:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.023921:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.030405:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.030459:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.030468:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.030477:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.3@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.030485:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.3@tcp1 of length 224 into portal 26 MB=0x5ffd42d5d7b00 00000400:00000200:14.0:1616513089.030491:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.3@tcp1 of length 224/224 into md 0xf97ccca9 [1] + 0 00000400:00000200:14.0:1616513089.030496:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e7f8 00000400:00000200:14.0:1616513089.030498:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.030501:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.030505:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e7f8 00000800:00000200:14.0:1616513089.030508:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.030511:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.030512:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.030515:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.030568:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661642803968 00010000:00000200:24.0:1616513089.030579:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688661642803968/t0(0) o400->67b465ff-07e3-4bd8-a317-5f4ba3928ab5@10.31.133.3@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.030594:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661642803968, offset 192 00000400:00000200:24.0:1616513089.030600:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.3@tcp1 00000400:00000200:24.0:1616513089.030607:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.3@tcp1 routed destination 00000400:00000200:24.0:1616513089.030622:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.3@tcp1(10.31.133.3@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.030628:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.030634:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.030636:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.030646:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.030651:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.030658:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.030659:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.030714:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.030721:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.030724:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.030730:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.030736:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3b18 00000800:00000200:16.0:1616513089.030740:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.030742:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.030787:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513089.030842:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.030851:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513089.030860:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.3@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.030868:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.3@tcp1 of length 224 into portal 12 MB=0x5ffd42d5d7bc0 00000400:00000200:2.0:1616513089.030875:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.3@tcp1 of length 224/224 into md 0xf964d519 [1] + 75464 00000400:00000200:2.0:1616513089.030880:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.030883:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513089.030896:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.030897:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.030900:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.030902:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513089.030960:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661642804160 00010000:00000200:22.0:1616513089.030972:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53ccc80 x1688661642804160/t0(0) o400->f94d46c5-86fc-21ea-5884-eaed2c604053@10.31.133.3@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.030985:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661642804160, offset 192 00000400:00000200:22.0:1616513089.030992:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.3@tcp1 00000400:00000200:22.0:1616513089.030998:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.3@tcp1 routed destination 00000400:00000200:22.0:1616513089.031012:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.3@tcp1(10.31.133.3@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.031018:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.031023:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.031026:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.031028:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.031032:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.031037:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.031047:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.031051:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.031054:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.031056:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513089.031059:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.031063:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a220 00000800:00000200:14.0:1616513089.031066:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.038769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.038836:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.038846:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.038855:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.232@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.038864:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.232@tcp1 of length 224 into portal 26 MB=0x604a2957331c0 00000400:00000200:26.0:1616513089.038870:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.232@tcp1 of length 224/224 into md 0xf97cccc9 [1] + 0 00000400:00000200:26.0:1616513089.038875:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f5c8 00000400:00000200:26.0:1616513089.038878:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.038881:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513089.038885:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f5c8 00000800:00000200:26.0:1616513089.038888:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.038890:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.038892:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.038895:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.038900:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513089.038901:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.038907:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.232@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.038912:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.232@tcp1 of length 224 into portal 12 MB=0x604a295733280 00000400:00000200:26.0:1616513089.038917:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.232@tcp1 of length 224/224 into md 0xf964d519 [1] + 75688 00000400:00000200:26.0:1616513089.038921:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.038923:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513089.038928:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.038930:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.038934:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.038948:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946198831552 00010000:00000200:24.0:1616513089.038959:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1693946198831552/t0(0) o400->5478b6aa-d498-c075-91c5-c411322d32c1@10.31.140.232@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.038973:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946198831552, offset 192 00000400:00000200:24.0:1616513089.038979:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.232@tcp1 00000400:00000200:24.0:1616513089.038987:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.232@tcp1 routed destination 00000400:00000200:24.0:1616513089.039002:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.232@tcp1(10.31.140.232@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513089.039006:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946198831744 00000800:00000200:24.0:1616513089.039008:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.039014:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.039017:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.039019:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513089.039019:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693180 x1693946198831744/t0(0) o400->7ec469f8-287a-8ff3-9c58-5d4e28d2f576@10.31.140.232@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513089.039023:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.039030:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.039032:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513089.039033:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946198831744, offset 192 00000400:00000200:22.0:1616513089.039040:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.232@tcp1 00000400:00000200:22.0:1616513089.039047:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.232@tcp1 routed destination 00000400:00000200:22.0:1616513089.039062:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.232@tcp1(10.31.140.232@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.039069:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.039075:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.039078:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.039082:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.039087:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513089.039087:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513089.039092:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513089.039094:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.039097:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513089.039099:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513089.039103:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.039109:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3298 00000800:00000200:12.0:1616513089.039112:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513089.039146:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.039155:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.039158:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513089.039159:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513089.039164:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.039168:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b210 00000800:00000200:14.0:1616513089.039172:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513089.050573:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927552, portal 4 00000100:00000200:6.0:1616513089.050577:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927552, offset 0 00000400:00000200:6.0:1616513089.050582:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:6.0:1616513089.050587:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:6.0:1616513089.050595:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513089.050600:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:6.0:1616513089.050605:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513089.050607:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:6.0:1616513089.050608:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:6.0:1616513089.050613:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513089.050660:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.050681:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513089.050687:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513089.050690:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513089.050697:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513089.050710:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc53ce780 x1693067725927552/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513089.050722:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00990 00000800:00000200:27.0:1616513089.050727:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.050730:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513089.050834:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513089.050843:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.050850:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513089.050858:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513089.050867:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627480 00000400:00000200:27.0:1616513089.050874:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166f7d [1] + 192 00000400:00000200:27.0:1616513089.050879:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513089.050884:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc53ce780 x1693067725927552/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513089.050897:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.050899:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513089.050902:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.050904:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:6.0:1616513089.050906:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc53ce780 x1693067725927552/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:6.0:1616513089.050930:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc53ce780 x1693067725927552/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513089.050937:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:7.2:1616513089.072516:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.072572:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.072583:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.072592:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.175@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.072602:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.175@tcp1 of length 224 into portal 26 MB=0x604a2960a4e80 00000400:00000200:26.0:1616513089.072608:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.175@tcp1 of length 224/224 into md 0xf97ccce9 [1] + 0 00000400:00000200:26.0:1616513089.072613:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e3b8 00000400:00000200:26.0:1616513089.072616:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.072619:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513089.072623:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e3b8 00000800:00000200:26.0:1616513089.072626:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.072627:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.072629:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.072634:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.072637:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.072638:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.072647:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.175@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.072656:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.175@tcp1 of length 224 into portal 12 MB=0x604a2960a4f40 00000400:00000200:18.0:1616513089.072662:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.175@tcp1 of length 224/224 into md 0xf964d519 [1] + 75912 00000400:00000200:18.0:1616513089.072668:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.072671:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.072678:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.072682:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.072683:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.072691:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946208734848 00010000:00000200:24.0:1616513089.072702:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1693946208734848/t0(0) o400->84da1dde-53ca-4f97-35ca-058b996db06c@10.31.140.175@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.072716:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946208734848, offset 192 00000400:00000200:24.0:1616513089.072723:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.175@tcp1 00000400:00000200:24.0:1616513089.072730:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.175@tcp1 routed destination 00000400:00000200:24.0:1616513089.072744:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.175@tcp1(10.31.140.175@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513089.072745:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946208735040 00000800:00000200:24.0:1616513089.072751:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.072757:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.072759:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:22.0:1616513089.072759:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461209200 x1693946208735040/t0(0) o400->4aa41066-be7c-9a26-0886-6a8fea701d99@10.31.140.175@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513089.072762:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.072767:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513089.072773:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946208735040, offset 192 00000800:00000200:11.2:1616513089.072773:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.072774:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513089.072781:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.175@tcp1 00000400:00000200:22.0:1616513089.072788:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.175@tcp1 routed destination 00000400:00000200:22.0:1616513089.072802:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.175@tcp1(10.31.140.175@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.072809:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.072815:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.072818:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.072820:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.072831:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.072835:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513089.072838:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.072845:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.072848:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.072855:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.072860:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2ff0 00000800:00000200:16.0:1616513089.072864:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.072866:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.072873:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513089.072876:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.072877:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.072878:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.072881:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513089.072889:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.072897:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.072900:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513089.072901:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513089.072906:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.072910:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a7f8 00000800:00000200:14.0:1616513089.072914:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.078711:0:50443:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.078767:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.078777:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.078786:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.187@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.078796:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.187@tcp1 of length 224 into portal 26 MB=0x5f95b6a9ae680 00000400:00000200:18.0:1616513089.078802:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.187@tcp1 of length 224/224 into md 0xf97cccf9 [1] + 0 00000400:00000200:18.0:1616513089.078807:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ebb0 00000400:00000200:18.0:1616513089.078810:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.078812:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513089.078816:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ebb0 00000800:00000200:18.0:1616513089.078820:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.078822:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.078822:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513089.078829:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.078833:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513089.078834:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.078841:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.187@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.078851:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.187@tcp1 of length 224 into portal 12 MB=0x5f95b6a9ae700 00000400:00000200:2.0:1616513089.078857:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.187@tcp1 of length 224/224 into md 0xf964d519 [1] + 76136 00000400:00000200:2.0:1616513089.078862:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.078865:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513089.078872:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.078875:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.078877:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.078882:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681545909429888 00010000:00000200:24.0:1616513089.078893:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1681545909429888/t0(0) o400->2d2bb9a5-1266-e53f-ad08-88d375ccc3fa@10.31.138.187@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.078907:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681545909429888, offset 192 00000400:00000200:24.0:1616513089.078913:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.187@tcp1 00000400:00000200:24.0:1616513089.078921:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.187@tcp1 routed destination 00000400:00000200:24.0:1616513089.078936:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.187@tcp1(10.31.138.187@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513089.078936:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681545909430016 00000800:00000200:24.0:1616513089.078942:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.078948:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513089.078949:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cf980 x1681545909430016/t0(0) o400->86139fee-9994-41b9-c528-82408657ebd8@10.31.138.187@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513089.078951:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.078953:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.078958:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513089.078963:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681545909430016, offset 192 00000100:00000200:24.0:1616513089.078964:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.078965:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513089.078970:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.187@tcp1 00000400:00000200:22.0:1616513089.078978:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.187@tcp1 routed destination 00000400:00000200:22.0:1616513089.078992:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.187@tcp1(10.31.138.187@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.078999:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.079005:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.079008:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.079010:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.079015:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.079020:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513089.079022:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.079029:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.079032:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513089.079034:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513089.079038:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.079043:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3f58 00000800:00000200:12.0:1616513089.079047:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513089.079074:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.079082:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.079085:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.079086:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513089.079091:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.079095:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b8f8 00000800:00000200:14.0:1616513089.079099:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.079660:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.079714:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.079723:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.079732:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.49@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.079741:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.49@tcp1 of length 224 into portal 26 MB=0x5f909e3fa7bc0 00000400:00000200:16.0:1616513089.079748:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.49@tcp1 of length 224/224 into md 0xf97ccd09 [1] + 0 00000400:00000200:16.0:1616513089.079753:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e198 00000400:00000200:16.0:1616513089.079757:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.079760:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513089.079765:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e198 00000800:00000200:20.0:1616513089.079767:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.079768:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.079771:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513089.079775:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:24.0:1616513089.079777:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681195758418880 00000800:00000200:16.0:1616513089.079778:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.079779:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00010000:00000200:24.0:1616513089.079783:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1681195758418880/t0(0) o400->05d286d1-f696-6ccb-49f1-7c9b88c4f9d0@10.31.131.49@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513089.079787:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.49@tcp1 : PUT - for me 00000100:00000200:24.0:1616513089.079795:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681195758418880, offset 192 00000400:00000200:20.0:1616513089.079796:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.49@tcp1 of length 224 into portal 12 MB=0x5f909e3fa7c40 00000400:00000200:24.0:1616513089.079800:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.49@tcp1 00000400:00000200:20.0:1616513089.079805:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.49@tcp1 of length 224/224 into md 0x865be291 [1] + 8400 00000400:00000200:24.0:1616513089.079806:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.49@tcp1 routed destination 00000400:00000200:20.0:1616513089.079810:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.079813:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513089.079819:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.49@tcp1(10.31.131.49@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513089.079824:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.079825:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513089.079827:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.079829:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513089.079830:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.079832:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.079834:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.079837:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.079842:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.079842:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513089.079843:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681195758419008 00000800:00000200:12.0:1616513089.079851:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.079855:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:0.0:1616513089.079855:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad4800 x1681195758419008/t0(0) o400->58173656-5f98-512c-316f-9448fabb4ed8@10.31.131.49@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513089.079856:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513089.079859:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513089.079862:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.079865:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2e58 00000800:00000200:12.0:1616513089.079867:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.079870:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681195758419008, offset 192 00000400:00000200:0.0:1616513089.079876:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.49@tcp1 00000400:00000200:0.0:1616513089.079884:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.49@tcp1 routed destination 00000400:00000200:0.0:1616513089.079899:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.49@tcp1(10.31.131.49@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.079905:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.079911:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.079914:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.079916:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.079921:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.079927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.079937:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.079941:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.079943:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513089.079946:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.079949:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.079952:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42198 00000800:00000200:2.0:1616513089.079955:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.082240:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.082293:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.082302:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.082312:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.58@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.082321:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.58@tcp1 of length 224 into portal 26 MB=0x60196a49487c0 00000400:00000200:18.0:1616513089.082327:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.58@tcp1 of length 224/224 into md 0xf97ccd29 [1] + 0 00000400:00000200:18.0:1616513089.082332:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777edd0 00000400:00000200:18.0:1616513089.082335:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.082337:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513089.082341:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777edd0 00000800:00000200:18.0:1616513089.082344:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.082346:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.082347:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.082350:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.082404:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690596378183616 00010000:00000200:24.0:1616513089.082415:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1690596378183616/t0(0) o400->b863e479-2789-0f94-f853-9a05b40f3026@10.31.141.58@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.082429:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690596378183616, offset 192 00000400:00000200:24.0:1616513089.082450:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.58@tcp1 00000400:00000200:24.0:1616513089.082458:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.58@tcp1 routed destination 00000400:00000200:24.0:1616513089.082473:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.58@tcp1(10.31.141.58@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.082480:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.082486:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.082488:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.082490:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.082495:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.082501:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.082502:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513089.082517:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.082523:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.082525:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.082531:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.082537:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2110 00000800:00000200:28.0:1616513089.082540:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.082542:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.082547:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.082555:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.58@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.082562:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.58@tcp1 of length 224 into portal 12 MB=0x60196a4948980 00000400:00000200:28.0:1616513089.082568:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.58@tcp1 of length 224/224 into md 0x865be291 [1] + 8624 00000800:00000200:20.0:1616513089.082570:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513089.082573:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.082575:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.082580:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.082583:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.082585:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.082603:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690596378184064 00010000:00000200:0.0:1616513089.082615:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa1b00 x1690596378184064/t0(0) o400->02cd4f09-ad75-10d9-72a7-02de3df195f6@10.31.141.58@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.082630:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690596378184064, offset 192 00000400:00000200:0.0:1616513089.082636:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.58@tcp1 00000400:00000200:0.0:1616513089.082644:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.58@tcp1 routed destination 00000400:00000200:0.0:1616513089.082659:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.58@tcp1(10.31.141.58@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.082665:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.082671:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.082674:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.082678:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.082683:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.082687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.082742:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.082749:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.082752:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.082758:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.082762:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43d38 00000800:00000200:26.0:1616513089.082766:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.082769:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.096286:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513089.096341:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.096350:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.096359:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.161@tcp1 : PUT - for me 00000800:00000200:7.2:1616513089.096359:0:50445:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513089.096368:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.161@tcp1 of length 224 into portal 12 MB=0x5ffde79a45940 00000400:00000200:12.0:1616513089.096375:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.161@tcp1 of length 224/224 into md 0x865be291 [1] + 8848 00000400:00000200:12.0:1616513089.096381:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.096383:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513089.096390:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.096393:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513089.096394:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513089.096400:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513089.096417:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.096427:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.096452:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.161@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.096461:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.161@tcp1 of length 224 into portal 26 MB=0x5ffde79a45680 00000400:00000200:14.0:1616513089.096467:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.161@tcp1 of length 224/224 into md 0xf97ccd49 [1] + 0 00000400:00000200:14.0:1616513089.096472:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ea18 00000400:00000200:14.0:1616513089.096475:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513089.096477:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705872189760 00000100:00000200:14.0:1616513089.096478:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513089.096479:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513089.096482:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ea18 00000800:00000200:14.0:1616513089.096485:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.096488:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.096489:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513089.096490:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d203a80 x1688705872189760/t0(0) o400->27f2235a-8911-1d2a-887c-eefae61d8846@10.31.141.161@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.096504:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705872189760, offset 192 00000400:00000200:0.0:1616513089.096511:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.161@tcp1 00000400:00000200:0.0:1616513089.096518:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.161@tcp1 routed destination 00000400:00000200:0.0:1616513089.096533:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.161@tcp1(10.31.141.161@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.096540:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513089.096546:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.096548:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.096551:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.096555:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.096557:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705872189056 00000800:00000200:11.2:1616513089.096561:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513089.096568:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688705872189056/t0(0) o400->be6b4e79-067b-5482-eb16-2fd9719f612a@10.31.141.161@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.096582:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705872189056, offset 192 00000400:00000200:24.0:1616513089.096588:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.161@tcp1 00000400:00000200:24.0:1616513089.096596:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.161@tcp1 routed destination 00000400:00000200:24.0:1616513089.096611:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.161@tcp1(10.31.141.161@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513089.096614:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.096618:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513089.096620:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.096622:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513089.096624:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.096626:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513089.096628:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513089.096628:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513089.096632:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:24.0:1616513089.096633:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513089.096636:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513089.096637:0:50445:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.096638:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.096640:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513089.096693:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.096699:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.096702:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.096708:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.096713:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3d38 00000800:00000200:26.0:1616513089.096716:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.096719:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513089.098494:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927568, portal 4 00000100:00000200:28.0:1616513089.098499:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927568, offset 0 00000400:00000200:28.0:1616513089.098505:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:28.0:1616513089.098511:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:28.0:1616513089.098522:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513089.098528:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:28.0:1616513089.098534:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513089.098537:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:28.0:1616513089.098539:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:28.0:1616513089.098545:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513089.098598:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.098641:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513089.098647:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513089.098650:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513089.098657:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513089.098664:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb23b600 x1693067725927568/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513089.098676:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc6e8 00000800:00000200:27.0:1616513089.098680:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.098683:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513089.098756:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513089.098765:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.098772:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513089.098782:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513089.098789:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627490 00000400:00000200:27.0:1616513089.098796:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166f8d [1] + 192 00000400:00000200:27.0:1616513089.098801:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513089.098805:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb23b600 x1693067725927568/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513089.098817:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.098819:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513089.098822:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.098824:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:28.0:1616513089.098828:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb23b600 x1693067725927568/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513089.098840:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb23b600 x1693067725927568/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513089.098849:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc7f8 00000800:00000200:7.2:1616513089.101364:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.101417:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.101427:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.101449:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.137@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.101458:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.137@tcp1 of length 224 into portal 12 MB=0x5ffdde34e09c0 00000400:00000200:18.0:1616513089.101465:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.137@tcp1 of length 224/224 into md 0xf964d519 [1] + 76360 00000400:00000200:18.0:1616513089.101470:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.101471:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:18.0:1616513089.101473:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.101480:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.101483:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.101485:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513089.101501:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703349950912 00000800:00000200:11.2:1616513089.101510:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:22.0:1616513089.101513:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461208000 x1688703349950912/t0(0) o400->82b759b8-6dd9-6e96-9518-8f4a0a2181ba@10.31.141.137@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.101527:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703349950912, offset 192 00000400:00000200:22.0:1616513089.101536:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.137@tcp1 00000400:00000200:22.0:1616513089.101543:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.137@tcp1 routed destination 00000400:00000200:22.0:1616513089.101558:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.137@tcp1(10.31.141.137@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.101564:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513089.101565:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.101570:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.101572:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.101574:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513089.101575:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:22.0:1616513089.101579:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:20.0:1616513089.101586:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.137@tcp1 : PUT - for me 00000400:00000200:20.0:1616513089.101595:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.137@tcp1 of length 224 into portal 26 MB=0x5ffdde34e0940 00000400:00000200:20.0:1616513089.101602:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.137@tcp1 of length 224/224 into md 0xf97ccd59 [1] + 0 00000400:00000200:20.0:1616513089.101607:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f540 00000400:00000200:20.0:1616513089.101610:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.101612:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513089.101617:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f540 00000800:00000200:12.0:1616513089.101618:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513089.101620:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513089.101623:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:20.0:1616513089.101625:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513089.101625:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513089.101628:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513089.101628:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513089.101635:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.101641:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a770 00000800:00000200:12.0:1616513089.101645:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.101692:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703349950784 00010000:00000200:24.0:1616513089.101702:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688703349950784/t0(0) o400->56b7f908-2f04-32fc-6201-96745261fad2@10.31.141.137@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.101715:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703349950784, offset 192 00000400:00000200:24.0:1616513089.101721:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.137@tcp1 00000400:00000200:24.0:1616513089.101728:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.137@tcp1 routed destination 00000400:00000200:24.0:1616513089.101742:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.137@tcp1(10.31.141.137@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.101752:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513089.101757:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.101759:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513089.101761:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513089.101766:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.101770:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513089.101772:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513089.101823:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.101830:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.101832:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.101838:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.101845:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3c28 00000800:00000200:26.0:1616513089.101848:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.101851:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.103005:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513089.103030:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513089.103090:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513089.103090:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513089.103099:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513089.103100:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513089.103109:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.55@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.103109:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.55@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.103118:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.55@tcp1 of length 224 into portal 26 MB=0x5ffddf9bc4080 00000400:00000200:14.0:1616513089.103119:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.55@tcp1 of length 224 into portal 12 MB=0x5ffddf9bc43c0 00000400:00000200:16.0:1616513089.103124:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.55@tcp1 of length 224/224 into md 0xf97ccd69 [1] + 0 00000400:00000200:16.0:1616513089.103129:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e660 00000400:00000200:16.0:1616513089.103132:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513089.103134:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.55@tcp1 of length 224/224 into md 0xf964d519 [1] + 76584 00000100:00000200:16.0:1616513089.103139:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.103139:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.103149:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e660 00000100:00000200:14.0:1616513089.103151:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513089.103152:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.103155:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.103156:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513089.103157:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513089.103159:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513089.103160:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.103162:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.103165:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.103227:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703726272640 00000100:00000200:22.0:1616513089.103232:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703726273472 00010000:00000200:24.0:1616513089.103237:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688703726272640/t0(0) o400->9d0cf8dd-ed9e-0ccc-b36c-e36dd217ff18@10.31.142.55@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:22.0:1616513089.103244:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05a00 x1688703726273472/t0(0) o400->a1cc0864-34c6-a34d-0c2a-e253f3828b3d@10.31.142.55@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513089.103251:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703726272640, offset 192 00000400:00000200:24.0:1616513089.103256:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.55@tcp1 00000100:00000200:22.0:1616513089.103258:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703726273472, offset 192 00000400:00000200:24.0:1616513089.103263:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.55@tcp1 routed destination 00000400:00000200:22.0:1616513089.103264:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.55@tcp1 00000400:00000200:22.0:1616513089.103272:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.55@tcp1 routed destination 00000400:00000200:24.0:1616513089.103277:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.55@tcp1(10.31.142.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.103283:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513089.103286:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.55@tcp1(10.31.142.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.103288:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.103291:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.103293:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.103293:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513089.103297:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513089.103298:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.103301:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513089.103303:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513089.103303:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513089.103303:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.103308:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.103312:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513089.103357:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.103364:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.103367:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513089.103369:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513089.103374:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513089.103374:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.103379:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2f68 00000800:00000200:26.0:1616513089.103380:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.103382:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513089.103382:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513089.103389:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.103392:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b430 00000800:00000200:26.0:1616513089.103396:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.103398:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.107596:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.107649:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.107658:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.107666:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.136@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.107675:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.136@tcp1 of length 224 into portal 26 MB=0x602bfa431e8c0 00000400:00000200:18.0:1616513089.107681:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.136@tcp1 of length 224/224 into md 0xf97ccd89 [1] + 0 00000400:00000200:18.0:1616513089.107686:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ed48 00000400:00000200:18.0:1616513089.107688:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.107692:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513089.107696:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ed48 00000800:00000200:18.0:1616513089.107699:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.107702:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.107702:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.107706:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.107757:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691871977007296 00010000:00000200:24.0:1616513089.107767:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1691871977007296/t0(0) o400->d8db5cf9-ff43-a844-87b6-496a414db27b@10.31.142.136@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.107780:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691871977007296, offset 192 00000400:00000200:24.0:1616513089.107785:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.136@tcp1 00000400:00000200:24.0:1616513089.107793:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.136@tcp1 routed destination 00000400:00000200:24.0:1616513089.107806:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.136@tcp1(10.31.142.136@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.107812:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.107817:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.107820:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.107823:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.107828:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.107834:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.107846:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.107900:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.107906:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.107909:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.107916:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.107921:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3e48 00000800:00000200:16.0:1616513089.107925:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.107927:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.108443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.108495:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.108504:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.108512:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.136@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.108520:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.136@tcp1 of length 224 into portal 12 MB=0x602bfa431f700 00000400:00000200:28.0:1616513089.108526:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.136@tcp1 of length 224/224 into md 0x865be291 [1] + 9072 00000400:00000200:28.0:1616513089.108531:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.108533:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.108540:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.108542:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.108544:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.108547:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.108563:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691871977010944 00010000:00000200:0.0:1616513089.108575:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa3a80 x1691871977010944/t0(0) o400->3a8b71b0-388b-d8cd-6050-96d6b30ace88@10.31.142.136@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.108588:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691871977010944, offset 192 00000400:00000200:0.0:1616513089.108594:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.136@tcp1 00000400:00000200:0.0:1616513089.108601:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.136@tcp1 routed destination 00000400:00000200:0.0:1616513089.108614:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.136@tcp1(10.31.142.136@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.108621:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.108626:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.108629:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.108631:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.108635:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.108640:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.108699:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.108714:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.108716:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.108722:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.108726:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43210 00000800:00000200:26.0:1616513089.108729:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.108732:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.111195:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.111249:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.111258:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.111266:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.101@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.111275:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.101@tcp1 of length 224 into portal 26 MB=0x5ffd44429bfc0 00000400:00000200:14.0:1616513089.111281:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.101@tcp1 of length 224/224 into md 0xf97ccda9 [1] + 0 00000400:00000200:14.0:1616513089.111285:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f760 00000400:00000200:14.0:1616513089.111288:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.111290:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.111294:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f760 00000800:00000200:14.0:1616513089.111297:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.111300:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.111301:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513089.111307:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.111310:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.111311:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.111319:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.101@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.111327:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.101@tcp1 of length 224 into portal 12 MB=0x5ffd44429c040 00000400:00000200:18.0:1616513089.111333:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.101@tcp1 of length 224/224 into md 0xf964d519 [1] + 76808 00000400:00000200:18.0:1616513089.111338:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.111340:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.111346:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.111349:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.111351:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.111355:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662025289664 00010000:00000200:24.0:1616513089.111365:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688662025289664/t0(0) o400->e41072be-e6cf-df36-ae04-9a4f4d6abf3b@10.31.131.101@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.111378:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662025289664, offset 192 00000400:00000200:24.0:1616513089.111384:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.101@tcp1 00000400:00000200:24.0:1616513089.111391:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.101@tcp1 routed destination 00000400:00000200:24.0:1616513089.111405:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.101@tcp1(10.31.131.101@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513089.111410:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662025289792 00000800:00000200:24.0:1616513089.111411:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.111416:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.111418:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.111420:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513089.111422:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461209b00 x1688662025289792/t0(0) o400->a675eff5-3632-b19a-cd22-6e177157fa33@10.31.131.101@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513089.111425:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.111430:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.111431:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513089.111451:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662025289792, offset 192 00000400:00000200:22.0:1616513089.111458:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.101@tcp1 00000400:00000200:22.0:1616513089.111465:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.101@tcp1 routed destination 00000400:00000200:22.0:1616513089.111480:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.101@tcp1(10.31.131.101@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513089.111482:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513089.111486:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513089.111488:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.111491:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513089.111492:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.111496:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513089.111497:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513089.111498:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513089.111502:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3dc0 00000800:00000200:22.0:1616513089.111503:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513089.111505:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513089.111508:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513089.111510:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513089.111563:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.111570:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.111572:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.111574:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.111578:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.111582:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bdc0 00000800:00000200:2.0:1616513089.111586:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.116577:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.116632:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.116641:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.116649:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.159@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.116657:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.159@tcp1 of length 224 into portal 26 MB=0x603e15815e380 00000400:00000200:18.0:1616513089.116663:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.159@tcp1 of length 224/224 into md 0xf97ccdb9 [1] + 0 00000400:00000200:18.0:1616513089.116668:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4b28 00000400:00000200:18.0:1616513089.116670:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.116673:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513089.116677:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4b28 00000800:00000200:18.0:1616513089.116680:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.116682:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.116682:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.116686:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.116737:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116240618368 00010000:00000200:24.0:1616513089.116747:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1693116240618368/t0(0) o400->b6f71e92-112e-491e-54b3-8d5e911292af@10.31.136.159@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.116761:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693116240618368, offset 192 00000400:00000200:24.0:1616513089.116766:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.159@tcp1 00000400:00000200:24.0:1616513089.116773:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.159@tcp1 routed destination 00000400:00000200:24.0:1616513089.116787:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.159@tcp1(10.31.136.159@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.116793:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.116798:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.116801:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.116803:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.116807:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.116814:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.116854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.116910:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.116917:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.116919:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513089.116925:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.116930:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f37e8 00000800:00000200:12.0:1616513089.116933:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.116936:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.116955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.117008:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.117017:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.117024:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.159@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.117032:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.159@tcp1 of length 224 into portal 12 MB=0x603e15815e4c0 00000400:00000200:14.0:1616513089.117038:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.159@tcp1 of length 224/224 into md 0xf964d519 [1] + 77032 00000400:00000200:14.0:1616513089.117042:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.117045:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513089.117051:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.117054:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.117058:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.117060:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513089.117111:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116240618688 00010000:00000200:22.0:1616513089.117122:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05e80 x1693116240618688/t0(0) o400->acead5fa-7c10-625a-168b-3c37b61bd8e8@10.31.136.159@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.117136:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693116240618688, offset 192 00000400:00000200:22.0:1616513089.117142:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.159@tcp1 00000400:00000200:22.0:1616513089.117148:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.159@tcp1 routed destination 00000400:00000200:22.0:1616513089.117162:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.159@tcp1(10.31.136.159@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.117168:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.117174:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.117176:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.117178:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.117183:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.117188:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.117197:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.117202:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.117204:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.117206:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.117209:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.117213:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4af68 00000800:00000200:26.0:1616513089.117216:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.118685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.118736:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.118745:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.118753:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.72@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.118761:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.72@tcp1 of length 224 into portal 26 MB=0x60020d8f996c0 00000400:00000200:14.0:1616513089.118767:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.72@tcp1 of length 224/224 into md 0xf97ccdc9 [1] + 0 00000400:00000200:14.0:1616513089.118771:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5ed0 00000400:00000200:14.0:1616513089.118774:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.118776:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.118780:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5ed0 00000800:00000200:14.0:1616513089.118783:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.118786:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.118787:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.118790:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.118841:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688990939453120 00000800:00000200:7.2:1616513089.118849:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:24.0:1616513089.118851:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688990939453120/t0(0) o400->036091eb-4e56-62bb-ac3a-a5799f058939@10.31.129.72@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.118866:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688990939453120, offset 192 00000400:00000200:24.0:1616513089.118872:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.72@tcp1 00000400:00000200:24.0:1616513089.118879:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.72@tcp1 routed destination 00000400:00000200:24.0:1616513089.118892:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.72@tcp1(10.31.129.72@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.118898:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.118904:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.118906:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513089.118906:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513089.118908:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.118913:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513089.118915:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:24.0:1616513089.118919:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513089.118923:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.72@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.118931:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.72@tcp1 of length 224 into portal 12 MB=0x60020d8f99bc0 00000400:00000200:18.0:1616513089.118937:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.72@tcp1 of length 224/224 into md 0xf964d519 [1] + 77256 00000400:00000200:18.0:1616513089.118943:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.118945:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.118951:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.118954:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.118956:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.118959:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.118967:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513089.119013:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688990939454400 00010000:00000200:22.0:1616513089.119024:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461208900 x1688990939454400/t0(0) o400->abf1aa75-8cfd-24bc-bb3b-79d67d75ebd7@10.31.129.72@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513089.119025:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.119032:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.119034:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:22.0:1616513089.119038:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688990939454400, offset 192 00000400:00000200:16.0:1616513089.119040:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513089.119044:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.72@tcp1 00000400:00000200:16.0:1616513089.119046:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2660 00000800:00000200:16.0:1616513089.119049:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513089.119050:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.72@tcp1 routed destination 00000800:00000200:16.0:1616513089.119051:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513089.119064:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.72@tcp1(10.31.129.72@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.119070:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.119075:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.119078:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.119080:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.119084:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.119089:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.119100:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.119106:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.119109:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.119110:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.119115:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.119119:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:2.0:1616513089.119122:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.121705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513089.121759:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.121768:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.121776:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.220@tcp1 : PUT - for me 00000400:00000200:12.0:1616513089.121785:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.220@tcp1 of length 224 into portal 26 MB=0x5ffd503b4cf80 00000400:00000200:12.0:1616513089.121791:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.220@tcp1 of length 224/224 into md 0xf97ccde9 [1] + 0 00000400:00000200:12.0:1616513089.121795:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb42a8 00000400:00000200:12.0:1616513089.121798:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.121800:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513089.121804:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb42a8 00000800:00000200:12.0:1616513089.121807:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.121809:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.121811:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.121818:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.121821:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513089.121821:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513089.121829:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.220@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.121838:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.220@tcp1 of length 224 into portal 12 MB=0x5ffd503b4d000 00000400:00000200:28.0:1616513089.121843:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.220@tcp1 of length 224/224 into md 0x865be291 [1] + 9296 00000400:00000200:28.0:1616513089.121848:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.121851:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.121857:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.121860:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.121861:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.121863:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665238851456 00010000:00000200:24.0:1616513089.121872:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688665238851456/t0(0) o400->18be651b-19fd-bd5e-1745-446aef6350b4@10.31.132.220@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.121884:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665238851456, offset 192 00000400:00000200:24.0:1616513089.121889:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.220@tcp1 00000400:00000200:24.0:1616513089.121895:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.220@tcp1 routed destination 00000400:00000200:24.0:1616513089.121908:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.220@tcp1(10.31.132.220@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.121914:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.121919:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513089.121919:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665238851584 00000800:00000200:24.0:1616513089.121921:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.121923:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.121927:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:0.0:1616513089.121929:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5580 x1688665238851584/t0(0) o400->e1b73e31-7114-064a-70b7-f04fa8a93367@10.31.132.220@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513089.121932:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.121933:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513089.121942:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665238851584, offset 192 00000400:00000200:0.0:1616513089.121949:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.220@tcp1 00000400:00000200:0.0:1616513089.121955:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.220@tcp1 routed destination 00000400:00000200:0.0:1616513089.121968:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.220@tcp1(10.31.132.220@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.121974:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.121979:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.121981:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.121983:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513089.121986:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513089.121987:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.121992:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513089.121993:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.121996:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513089.121996:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513089.122002:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.122008:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3dc0 00000800:00000200:20.0:1616513089.122012:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513089.122043:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.122048:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.122051:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.122057:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.122060:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ff0 00000800:00000200:26.0:1616513089.122063:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.122066:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.123018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.123081:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.123090:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.123098:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.131@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.123107:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.131@tcp1 of length 224 into portal 26 MB=0x6019512534700 00000400:00000200:28.0:1616513089.123114:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.131@tcp1 of length 224/224 into md 0xf97ccdf9 [1] + 0 00000400:00000200:28.0:1616513089.123119:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5320 00000400:00000200:28.0:1616513089.123121:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.123124:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.123128:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5320 00000800:00000200:28.0:1616513089.123130:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.123133:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.123135:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.123139:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.123176:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513089.123217:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589629466368 00010000:00000200:24.0:1616513089.123226:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1690589629466368/t0(0) o400->1bea22d0-eef1-2028-ebdf-d439282c68d1@10.31.140.131@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513089.123229:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.123238:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:24.0:1616513089.123239:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690589629466368, offset 192 00000400:00000200:24.0:1616513089.123244:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.131@tcp1 00000400:00000200:12.0:1616513089.123246:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.131@tcp1 : PUT - for me 00000400:00000200:24.0:1616513089.123251:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.131@tcp1 routed destination 00000400:00000200:24.0:1616513089.123265:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.131@tcp1(10.31.140.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.123270:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.123273:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.131@tcp1 of length 224 into portal 12 MB=0x60195125348c0 00000800:00000200:24.0:1616513089.123275:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.123278:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.123279:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:12.0:1616513089.123279:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.131@tcp1 of length 224/224 into md 0x865be291 [1] + 9520 00000800:00000200:24.0:1616513089.123284:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:12.0:1616513089.123285:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.123288:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:24.0:1616513089.123289:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513089.123294:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513089.123297:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513089.123298:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:12.0:1616513089.123305:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513089.123306:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513089.123308:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.123309:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513089.123315:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.123321:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2f68 00000800:00000200:20.0:1616513089.123324:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.123351:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589629466816 00010000:00000200:0.0:1616513089.123361:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d204800 x1690589629466816/t0(0) o400->e1c3a6b3-a61e-cc45-2672-c2aaf1037a71@10.31.140.131@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.123373:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690589629466816, offset 192 00000400:00000200:0.0:1616513089.123378:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.131@tcp1 00000400:00000200:0.0:1616513089.123384:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.131@tcp1 routed destination 00000400:00000200:0.0:1616513089.123396:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.131@tcp1(10.31.140.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.123401:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.123406:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.123408:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.123410:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.123414:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.123429:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.123488:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.123494:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.123496:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513089.123502:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.123505:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a436d8 00000800:00000200:14.0:1616513089.123509:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.123511:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.132189:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.132241:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513089.132242:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513089.132249:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.132257:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.39@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.132266:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.39@tcp1 of length 224 into portal 12 MB=0x5ffdb348b9300 00000400:00000200:26.0:1616513089.132272:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.39@tcp1 of length 224/224 into md 0xf964d519 [1] + 77480 00000400:00000200:26.0:1616513089.132276:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.132278:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513089.132284:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.132286:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.132288:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.132290:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513089.132315:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.132324:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.132332:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.39@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.132340:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.39@tcp1 of length 224 into portal 26 MB=0x5ffdb348b9040 00000400:00000200:28.0:1616513089.132346:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.39@tcp1 of length 224/224 into md 0xf97cce09 [1] + 0 00000100:00000200:22.0:1616513089.132349:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691828036352 00000400:00000200:28.0:1616513089.132351:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4770 00000400:00000200:28.0:1616513089.132353:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.132356:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.132360:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4770 00010000:00000200:22.0:1616513089.132360:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691200 x1688691828036352/t0(0) o400->ec2ad5bd-c830-ddb4-61cf-862d6fb314bc@10.31.140.39@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513089.132363:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.132365:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513089.132367:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.132370:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513089.132384:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688691828036352, offset 192 00000400:00000200:22.0:1616513089.132389:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.39@tcp1 00000400:00000200:22.0:1616513089.132396:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.39@tcp1 routed destination 00000400:00000200:22.0:1616513089.132410:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.39@tcp1(10.31.140.39@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.132416:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513089.132419:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691828035648 00000800:00000200:22.0:1616513089.132421:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.132423:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513089.132425:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513089.132428:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688691828035648/t0(0) o400->e7f4a9e0-8c87-e8a3-91d3-84d9137ff294@10.31.140.39@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513089.132430:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.132438:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.132452:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688691828035648, offset 192 00000400:00000200:24.0:1616513089.132458:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.39@tcp1 00000400:00000200:24.0:1616513089.132464:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.39@tcp1 routed destination 00000400:00000200:24.0:1616513089.132477:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.39@tcp1(10.31.140.39@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.132483:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513089.132488:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.132490:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513089.132492:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513089.132496:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.132501:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513089.132502:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513089.132510:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.132517:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.132519:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.132519:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513089.132526:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513089.132526:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.132528:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.132530:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:16.0:1616513089.132534:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513089.132534:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513089.132537:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513089.132539:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3d38 00000800:00000200:14.0:1616513089.132542:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.132545:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.143196:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513089.143253:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.143262:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.143271:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.73@tcp1 : PUT - for me 00000400:00000200:12.0:1616513089.143279:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.73@tcp1 of length 224 into portal 26 MB=0x5ffd4d355fd40 00000400:00000200:12.0:1616513089.143286:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.73@tcp1 of length 224/224 into md 0xf97cce19 [1] + 0 00000400:00000200:12.0:1616513089.143291:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4ff0 00000400:00000200:12.0:1616513089.143293:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.143309:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513089.143313:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4ff0 00000800:00000200:12.0:1616513089.143315:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.143318:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513089.143318:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513089.143324:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513089.143326:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.143327:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.143335:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.73@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.143342:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.73@tcp1 of length 224 into portal 12 MB=0x5ffd4d355fdc0 00000400:00000200:28.0:1616513089.143347:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.73@tcp1 of length 224/224 into md 0x865be291 [1] + 9744 00000400:00000200:28.0:1616513089.143352:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.143354:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.143360:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.143363:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.143365:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.143370:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664427330880 00010000:00000200:24.0:1616513089.143378:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688664427330880/t0(0) o400->02fb6be6-3dd4-5b2e-0d67-0e4647a9726c@10.31.131.73@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.143390:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664427330880, offset 192 00000400:00000200:24.0:1616513089.143395:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.73@tcp1 00000400:00000200:24.0:1616513089.143400:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.73@tcp1 routed destination 00000400:00000200:24.0:1616513089.143412:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.73@tcp1(10.31.131.73@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.143418:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.143422:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513089.143423:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664427331008 00000800:00000200:24.0:1616513089.143426:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.143427:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.143432:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.143437:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.143447:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:0.0:1616513089.143448:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa0480 x1688664427331008/t0(0) o400->ffc1644f-4f30-bc0d-7d9d-18ce023d71e7@10.31.131.73@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.143460:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664427331008, offset 192 00000400:00000200:0.0:1616513089.143465:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.73@tcp1 00000400:00000200:0.0:1616513089.143471:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.73@tcp1 routed destination 00000400:00000200:0.0:1616513089.143483:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.73@tcp1(10.31.131.73@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.143488:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.143493:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513089.143495:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513089.143497:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.143498:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.143502:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513089.143503:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.143506:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513089.143506:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513089.143509:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513089.143512:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.143518:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2e58 00000800:00000200:20.0:1616513089.143521:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513089.143563:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.143570:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.143573:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.143580:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.143584:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43430 00000800:00000200:18.0:1616513089.143588:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.143591:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.152868:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.152938:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.152958:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.152966:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.36@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.152973:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.36@tcp1 of length 224 into portal 26 MB=0x6021520be1dc0 00000400:00000200:14.0:1616513089.152979:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.36@tcp1 of length 224/224 into md 0xf97cce49 [1] + 0 00000400:00000200:14.0:1616513089.152985:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb46e8 00000400:00000200:14.0:1616513089.152987:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.152990:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.152994:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb46e8 00000800:00000200:14.0:1616513089.152996:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.152999:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.153000:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.153003:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.153053:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691139627163072 00010000:00000200:24.0:1616513089.153062:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1691139627163072/t0(0) o400->cddc9582-e60f-0445-16b8-1aec21d18a45@10.31.141.36@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.153073:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691139627163072, offset 192 00000400:00000200:24.0:1616513089.153078:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.36@tcp1 00000400:00000200:24.0:1616513089.153084:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.36@tcp1 routed destination 00000400:00000200:24.0:1616513089.153096:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.36@tcp1(10.31.141.36@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.153102:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.153106:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.153108:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.153110:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.153114:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.153119:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.153120:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.153137:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.153144:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.153149:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.153156:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.153161:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2ff0 00000800:00000200:28.0:1616513089.153164:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.153167:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.153963:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513089.154019:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.154029:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.154038:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.36@tcp1 : PUT - for me 00000400:00000200:12.0:1616513089.154046:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.36@tcp1 of length 224 into portal 12 MB=0x6021520be2ac0 00000400:00000200:12.0:1616513089.154053:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.36@tcp1 of length 224/224 into md 0x865be291 [1] + 9968 00000400:00000200:12.0:1616513089.154058:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.154061:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513089.154068:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.154071:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513089.154073:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.154075:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.154134:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691139627166400 00010000:00000200:0.0:1616513089.154146:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d205100 x1691139627166400/t0(0) o400->5f9ec77e-3084-8285-b2a8-953845a5c88d@10.31.141.36@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.154161:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691139627166400, offset 192 00000400:00000200:0.0:1616513089.154167:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.36@tcp1 00000400:00000200:0.0:1616513089.154174:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.36@tcp1 routed destination 00000400:00000200:0.0:1616513089.154189:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.36@tcp1(10.31.141.36@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.154196:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.154201:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.154204:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.154206:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.154211:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.154216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.154269:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.154276:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.154279:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.154285:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.154289:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43cb0 00000800:00000200:18.0:1616513089.154293:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.154296:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.172265:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.172284:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.172293:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.172302:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.241@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.172313:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.241@tcp1 of length 224 into portal 26 MB=0x5f90ae9f6c180 00000400:00000200:14.0:1616513089.172319:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.241@tcp1 of length 224/224 into md 0xf97cce59 [1] + 0 00000400:00000200:14.0:1616513089.172324:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb53a8 00000400:00000200:14.0:1616513089.172327:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.172329:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.172334:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb53a8 00000800:00000200:14.0:1616513089.172337:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.172340:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.172340:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.172344:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.172398:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681200153805184 00010000:00000200:24.0:1616513089.172419:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1681200153805184/t0(0) o400->b3514223-f89d-9c1a-713d-bbeee18d6395@10.31.130.241@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513089.172425:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513089.172430:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681200153805184, offset 192 00000400:00000200:24.0:1616513089.172447:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.241@tcp1 00000400:00000200:24.0:1616513089.172453:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.241@tcp1 routed destination 00000400:00000200:24.0:1616513089.172465:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.241@tcp1(10.31.130.241@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.172471:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.172476:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.172478:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.172480:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.172484:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513089.172484:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513089.172490:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.172492:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513089.172494:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.172503:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.241@tcp1 : PUT - for me 00000800:00000200:28.0:1616513089.172509:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513089.172512:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.241@tcp1 of length 224 into portal 12 MB=0x5f90ae9f6c200 00000800:00000200:28.0:1616513089.172516:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.172518:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.172518:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.241@tcp1 of length 224/224 into md 0xf964d519 [1] + 77704 00000400:00000200:18.0:1616513089.172524:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.172525:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:18.0:1616513089.172527:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513089.172530:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3b18 00000800:00000200:28.0:1616513089.172534:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513089.172534:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513089.172537:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513089.172538:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.172539:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.172543:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513089.172598:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681200153805312 00010000:00000200:22.0:1616513089.172611:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120d100 x1681200153805312/t0(0) o400->b75cc7e7-19cf-2514-d8ed-3ad5bb1d0b8a@10.31.130.241@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.172624:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681200153805312, offset 192 00000400:00000200:22.0:1616513089.172630:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.241@tcp1 00000400:00000200:22.0:1616513089.172637:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.241@tcp1 routed destination 00000400:00000200:22.0:1616513089.172650:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.241@tcp1(10.31.130.241@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.172656:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.172662:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.172664:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.172666:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.172671:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.172686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.172697:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.172703:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.172705:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.172707:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.172712:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.172716:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4af68 00000800:00000200:2.0:1616513089.172720:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.173072:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.173083:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.173091:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513089.173093:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.173100:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.147@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.173110:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.147@tcp1 of length 224 into portal 12 MB=0x603e156884980 00000400:00000200:26.0:1616513089.173116:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.147@tcp1 of length 224/224 into md 0xf964d519 [1] + 77928 00000400:00000200:26.0:1616513089.173121:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.173124:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513089.173131:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.173134:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.173136:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513089.173142:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116214561152 00010000:00000200:22.0:1616513089.173148:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c690900 x1693116214561152/t0(0) o400->595842cc-67ab-f819-4746-6fac4d084b01@10.31.136.147@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.173160:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693116214561152, offset 192 00000400:00000200:22.0:1616513089.173165:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.147@tcp1 00000400:00000200:22.0:1616513089.173172:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.147@tcp1 routed destination 00000400:00000200:22.0:1616513089.173185:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.147@tcp1(10.31.136.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.173193:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513089.173197:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.173200:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513089.173202:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.173205:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.173210:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.173265:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.173272:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.173276:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513089.173283:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.173287:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bdc0 00000800:00000200:12.0:1616513089.173291:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.173294:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.174262:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.174332:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.174341:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.174350:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.147@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.174358:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.147@tcp1 of length 224 into portal 26 MB=0x603e156884800 00000400:00000200:14.0:1616513089.174365:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.147@tcp1 of length 224/224 into md 0xf97cce69 [1] + 0 00000400:00000200:14.0:1616513089.174369:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4bb0 00000400:00000200:14.0:1616513089.174372:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.174375:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.174379:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4bb0 00000800:00000200:14.0:1616513089.174381:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.174384:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.174386:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.174391:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.174459:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116214560768 00010000:00000200:24.0:1616513089.174470:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1693116214560768/t0(0) o400->7878b823-220c-8565-3b42-25a4743b9579@10.31.136.147@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.174484:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693116214560768, offset 192 00000400:00000200:24.0:1616513089.174490:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.147@tcp1 00000400:00000200:24.0:1616513089.174498:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.147@tcp1 routed destination 00000400:00000200:24.0:1616513089.174513:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.147@tcp1(10.31.136.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.174519:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513089.174525:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.174528:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513089.174530:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513089.174535:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.174540:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513089.174541:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513089.174592:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.174599:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.174602:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.174608:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.174616:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3cb0 00000800:00000200:18.0:1616513089.174619:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.174622:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.184388:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513089.184465:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.184474:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513089.184483:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.17@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.184492:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.17@tcp1 of length 224 into portal 26 MB=0x5ffe677b8bc00 00000400:00000200:2.0:1616513089.184499:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.17@tcp1 of length 224/224 into md 0xf97cce89 [1] + 0 00000400:00000200:2.0:1616513089.184503:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5d38 00000400:00000200:2.0:1616513089.184506:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.184509:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513089.184515:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5d38 00000800:00000200:14.0:1616513089.184518:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.184519:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.184521:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.184526:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.184529:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.184530:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.184538:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.17@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.184547:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.17@tcp1 of length 224 into portal 12 MB=0x5ffe677b8bc80 00000400:00000200:14.0:1616513089.184553:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.17@tcp1 of length 224/224 into md 0xf964d519 [1] + 78152 00000400:00000200:14.0:1616513089.184558:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.184561:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513089.184568:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.184571:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.184573:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.184579:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688740199709696 00010000:00000200:24.0:1616513089.184590:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688740199709696/t0(0) o400->a418598a-7f91-e049-adb2-af3d44974cd9@10.31.142.17@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.184604:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688740199709696, offset 192 00000400:00000200:24.0:1616513089.184610:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.17@tcp1 00000400:00000200:24.0:1616513089.184618:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.17@tcp1 routed destination 00000400:00000200:24.0:1616513089.184632:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.17@tcp1(10.31.142.17@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.184639:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.184644:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.184647:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513089.184648:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688740199709824 00000800:00000200:24.0:1616513089.184649:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.184654:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.184661:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:22.0:1616513089.184661:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06300 x1688740199709824/t0(0) o400->a7826536-7af3-cab0-0a88-8f0c9eba9428@10.31.142.17@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513089.184661:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513089.184675:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688740199709824, offset 192 00000400:00000200:22.0:1616513089.184682:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.17@tcp1 00000400:00000200:22.0:1616513089.184689:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.17@tcp1 routed destination 00000400:00000200:22.0:1616513089.184704:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.17@tcp1(10.31.142.17@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.184710:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513089.184716:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513089.184716:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.184719:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.184721:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.184723:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.184725:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513089.184725:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.184730:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513089.184732:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.184737:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f25d8 00000800:00000200:28.0:1616513089.184741:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513089.184741:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513089.184744:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513089.184747:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.184750:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513089.184751:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.184756:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.184759:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b430 00000800:00000200:26.0:1616513089.184763:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:21.0:1616513089.195912:0:6490:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927584, portal 4 00000100:00000200:21.0:1616513089.195919:0:6490:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927584, offset 0 00000400:00000200:21.0:1616513089.195926:0:6490:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:21.0:1616513089.195933:0:6490:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:21.0:1616513089.195946:0:6490:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:21.0:1616513089.195953:0:6490:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:21.0:1616513089.195960:0:6490:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513089.195963:0:6490:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:21.0:1616513089.195966:0:6490:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:21.0:1616513089.195971:0:6490:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513089.196018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.196038:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513089.196043:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513089.196046:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513089.196052:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513089.196059:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4558ad5580 x1693067725927584/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513089.196071:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51ed0 00000800:00000200:27.0:1616513089.196076:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.196078:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513089.196192:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513089.196202:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.196209:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513089.196217:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513089.196224:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6274a0 00000400:00000200:27.0:1616513089.196231:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166f9d [1] + 192 00000400:00000200:27.0:1616513089.196236:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513089.196240:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4558ad5580 x1693067725927584/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513089.196254:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.196254:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513089.196256:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513089.196259:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:21.0:1616513089.196262:0:6490:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4558ad5580 x1693067725927584/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:21.0:1616513089.196275:0:6490:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4558ad5580 x1693067725927584/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:21.0:1616513089.196285:0:6490:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51650 00000800:00000200:7.2:1616513089.216705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.216761:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.216771:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.216781:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.7@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.216790:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.7@tcp1 of length 224 into portal 26 MB=0x5ffd4838cc400 00000400:00000200:14.0:1616513089.216797:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.7@tcp1 of length 224/224 into md 0xf97cce99 [1] + 0 00000400:00000200:14.0:1616513089.216802:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5298 00000400:00000200:14.0:1616513089.216804:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.216807:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.216812:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5298 00000800:00000200:14.0:1616513089.216815:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.216817:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.216818:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.216824:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.216827:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.216828:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.216837:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.7@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.216846:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.7@tcp1 of length 224 into portal 12 MB=0x5ffd4838cc480 00000400:00000200:18.0:1616513089.216852:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.7@tcp1 of length 224/224 into md 0xf964d519 [1] + 78376 00000400:00000200:18.0:1616513089.216857:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.216860:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.216868:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.216871:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.216873:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.216876:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663088743424 00010000:00000200:24.0:1616513089.216888:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688663088743424/t0(0) o400->cdb64379-0818-2411-39c3-58ac9f2d31a1@10.31.138.7@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.216902:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663088743424, offset 192 00000400:00000200:24.0:1616513089.216910:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.7@tcp1 00000400:00000200:24.0:1616513089.216917:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.7@tcp1 routed destination 00000400:00000200:24.0:1616513089.216933:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.7@tcp1(10.31.138.7@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513089.216934:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663088743552 00000800:00000200:24.0:1616513089.216939:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.216957:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.216959:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:22.0:1616513089.216959:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120e780 x1688663088743552/t0(0) o400->2c84035b-ea49-6c52-6145-7157652c99f8@10.31.138.7@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513089.216961:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.216966:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.216972:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513089.216973:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663088743552, offset 192 00000800:00000200:11.2:1616513089.216973:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513089.216980:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.7@tcp1 00000400:00000200:22.0:1616513089.216986:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.7@tcp1 routed destination 00000400:00000200:22.0:1616513089.217000:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.7@tcp1(10.31.138.7@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.217006:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.217011:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.217014:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.217016:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.217021:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.217026:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513089.217027:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.217034:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.217037:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513089.217044:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.217049:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2440 00000800:00000200:12.0:1616513089.217053:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.217056:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.217062:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513089.217064:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.217066:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513089.217068:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.217070:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513089.217080:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.217087:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.217089:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.217091:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.217095:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.217099:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a770 00000800:00000200:26.0:1616513089.217103:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.248523:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.248579:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.248588:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.248597:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.112@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.248609:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.112@tcp1 of length 224 into portal 26 MB=0x60002866a2940 00000400:00000200:16.0:1616513089.248616:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.112@tcp1 of length 224/224 into md 0xf97cceb9 [1] + 0 00000400:00000200:16.0:1616513089.248621:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5870 00000400:00000200:16.0:1616513089.248624:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.248627:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513089.248631:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5870 00000800:00000200:12.0:1616513089.248634:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.248635:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.248638:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513089.248641:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.248644:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.248645:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.248654:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.112@tcp1 : PUT - for me 00000400:00000200:12.0:1616513089.248664:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.112@tcp1 of length 224 into portal 12 MB=0x60002866a29c0 00000400:00000200:12.0:1616513089.248671:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.112@tcp1 of length 224/224 into md 0x865be291 [1] + 10192 00000400:00000200:12.0:1616513089.248676:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.248679:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513089.248687:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.248690:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.248692:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.248709:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688860705302848 00010000:00000200:24.0:1616513089.248720:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688860705302848/t0(0) o400->e550e619-0edd-a4e1-581d-57a0d846ed88@10.31.138.112@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.248735:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688860705302848, offset 192 00000400:00000200:24.0:1616513089.248741:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.112@tcp1 00000400:00000200:24.0:1616513089.248749:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.112@tcp1 routed destination 00000100:00000200:0.0:1616513089.248751:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688860705302976 00010000:00000200:0.0:1616513089.248763:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d202400 x1688860705302976/t0(0) o400->fb3971f9-a11e-675d-09f6-6103855e98bc@10.31.138.112@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513089.248765:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.112@tcp1(10.31.138.112@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.248772:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:0.0:1616513089.248777:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688860705302976, offset 192 00000800:00000200:24.0:1616513089.248778:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.248780:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.248783:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513089.248783:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.112@tcp1 00000800:00000200:24.0:1616513089.248788:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513089.248791:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.112@tcp1 routed destination 00000800:00000200:11.2:1616513089.248793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.248795:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513089.248805:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.112@tcp1(10.31.138.112@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.248811:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.248817:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.248820:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.248822:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.248827:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.248834:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513089.248847:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.248854:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.248858:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.248859:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513089.248866:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.248871:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3ba0 00000800:00000200:20.0:1616513089.248875:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513089.248891:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.248898:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.248901:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.248907:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.248912:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a434b8 00000800:00000200:18.0:1616513089.248916:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.248918:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.257446:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513089.257500:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.257510:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.257519:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.163@tcp1 : PUT - for me 00000400:00000200:12.0:1616513089.257528:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.163@tcp1 of length 224 into portal 12 MB=0x5ffd48ae32700 00000400:00000200:12.0:1616513089.257535:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.163@tcp1 of length 224/224 into md 0x865be291 [1] + 10416 00000400:00000200:12.0:1616513089.257540:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.257544:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513089.257551:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.257554:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.257555:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513089.257562:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.257565:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513089.257565:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513089.257574:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.38@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.257583:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.38@tcp1 of length 224 into portal 26 MB=0x6018671672800 00000400:00000200:28.0:1616513089.257590:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.38@tcp1 of length 224/224 into md 0xf97ccee9 [1] + 0 00000400:00000200:28.0:1616513089.257594:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb58f8 00000400:00000200:28.0:1616513089.257597:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.257600:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.257604:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb58f8 00000800:00000200:28.0:1616513089.257607:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.257610:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.257612:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.257620:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513089.257626:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663211845376 00010000:00000200:0.0:1616513089.257638:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d201b00 x1688663211845376/t0(0) o400->54c3923e-e5a3-e916-0719-df7783de2a6c@10.31.136.163@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.257664:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663211845376, offset 192 00000100:00000200:24.0:1616513089.257667:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690526800095232 00000400:00000200:0.0:1616513089.257670:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.163@tcp1 00010000:00000200:24.0:1616513089.257678:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1690526800095232/t0(0) o400->336312e3-15e2-420d-ad59-218cd04f6004@10.31.140.38@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513089.257678:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.163@tcp1 routed destination 00000800:00000200:14.0:1616513089.257679:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.257689:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000100:00000200:24.0:1616513089.257693:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690526800095232, offset 192 00000400:00000200:0.0:1616513089.257693:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.163@tcp1(10.31.136.163@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513089.257698:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.163@tcp1 : PUT - for me 00000400:00000200:24.0:1616513089.257699:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.38@tcp1 00000800:00000200:0.0:1616513089.257699:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513089.257706:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.38@tcp1 routed destination 00000400:00000200:14.0:1616513089.257708:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.163@tcp1 of length 224 into portal 26 MB=0x5ffd48ae32680 00000800:00000200:0.0:1616513089.257714:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513089.257715:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.163@tcp1 of length 224/224 into md 0xf97ccef9 [1] + 0 00000800:00000200:0.0:1616513089.257717:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513089.257719:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5c28 00000800:00000200:0.0:1616513089.257719:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513089.257722:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.38@tcp1(10.31.140.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513089.257722:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513089.257724:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:14.0:1616513089.257727:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513089.257729:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513089.257729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513089.257731:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5c28 00000800:00000200:14.0:1616513089.257734:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513089.257736:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513089.257737:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513089.257737:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513089.257739:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513089.257741:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (24)++ 00000800:00000200:14.0:1616513089.257744:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (25)-- 00000800:00000200:24.0:1616513089.257746:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513089.257747:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513089.257748:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:7.2:1616513089.257751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:24.0:1616513089.257753:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:4.0:1616513089.257755:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663211845248 00000400:00000200:26.0:1616513089.257759:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.38@tcp1 : PUT - for me 00010000:00000200:4.0:1616513089.257766:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688663211845248/t0(0) o400->ba36fcb2-55ea-de0c-c874-0ea074ee3359@10.31.136.163@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513089.257768:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.38@tcp1 of length 224 into portal 12 MB=0x6018671672880 00000400:00000200:26.0:1616513089.257774:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.38@tcp1 of length 224/224 into md 0xf964d519 [1] + 78600 00000400:00000200:26.0:1616513089.257779:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513089.257781:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663211845248, offset 192 00000100:00000200:26.0:1616513089.257782:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513089.257783:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:4.0:1616513089.257787:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.163@tcp1 00000800:00000200:26.0:1616513089.257790:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:20.0:1616513089.257790:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513089.257793:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:20.0:1616513089.257793:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513089.257794:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513089.257794:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.163@tcp1 routed destination 00000800:00000200:26.0:1616513089.257795:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.257797:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513089.257799:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513089.257802:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.257804:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513089.257804:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42550 00000800:00000200:14.0:1616513089.257807:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.257810:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:4.0:1616513089.257810:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.163@tcp1(10.31.136.163@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513089.257815:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2088 00000800:00000200:26.0:1616513089.257818:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513089.257820:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:4.0:1616513089.257821:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.257827:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.257830:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.257832:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.257836:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.257841:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.257843:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513089.257853:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:22.0:1616513089.257857:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690526800095360 00000800:00000200:16.0:1616513089.257860:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.257863:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513089.257864:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513089.257869:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691680 x1690526800095360/t0(0) o400->72864200-6cbb-8b8f-c355-4594d21b7f9d@10.31.140.38@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513089.257869:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.257878:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba5d8 00000800:00000200:16.0:1616513089.257881:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513089.257884:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690526800095360, offset 192 00000400:00000200:22.0:1616513089.257890:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.38@tcp1 00000400:00000200:22.0:1616513089.257898:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.38@tcp1 routed destination 00000400:00000200:22.0:1616513089.257913:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.38@tcp1(10.31.140.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.257919:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.257925:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.257928:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.257930:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.257935:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.257940:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.257948:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.257952:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.257955:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.257958:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513089.257960:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.257964:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b8f8 00000800:00000200:14.0:1616513089.257967:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513089.290615:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927600, portal 4 00000100:00000200:30.0:1616513089.290621:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927600, offset 0 00000400:00000200:30.0:1616513089.290627:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513089.290633:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513089.290645:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513089.290651:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513089.290657:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513089.290660:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513089.290662:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513089.290668:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513089.290730:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:7.0:1616513089.290766:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927616, portal 4 00000100:00000200:7.0:1616513089.290772:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927616, offset 0 00000400:00000200:7.0:1616513089.290779:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:7.0:1616513089.290785:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:12.0:1616513089.290787:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513089.290793:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513089.290796:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:7.0:1616513089.290798:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513089.290803:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:7.0:1616513089.290805:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000100:00000200:12.0:1616513089.290810:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc53cc380 x1693067725927600/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:7.0:1616513089.290812:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513089.290815:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:7.0:1616513089.290817:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:12.0:1616513089.290822:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee990 00000800:00000200:7.0:1616513089.290823:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:12.0:1616513089.290826:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513089.290829:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.2:1616513089.290869:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:5.2:1616513089.290869:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513089.290886:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513089.290888:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513089.290893:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513089.290895:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513089.290896:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:3.0:1616513089.290898:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513089.290905:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.290908:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000100:00000200:27.0:1616513089.290911:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc567080 x1693067725927616/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513089.290916:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6274b0 00000400:00000200:28.0:1616513089.290923:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d9f41 [1] + 192 00000400:00000200:27.0:1616513089.290923:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637ecc0 00000800:00000200:27.0:1616513089.290927:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:28.0:1616513089.290928:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.290933:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc53cc380 x1693067725927600/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513089.290940:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513089.290948:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513089.290951:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513089.290953:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513089.290955:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc53cc380 x1693067725927600/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513089.290967:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc53cc380 x1693067725927600/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513089.290976:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efcb0 00000800:00000200:27.2:1616513089.290995:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513089.291005:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.291011:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513089.291015:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513089.291019:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513089.291027:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6274c0 00000400:00000200:27.0:1616513089.291033:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166fad [1] + 192 00000400:00000200:27.0:1616513089.291038:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513089.291043:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc567080 x1693067725927616/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513089.291056:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.291059:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.291061:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:7.0:1616513089.291061:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc567080 x1693067725927616/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513089.291070:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc567080 x1693067725927616/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513089.291078:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f7e8 00000800:00000200:11.2:1616513089.296793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513089.296848:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.296857:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.296866:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.78@tcp1 : PUT - for me 00000400:00000200:12.0:1616513089.296876:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.78@tcp1 of length 224 into portal 26 MB=0x6018736d78240 00000800:00000200:7.2:1616513089.296878:0:50465:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513089.296883:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.78@tcp1 of length 224/224 into md 0xf97ccf09 [1] + 0 00000400:00000200:12.0:1616513089.296888:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb54b8 00000400:00000200:12.0:1616513089.296891:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.296894:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513089.296899:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb54b8 00000800:00000200:20.0:1616513089.296900:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.296902:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.296905:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.296907:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513089.296935:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.296945:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513089.296955:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.78@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.296964:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.78@tcp1 of length 224 into portal 12 MB=0x6018736d78600 00000100:00000200:4.0:1616513089.296970:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690530112569920 00000400:00000200:2.0:1616513089.296971:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.78@tcp1 of length 224/224 into md 0xf964d519 [1] + 78824 00000400:00000200:2.0:1616513089.296976:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.296979:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:4.0:1616513089.296982:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1690530112569920/t0(0) o400->2205e343-df10-e11d-f954-0ef98d35f491@10.31.132.78@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513089.296986:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.296989:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.296989:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.296993:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513089.296997:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690530112569920, offset 192 00000400:00000200:4.0:1616513089.297003:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.78@tcp1 00000400:00000200:4.0:1616513089.297010:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.78@tcp1 routed destination 00000400:00000200:4.0:1616513089.297026:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.78@tcp1(10.31.132.78@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.297033:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.297039:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.297042:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.297044:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.297050:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.297055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.297057:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513089.297059:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690530112570880 00010000:00000200:22.0:1616513089.297072:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c9680 x1690530112570880/t0(0) o400->12994cf6-3835-41c5-2795-c85e65b9cac4@10.31.132.78@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.297087:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690530112570880, offset 192 00000400:00000200:22.0:1616513089.297093:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.78@tcp1 00000400:00000200:22.0:1616513089.297100:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.78@tcp1 routed destination 00000800:00000200:16.0:1616513089.297112:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513089.297115:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.78@tcp1(10.31.132.78@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513089.297119:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513089.297122:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513089.297122:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513089.297128:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513089.297128:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513089.297131:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.297133:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513089.297134:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6babb0 00000800:00000200:22.0:1616513089.297138:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513089.297138:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.297141:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.297142:0:50465:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.297200:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.297207:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.297210:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.297216:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.297223:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a7f8 00000800:00000200:18.0:1616513089.297227:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.297229:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.297564:0:50465:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.297576:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.297584:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513089.297586:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.297593:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.115@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.297602:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.115@tcp1 of length 224 into portal 26 MB=0x5ffd733841e40 00000400:00000200:26.0:1616513089.297608:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.115@tcp1 of length 224/224 into md 0xf97ccf19 [1] + 0 00000400:00000200:26.0:1616513089.297613:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5430 00000400:00000200:26.0:1616513089.297616:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.297618:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513089.297623:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5430 00000800:00000200:26.0:1616513089.297626:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.297629:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.297630:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513089.297637:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674630901312 00010000:00000200:4.0:1616513089.297643:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688674630901312/t0(0) o400->32abb343-571f-bbf6-9357-235bb08afdf0@10.31.132.115@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.297653:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688674630901312, offset 192 00000400:00000200:4.0:1616513089.297659:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.115@tcp1 00000400:00000200:4.0:1616513089.297665:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.115@tcp1 routed destination 00000400:00000200:4.0:1616513089.297678:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.115@tcp1(10.31.132.115@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.297684:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.297688:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.297691:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.297694:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.297697:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:4.0:1616513089.297702:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.297703:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513089.297705:0:50465:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513089.297714:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513089.297716:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513089.297719:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.297722:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.297722:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:12.0:1616513089.297723:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513089.297725:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513089.297729:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.297730:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.115@tcp1 : PUT - for me 00000400:00000200:20.0:1616513089.297734:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb5c8 00000800:00000200:20.0:1616513089.297738:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513089.297738:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.115@tcp1 of length 224 into portal 12 MB=0x5ffd7338422c0 00000400:00000200:18.0:1616513089.297744:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.115@tcp1 of length 224/224 into md 0xf964d519 [1] + 79048 00000400:00000200:18.0:1616513089.297751:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.297754:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.297761:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.297763:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.297765:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513089.297773:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674630902464 00010000:00000200:22.0:1616513089.297779:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120a880 x1688674630902464/t0(0) o400->0fe8c420-b835-2579-ffbb-34f27591b4bb@10.31.132.115@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.297790:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688674630902464, offset 192 00000400:00000200:22.0:1616513089.297795:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.115@tcp1 00000400:00000200:22.0:1616513089.297801:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.115@tcp1 routed destination 00000400:00000200:22.0:1616513089.297816:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.115@tcp1(10.31.132.115@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.297822:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.297827:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.297829:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.297831:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.297834:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.297838:0:50465:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.297847:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.297851:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.297853:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513089.297855:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.297859:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.297862:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b210 00000800:00000200:2.0:1616513089.297866:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.300051:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.300105:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.300114:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.300122:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.201@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.300133:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.201@tcp1 of length 224 into portal 26 MB=0x5ffd4d5158f40 00000400:00000200:16.0:1616513089.300139:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.201@tcp1 of length 224/224 into md 0xf97ccf39 [1] + 0 00000400:00000200:16.0:1616513089.300144:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5b18 00000400:00000200:16.0:1616513089.300147:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.300149:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513089.300154:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5b18 00000800:00000200:16.0:1616513089.300157:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.300158:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.300160:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.300161:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513089.300178:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664456662848 00010000:00000200:4.0:1616513089.300189:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688664456662848/t0(0) o400->2b805b54-e7dd-5dc1-29ab-9b498b7ce103@10.31.136.201@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513089.300203:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:4.0:1616513089.300203:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664456662848, offset 192 00000400:00000200:4.0:1616513089.300209:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.201@tcp1 00000400:00000200:4.0:1616513089.300218:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.201@tcp1 routed destination 00000400:00000200:4.0:1616513089.300232:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.201@tcp1(10.31.136.201@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.300239:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.300245:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.300247:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.300250:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.300254:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513089.300257:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513089.300260:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.300261:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513089.300267:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.300276:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.201@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.300284:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.201@tcp1 of length 224 into portal 12 MB=0x5ffd4d5159000 00000400:00000200:14.0:1616513089.300291:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.201@tcp1 of length 224/224 into md 0xf964d519 [1] + 79272 00000400:00000200:14.0:1616513089.300296:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.300299:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513089.300305:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.300308:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.300310:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513089.300312:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513089.300313:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513089.300318:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.300321:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513089.300328:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.300333:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bae58 00000800:00000200:12.0:1616513089.300336:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.300341:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513089.300387:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664456663040 00010000:00000200:22.0:1616513089.300399:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06780 x1688664456663040/t0(0) o400->e40123ef-803c-ed24-c81a-f397489e6051@10.31.136.201@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.300414:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664456663040, offset 192 00000400:00000200:22.0:1616513089.300420:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.201@tcp1 00000400:00000200:22.0:1616513089.300428:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.201@tcp1 routed destination 00000400:00000200:22.0:1616513089.300454:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.201@tcp1(10.31.136.201@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.300461:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.300467:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.300470:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.300472:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.300477:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.300482:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.300536:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.300542:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.300545:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.300551:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.300556:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a220 00000800:00000200:26.0:1616513089.300561:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.300564:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:31.2:1616513089.302465:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:14.0:1616513089.302522:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:14.0:1616513089.302531:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:14.0:1616513089.302539:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513089.302548:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 224 into portal 24 MB=0x603db0154c920 00000400:00000200:14.0:1616513089.302555:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 18 from 12345-10.242.46.234@o2ib1 of length 224/224 into md 0xf973ab29 [1] + 0 00000400:00000200:14.0:1616513089.302559:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82baa4b100 00000400:00000200:14.0:1616513089.302562:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.302565:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt_out 00000400:00000200:14.0:1616513089.302573:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b100 00000800:00000200:18.0:1616513089.302575:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:14.0:1616513089.302577:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:14.0:1616513089.302580:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:14.0:1616513089.302582:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:22.0:1616513089.302639:0:185125:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693089015318816 00010000:00000200:22.0:1616513089.302662:0:185125:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74da74f850 x1693089015318816/t0(0) o41->boslfs02-MDT0000-mdtlov_UUID@10.242.46.234@o2ib1:715/0 lens 224/368 e 0 to 0 dl 1616513095 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513089.302676:0:185125:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 4, xid 1693089015318816, offset 192 00000400:00000200:22.0:1616513089.302682:0:185125:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:22.0:1616513089.302690:0:185125:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:22.0:1616513089.302703:0:185125:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.302709:0:185125:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:22.0:1616513089.302715:0:185125:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.302718:0:185125:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:22.0:1616513089.302722:0:185125:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:22.0:1616513089.302727:0:185125:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513089.302732:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:22.0:1616513089.302734:0:185125:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000800:00000200:26.0:1616513089.302744:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:26.0:1616513089.302751:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:26.0:1616513089.302754:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.302760:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.302766:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a440 00000800:00000200:26.0:1616513089.302769:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:26.0:1616513089.302772:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:11.2:1616513089.310456:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.310512:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.310521:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.310530:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.79@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.310539:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.79@tcp1 of length 224 into portal 26 MB=0x60214f35a5e40 00000400:00000200:28.0:1616513089.310546:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.79@tcp1 of length 224/224 into md 0xf97ccf59 [1] + 0 00000400:00000200:28.0:1616513089.310551:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb43b8 00000400:00000200:28.0:1616513089.310554:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.310556:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.310561:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb43b8 00000800:00000200:28.0:1616513089.310564:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.310566:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.310567:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.310570:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513089.310580:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691138865651264 00010000:00000200:4.0:1616513089.310591:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1691138865651264/t0(0) o400->d7e3d514-bfef-5f6e-69f4-179b7d0f86d0@10.31.141.79@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.310605:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691138865651264, offset 192 00000400:00000200:4.0:1616513089.310611:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.79@tcp1 00000400:00000200:4.0:1616513089.310619:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.79@tcp1 routed destination 00000400:00000200:4.0:1616513089.310634:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.79@tcp1(10.31.141.79@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.310641:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.310647:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.310649:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.310652:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.310656:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.310661:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.310662:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513089.310664:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513089.310715:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.310722:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.310725:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513089.310731:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513089.310734:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513089.310737:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000800:00000200:12.0:1616513089.310740:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513089.310743:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:12.0:1616513089.310743:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513089.310752:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.79@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.310761:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.79@tcp1 of length 224 into portal 12 MB=0x60214f35a5f40 00000400:00000200:18.0:1616513089.310767:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.79@tcp1 of length 224/224 into md 0xf964d519 [1] + 79496 00000400:00000200:18.0:1616513089.310772:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.310775:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.310781:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.310784:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.310786:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.310789:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513089.310844:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691138865651520 00010000:00000200:22.0:1616513089.310856:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120f500 x1691138865651520/t0(0) o400->ceaa8688-13f2-1606-b355-2e1da212d5ad@10.31.141.79@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.310870:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691138865651520, offset 192 00000400:00000200:22.0:1616513089.310876:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.79@tcp1 00000400:00000200:22.0:1616513089.310883:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.79@tcp1 routed destination 00000400:00000200:22.0:1616513089.310898:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.79@tcp1(10.31.141.79@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.310904:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.310910:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.310913:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.310915:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.310919:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.310925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.310977:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.310984:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.310987:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.310993:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.310997:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a088 00000800:00000200:26.0:1616513089.311001:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.311004:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.318997:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.319052:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.319061:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513089.319067:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513089.319070:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.211@tcp1 : PUT - for me 00000800:00000200:18.0:1616513089.319077:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.319079:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.211@tcp1 of length 224 into portal 26 MB=0x601a15c652b80 00000400:00000200:18.0:1616513089.319086:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.211@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.319086:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.211@tcp1 of length 224/224 into md 0xf97ccf79 [1] + 0 00000400:00000200:14.0:1616513089.319091:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5210 00000400:00000200:14.0:1616513089.319093:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.319095:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.211@tcp1 of length 224 into portal 12 MB=0x601a15c652c40 00000100:00000200:14.0:1616513089.319096:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.319101:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5210 00000800:00000200:14.0:1616513089.319104:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000400:00000200:18.0:1616513089.319105:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.211@tcp1 of length 224/224 into md 0xf964d519 [1] + 79720 00000800:00000200:14.0:1616513089.319106:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513089.319108:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.319111:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.319111:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:18.0:1616513089.319113:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.319120:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.319123:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.319125:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513089.319125:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690642411760512 00010000:00000200:4.0:1616513089.319136:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1690642411760512/t0(0) o400->fe3fd9c7-dd6c-f5e7-6fc3-ea8cae992fde@10.31.136.211@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.319152:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690642411760512, offset 192 00000400:00000200:4.0:1616513089.319159:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.211@tcp1 00000400:00000200:4.0:1616513089.319166:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.211@tcp1 routed destination 00000400:00000200:4.0:1616513089.319181:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.211@tcp1(10.31.136.211@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.319187:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.319193:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:22.0:1616513089.319194:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690642411760704 00000800:00000200:4.0:1616513089.319196:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.319198:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.319203:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513089.319206:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120b180 x1690642411760704/t0(0) o400->19cf7ade-f9cd-ca4e-6672-9c4d4fb2d287@10.31.136.211@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513089.319210:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.319210:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513089.319221:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690642411760704, offset 192 00000400:00000200:22.0:1616513089.319227:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.211@tcp1 00000400:00000200:22.0:1616513089.319234:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.211@tcp1 routed destination 00000400:00000200:22.0:1616513089.319249:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.211@tcp1(10.31.136.211@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.319255:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.319261:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.319263:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513089.319264:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513089.319266:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.319270:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513089.319271:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.319274:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513089.319275:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.319280:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513089.319281:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.319293:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba000 00000800:00000200:16.0:1616513089.319296:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513089.319336:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.319343:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.319346:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.319347:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.319352:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.319356:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ab28 00000800:00000200:2.0:1616513089.319359:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.323214:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.323266:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.323275:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.323283:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.39@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.323292:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.39@tcp1 of length 224 into portal 26 MB=0x5ffd3d5936c80 00000400:00000200:26.0:1616513089.323298:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.39@tcp1 of length 224/224 into md 0xf97ccf89 [1] + 0 00000400:00000200:26.0:1616513089.323303:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5078 00000400:00000200:26.0:1616513089.323305:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.323308:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513089.323311:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5078 00000800:00000200:26.0:1616513089.323314:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.323317:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.323318:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.323322:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513089.323373:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660169944192 00010000:00000200:4.0:1616513089.323383:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688660169944192/t0(0) o400->4c8172eb-13cf-262d-0101-bcaad9d7d188@10.31.129.39@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513089.323396:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:4.0:1616513089.323396:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660169944192, offset 192 00000400:00000200:4.0:1616513089.323402:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.39@tcp1 00000400:00000200:4.0:1616513089.323409:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.39@tcp1 routed destination 00000400:00000200:4.0:1616513089.323423:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.39@tcp1(10.31.129.39@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.323429:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.323434:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.323437:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.323455:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513089.323459:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513089.323459:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:4.0:1616513089.323466:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513089.323467:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.323475:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.39@tcp1 : PUT - for me 00000800:00000200:11.2:1616513089.323477:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513089.323483:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.39@tcp1 of length 224 into portal 12 MB=0x5ffd3d5937080 00000400:00000200:14.0:1616513089.323489:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.39@tcp1 of length 224/224 into md 0xf964d519 [1] + 79944 00000400:00000200:14.0:1616513089.323494:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.323496:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513089.323502:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.323505:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.323507:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.323509:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513089.323552:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.323558:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.323561:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:22.0:1616513089.323565:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660169945216 00000400:00000200:12.0:1616513089.323567:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.323572:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baf68 00000800:00000200:12.0:1616513089.323575:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513089.323577:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03600 x1688660169945216/t0(0) o400->9f5ba660-876c-617d-48ff-8525b218c3e1@10.31.129.39@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513089.323578:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513089.323590:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660169945216, offset 192 00000400:00000200:22.0:1616513089.323597:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.39@tcp1 00000400:00000200:22.0:1616513089.323605:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.39@tcp1 routed destination 00000400:00000200:22.0:1616513089.323618:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.39@tcp1(10.31.129.39@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.323624:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.323630:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.323632:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.323634:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.323639:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.323644:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.323653:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.323658:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.323661:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513089.323662:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.323667:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.323671:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b3a8 00000800:00000200:18.0:1616513089.323676:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.335182:0:50469:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.335239:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.335260:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.335269:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.152@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.335278:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.152@tcp1 of length 224 into portal 26 MB=0x603e41f81bf80 00000400:00000200:14.0:1616513089.335286:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.152@tcp1 of length 224/224 into md 0xf97ccfb9 [1] + 0 00000400:00000200:14.0:1616513089.335291:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5650 00000400:00000200:14.0:1616513089.335294:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.335296:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513089.335297:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513089.335300:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5650 00000800:00000200:14.0:1616513089.335303:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513089.335306:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513089.335307:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513089.335308:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.335310:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.335316:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.152@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.335325:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.152@tcp1 of length 224 into portal 12 MB=0x603e41f81c040 00000400:00000200:2.0:1616513089.335332:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.152@tcp1 of length 224/224 into md 0xf964d519 [1] + 80168 00000400:00000200:2.0:1616513089.335339:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.335342:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513089.335348:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.335352:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.335354:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513089.335364:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693128176287616 00010000:00000200:4.0:1616513089.335374:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1693128176287616/t0(0) o400->734d7e7f-20a8-dde6-831e-175dde1b2792@10.31.138.152@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.335387:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693128176287616, offset 192 00000400:00000200:4.0:1616513089.335393:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.152@tcp1 00000400:00000200:4.0:1616513089.335400:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.152@tcp1 routed destination 00000400:00000200:4.0:1616513089.335415:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.152@tcp1(10.31.138.152@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.335421:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:22.0:1616513089.335427:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693128176287808 00000800:00000200:4.0:1616513089.335427:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.335429:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.335431:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.335449:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513089.335452:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cec00 x1693128176287808/t0(0) o400->b5ad09d8-387c-b61e-34ca-50eb29236fda@10.31.138.152@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513089.335455:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.335455:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513089.335466:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693128176287808, offset 192 00000400:00000200:22.0:1616513089.335472:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.152@tcp1 00000400:00000200:22.0:1616513089.335479:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.152@tcp1 routed destination 00000400:00000200:22.0:1616513089.335493:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.152@tcp1(10.31.138.152@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.335499:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.335504:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.335507:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.335509:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.335510:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513089.335513:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513089.335517:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513089.335518:0:50469:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513089.335520:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.335526:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.335532:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bab28 00000800:00000200:28.0:1616513089.335535:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.335538:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.335543:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.335546:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.335548:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.335551:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.335554:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513089.335601:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.335608:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.335611:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.335612:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.335617:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.335621:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b7e8 00000800:00000200:18.0:1616513089.335625:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.338434:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513089.338488:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.338497:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513089.338506:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.58@tcp1 : PUT - for me 00000400:00000200:20.0:1616513089.338516:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.58@tcp1 of length 224 into portal 26 MB=0x603e1a9662e00 00000400:00000200:20.0:1616513089.338522:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.58@tcp1 of length 224/224 into md 0xf97ccfc9 [1] + 0 00000400:00000200:20.0:1616513089.338527:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4f68 00000400:00000200:20.0:1616513089.338530:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:7.0:1616513089.338533:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927632, portal 4 00000100:00000200:20.0:1616513089.338534:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513089.338538:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4f68 00000100:00000200:7.0:1616513089.338538:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927632, offset 0 00000800:00000200:20.0:1616513089.338541:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.338542:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.338544:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:7.0:1616513089.338545:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:20.0:1616513089.338549:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513089.338552:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.338553:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:7.0:1616513089.338553:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000100:00000200:4.0:1616513089.338562:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693117604834816 00000400:00000200:28.0:1616513089.338564:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.58@tcp1 : PUT - for me 00000400:00000200:7.0:1616513089.338567:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513089.338574:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.58@tcp1 of length 224 into portal 12 MB=0x603e1a9662f40 00000800:00000200:7.0:1616513089.338574:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00010000:00000200:4.0:1616513089.338574:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1693117604834816/t0(0) o400->bb14d053-0f52-3091-583d-5783c412abf4@10.31.136.58@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513089.338580:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.58@tcp1 of length 224/224 into md 0x865be291 [1] + 10640 00000800:00000200:7.0:1616513089.338581:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513089.338585:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000400:00000200:28.0:1616513089.338586:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:7.0:1616513089.338587:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000100:00000200:4.0:1616513089.338588:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693117604834816, offset 192 00000100:00000200:28.0:1616513089.338589:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.0:1616513089.338592:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:28.0:1616513089.338596:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:4.0:1616513089.338598:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.58@tcp1 00000800:00000200:28.0:1616513089.338600:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.338602:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:4.0:1616513089.338606:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.58@tcp1 routed destination 00000400:00000200:4.0:1616513089.338621:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.58@tcp1(10.31.136.58@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.338628:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.338633:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.338636:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:25.2:1616513089.338638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:4.0:1616513089.338638:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.338643:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.338648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.338650:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513089.338663:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693117604835136 00010000:00000200:0.0:1616513089.338675:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa3600 x1693117604835136/t0(0) o400->cfa25527-17a4-de25-e0c8-770c172de659@10.31.136.58@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.338690:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693117604835136, offset 192 00000800:00000200:2.0:1616513089.338696:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000400:00000200:0.0:1616513089.338696:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.58@tcp1 00000800:00000200:2.0:1616513089.338703:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:0.0:1616513089.338703:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.58@tcp1 routed destination 00000400:00000200:2.0:1616513089.338706:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.338713:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:12.0:1616513089.338715:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513089.338719:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.58@tcp1(10.31.136.58@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:2.0:1616513089.338720:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456e9b00 x1693067725927632/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513089.338722:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.338725:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513089.338726:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513089.338726:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513089.338731:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.338732:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637fe48 00000800:00000200:0.0:1616513089.338732:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.338734:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513089.338736:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:12.0:1616513089.338737:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb320 00000800:00000200:0.0:1616513089.338737:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.338739:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:12.0:1616513089.338740:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513089.338741:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.338748:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.338763:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.338782:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.338785:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.338792:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.338796:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42330 00000800:00000200:26.0:1616513089.338801:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.338804:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:25.2:1616513089.338811:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513089.338823:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513089.338830:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:26.0:1616513089.338832:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513089.338838:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513089.338845:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6274d0 00000400:00000200:2.0:1616513089.338852:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cd689 [1] + 192 00000400:00000200:2.0:1616513089.338857:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.338861:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456e9b00 x1693067725927632/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513089.338875:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513089.338878:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513089.338879:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:7.0:1616513089.338882:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456e9b00 x1693067725927632/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513089.338892:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456e9b00 x1693067725927632/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513089.338900:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f7e8 00000800:00000200:7.2:1616513089.384831:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.384889:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.384900:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.384910:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.37@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.384920:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.37@tcp1 of length 224 into portal 12 MB=0x5ffd435acb140 00000400:00000200:26.0:1616513089.384927:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.37@tcp1 of length 224/224 into md 0xf964d519 [1] + 80392 00000400:00000200:26.0:1616513089.384932:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:11.2:1616513089.384933:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513089.384935:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513089.384943:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.384946:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.384947:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.384950:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513089.385000:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.385010:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.385019:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.37@tcp1 : PUT - for me 00000800:00000200:28.0:1616513089.385023:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513089.385024:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661782212928 00000400:00000200:16.0:1616513089.385028:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.37@tcp1 of length 224 into portal 26 MB=0x5ffd435acaf80 00000400:00000200:16.0:1616513089.385035:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.37@tcp1 of length 224/224 into md 0xf97ccfd9 [1] + 0 00010000:00000200:22.0:1616513089.385038:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695e80 x1688661782212928/t0(0) o400->12dbde4d-2f09-944b-a4b8-07870b055fe6@10.31.138.37@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513089.385042:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb57e8 00000400:00000200:16.0:1616513089.385045:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.385048:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513089.385053:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb57e8 00000100:00000200:22.0:1616513089.385056:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661782212928, offset 192 00000800:00000200:16.0:1616513089.385056:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.385059:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.385061:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513089.385062:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.37@tcp1 00000400:00000200:22.0:1616513089.385070:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.37@tcp1 routed destination 00000400:00000200:22.0:1616513089.385086:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.37@tcp1(10.31.138.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.385092:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513089.385098:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.385103:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513089.385105:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.385111:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:4.0:1616513089.385115:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661782212480 00000800:00000200:11.2:1616513089.385116:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:4.0:1616513089.385125:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688661782212480/t0(0) o400->12c4a25b-e6c6-287d-5932-68d6d42334b7@10.31.138.37@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.385138:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661782212480, offset 192 00000400:00000200:4.0:1616513089.385143:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.37@tcp1 00000400:00000200:4.0:1616513089.385150:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.37@tcp1 routed destination 00000400:00000200:4.0:1616513089.385163:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.37@tcp1(10.31.138.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.385169:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513089.385174:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.385178:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513089.385179:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513089.385180:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513089.385185:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513089.385186:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.385189:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513089.385190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513089.385191:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513089.385198:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.385202:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a198 00000800:00000200:20.0:1616513089.385206:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.385209:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513089.385256:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.385264:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.385267:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513089.385273:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.385279:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb3a8 00000800:00000200:14.0:1616513089.385282:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.385285:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513089.386531:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927648, portal 4 00000100:00000200:23.0:1616513089.386536:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927648, offset 0 00000400:00000200:23.0:1616513089.386545:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:23.0:1616513089.386552:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:23.0:1616513089.386565:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513089.386572:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:23.0:1616513089.386578:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513089.386581:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:23.0:1616513089.386583:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:23.0:1616513089.386589:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513089.386635:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513089.386696:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513089.386703:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513089.386706:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.386713:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513089.386719:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600900 x1693067725927648/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513089.386731:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f430 00000800:00000200:28.0:1616513089.386735:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513089.386739:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513089.386825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513089.386879:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513089.386888:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513089.386897:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513089.386905:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6274e0 00000400:00000200:20.0:1616513089.386912:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866d9ff1 [1] + 192 00000400:00000200:20.0:1616513089.386917:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.386922:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600900 x1693067725927648/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513089.386931:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513089.386937:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513089.386940:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513089.386942:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:23.0:1616513089.386946:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600900 x1693067725927648/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513089.386960:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600900 x1693067725927648/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513089.386970:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f4b8 00000800:00000200:11.2:1616513089.398476:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.398532:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.398541:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.398550:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.21@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.398559:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.21@tcp1 of length 224 into portal 26 MB=0x5ffd3b62cb240 00000400:00000200:28.0:1616513089.398566:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.21@tcp1 of length 224/224 into md 0xf97cd009 [1] + 0 00000400:00000200:28.0:1616513089.398570:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5cb0 00000400:00000200:28.0:1616513089.398573:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.398576:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.398580:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5cb0 00000800:00000200:28.0:1616513089.398583:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.398586:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.398586:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.398590:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513089.398601:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659643118144 00010000:00000200:4.0:1616513089.398612:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688659643118144/t0(0) o400->e56c1e50-d945-cdd5-4a92-11fe39ea1df3@10.31.138.21@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.398627:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688659643118144, offset 192 00000400:00000200:4.0:1616513089.398633:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.21@tcp1 00000400:00000200:4.0:1616513089.398641:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.21@tcp1 routed destination 00000400:00000200:4.0:1616513089.398656:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.21@tcp1(10.31.138.21@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.398663:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.398669:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.398672:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.398674:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.398679:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.398684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.398685:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513089.398739:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.398745:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.398748:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513089.398754:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.398760:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbdc0 00000800:00000200:12.0:1616513089.398763:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.398766:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.398900:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.398911:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.398919:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.398928:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.21@tcp1 : PUT - for me 00000800:00000200:28.0:1616513089.398931:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513089.398937:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.21@tcp1 of length 224 into portal 12 MB=0x5ffd3b62cb3c0 00000400:00000200:16.0:1616513089.398943:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.21@tcp1 of length 224/224 into md 0x865be291 [1] + 10864 00000400:00000200:16.0:1616513089.398949:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.398951:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513089.398959:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.398962:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.398964:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.399024:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659643118528 00010000:00000200:0.0:1616513089.399036:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd0900 x1688659643118528/t0(0) o400->f9534722-6fb1-6f96-5164-75f28abff443@10.31.138.21@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.399051:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688659643118528, offset 192 00000400:00000200:0.0:1616513089.399057:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.21@tcp1 00000400:00000200:0.0:1616513089.399065:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.21@tcp1 routed destination 00000400:00000200:0.0:1616513089.399079:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.21@tcp1(10.31.138.21@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.399088:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.399093:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.399096:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.399098:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.399103:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.399110:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.399166:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.399173:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.399175:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.399182:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.399186:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a08 00000800:00000200:18.0:1616513089.399190:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.399193:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.437729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513089.437786:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513089.437793:0:50480:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513089.437805:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.437814:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.91@tcp1 : PUT - for me 00000400:00000200:12.0:1616513089.437825:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.91@tcp1 of length 224 into portal 12 MB=0x5ffd5b28b7c00 00000400:00000200:12.0:1616513089.437831:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.91@tcp1 of length 224/224 into md 0x865be291 [1] + 11088 00000400:00000200:12.0:1616513089.437842:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.437845:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513089.437852:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.437856:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513089.437858:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.437862:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513089.437863:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.437873:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.437882:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.91@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.437893:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.91@tcp1 of length 224 into portal 26 MB=0x5ffd5b28b7980 00000400:00000200:14.0:1616513089.437900:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.91@tcp1 of length 224/224 into md 0xf97cd019 [1] + 0 00000400:00000200:14.0:1616513089.437904:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb5a90 00000400:00000200:14.0:1616513089.437908:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.437910:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.437915:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb5a90 00000800:00000200:14.0:1616513089.437919:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:0.0:1616513089.437919:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668172155904 00000800:00000200:14.0:1616513089.437922:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.437922:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.437926:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513089.437933:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d204c80 x1688668172155904/t0(0) o400->91705222-a733-a68e-66b6-3aad6a59fc8a@10.31.136.91@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.437948:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668172155904, offset 192 00000400:00000200:0.0:1616513089.437954:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.91@tcp1 00000400:00000200:0.0:1616513089.437962:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.91@tcp1 routed destination 00000400:00000200:0.0:1616513089.437978:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.91@tcp1(10.31.136.91@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.437984:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513089.437990:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:4.0:1616513089.437991:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668172155264 00000800:00000200:0.0:1616513089.437993:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.437995:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.438001:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:4.0:1616513089.438002:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688668172155264/t0(0) o400->10b3eae7-64a7-4621-d829-79bdc630dd32@10.31.136.91@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513089.438006:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.438016:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668172155264, offset 192 00000400:00000200:4.0:1616513089.438021:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.91@tcp1 00000400:00000200:4.0:1616513089.438029:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.91@tcp1 routed destination 00000400:00000200:4.0:1616513089.438043:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.91@tcp1(10.31.136.91@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.438049:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513089.438055:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.438058:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513089.438060:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.438063:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513089.438065:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513089.438070:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513089.438070:0:50480:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513089.438071:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513089.438073:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.438079:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.438084:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43320 00000800:00000200:28.0:1616513089.438088:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.438091:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513089.438137:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.438145:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.438148:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513089.438151:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.438155:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.438160:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00000800:00000200:18.0:1616513089.438164:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.440533:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513089.440586:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.440595:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513089.440604:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.103@tcp1 : PUT - for me 00000400:00000200:20.0:1616513089.440613:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.103@tcp1 of length 224 into portal 26 MB=0x6000b1b065a00 00000400:00000200:20.0:1616513089.440619:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.103@tcp1 of length 224/224 into md 0xf97cd029 [1] + 0 00000400:00000200:20.0:1616513089.440624:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4990 00000400:00000200:20.0:1616513089.440627:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.440630:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513089.440634:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4990 00000800:00000200:20.0:1616513089.440637:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.440640:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513089.440640:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513089.440644:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.440655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:4.0:1616513089.440659:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688897558305280 00000800:00000200:28.0:1616513089.440670:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:4.0:1616513089.440670:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688897558305280/t0(0) o400->6650d1fa-37ca-1558-5afb-9ef84990557d@10.31.138.103@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513089.440679:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:4.0:1616513089.440684:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688897558305280, offset 192 00000400:00000200:28.0:1616513089.440688:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.103@tcp1 : PUT - for me 00000400:00000200:4.0:1616513089.440690:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.103@tcp1 00000400:00000200:4.0:1616513089.440699:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.103@tcp1 routed destination 00000400:00000200:4.0:1616513089.440714:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.103@tcp1(10.31.138.103@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.440721:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.440722:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.103@tcp1 of length 224 into portal 12 MB=0x6000b1b065a80 00000800:00000200:16.0:1616513089.440727:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513089.440727:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513089.440728:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.103@tcp1 of length 224/224 into md 0x865be291 [1] + 11312 00000800:00000200:4.0:1616513089.440730:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513089.440732:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513089.440734:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.440736:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513089.440737:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:11.2:1616513089.440742:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.440743:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:4.0:1616513089.440744:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513089.440746:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513089.440748:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.440751:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.440755:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513089.440755:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.440757:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.440763:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.440767:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb298 00000800:00000200:28.0:1616513089.440770:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.440807:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688897558305408 00010000:00000200:0.0:1616513089.440819:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa4800 x1688897558305408/t0(0) o400->e01beb4c-9028-6b8c-560f-8aba4f053d1a@10.31.138.103@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.440834:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688897558305408, offset 192 00000400:00000200:0.0:1616513089.440840:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.103@tcp1 00000400:00000200:0.0:1616513089.440847:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.103@tcp1 routed destination 00000400:00000200:0.0:1616513089.440862:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.103@tcp1(10.31.138.103@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.440868:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.440874:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.440877:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.440879:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.440884:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.440892:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.440947:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.440954:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.440957:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.440963:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.440967:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42e58 00000800:00000200:2.0:1616513089.440971:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.440974:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.441782:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.441793:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.441800:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513089.441801:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513089.441809:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.232@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.441819:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.232@tcp1 of length 224 into portal 12 MB=0x603e0f3779740 00000400:00000200:16.0:1616513089.441825:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.232@tcp1 of length 224/224 into md 0x865be291 [1] + 11536 00000400:00000200:16.0:1616513089.441830:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.441833:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513089.441840:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.441842:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.441844:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.441851:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114552522560 00010000:00000200:0.0:1616513089.441858:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd0d80 x1693114552522560/t0(0) o400->3f42730e-c616-6112-ce7d-6cc1bbfb31a2@10.31.136.232@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.441869:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114552522560, offset 192 00000400:00000200:0.0:1616513089.441874:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.232@tcp1 00000400:00000200:0.0:1616513089.441880:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.232@tcp1 routed destination 00000400:00000200:0.0:1616513089.441893:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.232@tcp1(10.31.136.232@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.441899:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513089.441904:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.441906:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.441908:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.441911:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.441916:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513089.441959:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513089.441974:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.441981:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.441983:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513089.441990:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.441994:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42c38 00000800:00000200:20.0:1616513089.442000:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.442003:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513089.442013:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.442022:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.442032:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.232@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.442041:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.232@tcp1 of length 224 into portal 26 MB=0x603e0f37796c0 00000400:00000200:26.0:1616513089.442047:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.232@tcp1 of length 224/224 into md 0xf97cd039 [1] + 0 00000400:00000200:26.0:1616513089.442052:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4cbdb4dd0 00000400:00000200:26.0:1616513089.442054:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.442057:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513089.442061:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbdb4dd0 00000800:00000200:26.0:1616513089.442064:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.442067:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.442067:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.442071:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513089.442125:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114552522432 00010000:00000200:4.0:1616513089.442136:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1693114552522432/t0(0) o400->9132ce6f-4f37-5483-0461-72effdd4065b@10.31.136.232@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.442151:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114552522432, offset 192 00000400:00000200:4.0:1616513089.442157:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.232@tcp1 00000400:00000200:4.0:1616513089.442164:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.232@tcp1 routed destination 00000400:00000200:4.0:1616513089.442179:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.232@tcp1(10.31.136.232@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.442186:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513089.442192:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.442194:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513089.442197:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513089.442201:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.442206:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513089.442208:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513089.442260:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.442267:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.442270:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.442276:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.442282:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb7e8 00000800:00000200:2.0:1616513089.442286:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.442288:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.442372:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.442383:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.442391:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513089.442393:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.442400:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.198@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.442411:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.198@tcp1 of length 224 into portal 26 MB=0x5ffe07f35fd80 00000400:00000200:18.0:1616513089.442417:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.198@tcp1 of length 224/224 into md 0xf97cd059 [1] + 0 00000400:00000200:18.0:1616513089.442422:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8cc7bfba90 00000400:00000200:18.0:1616513089.442425:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.442427:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513089.442431:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfba90 00000800:00000200:18.0:1616513089.442452:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513089.442454:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714555555200 00000800:00000200:18.0:1616513089.442455:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.442457:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:4.0:1616513089.442460:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688714555555200/t0(0) o400->6e2e5a28-ac53-5b78-9174-619f3a1036d0@10.31.142.198@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.442472:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688714555555200, offset 192 00000400:00000200:4.0:1616513089.442479:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.198@tcp1 00000400:00000200:4.0:1616513089.442486:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.198@tcp1 routed destination 00000400:00000200:4.0:1616513089.442499:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.198@tcp1(10.31.142.198@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.442505:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.442509:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.442512:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.442513:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.442517:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.442522:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.442522:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513089.442532:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.442537:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513089.442539:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513089.442541:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513089.442546:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513089.442552:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba330 00000800:00000200:12.0:1616513089.442555:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.443196:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513089.443248:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.443257:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513089.443267:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.198@tcp1 : PUT - for me 00000400:00000200:20.0:1616513089.443276:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.198@tcp1 of length 224 into portal 12 MB=0x5ffe07f360000 00000400:00000200:20.0:1616513089.443282:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.198@tcp1 of length 224/224 into md 0x865be291 [1] + 11760 00000400:00000200:20.0:1616513089.443287:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.443290:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513089.443296:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.443299:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.443301:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513089.443306:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.443361:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688714555555840 00010000:00000200:0.0:1616513089.443374:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5580 x1688714555555840/t0(0) o400->2c36aa8f-0a8c-fbbb-c2fd-f432b23e45d9@10.31.142.198@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.443388:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688714555555840, offset 192 00000400:00000200:0.0:1616513089.443394:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.198@tcp1 00000400:00000200:0.0:1616513089.443402:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.198@tcp1 routed destination 00000400:00000200:0.0:1616513089.443417:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.198@tcp1(10.31.142.198@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.443424:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.443430:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.443433:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.443449:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.443454:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.443458:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.443512:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.443519:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.443522:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.443528:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.443539:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43100 00000800:00000200:2.0:1616513089.443543:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.443546:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513089.450542:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927664, portal 4 00000100:00000200:14.0:1616513089.450548:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927664, offset 0 00000400:00000200:14.0:1616513089.450555:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:14.0:1616513089.450561:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:14.0:1616513089.450576:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513089.450582:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513089.450588:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513089.450591:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513089.450593:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513089.450599:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513089.450646:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513089.450666:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513089.450673:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513089.450675:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.450678:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:26.0:1616513089.450682:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513089.450688:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d7461209680 x1693067725927664/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513089.450712:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757210 00000800:00000200:26.0:1616513089.450716:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513089.450811:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513089.450829:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513089.450838:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:14.0:1616513089.450841:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513089.450847:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513089.450855:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6274f0 00000400:00000200:2.0:1616513089.450862:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cd719 [1] + 192 00000400:00000200:2.0:1616513089.450867:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.450872:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d7461209680 x1693067725927664/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513089.450887:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513089.450890:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513089.450892:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:14.0:1616513089.450893:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d7461209680 x1693067725927664/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513089.450906:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d7461209680 x1693067725927664/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513089.450916:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757980 00000800:00000200:11.2:1616513089.468902:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.468959:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.468968:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.468976:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.190@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.468986:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.190@tcp1 of length 224 into portal 26 MB=0x5ffd33d8718c0 00000400:00000200:16.0:1616513089.468992:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.190@tcp1 of length 224/224 into md 0xf97cd069 [1] + 0 00000400:00000200:16.0:1616513089.468996:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8cc7bfb078 00000400:00000200:16.0:1616513089.468999:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.469002:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513089.469006:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfb078 00000800:00000200:16.0:1616513089.469009:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.469012:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513089.469012:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513089.469019:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.469022:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.469023:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.469031:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.190@tcp1 : PUT - for me 00000400:00000200:12.0:1616513089.469040:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.190@tcp1 of length 224 into portal 12 MB=0x5ffd33d871940 00000400:00000200:12.0:1616513089.469047:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.190@tcp1 of length 224/224 into md 0x865be291 [1] + 11984 00000400:00000200:12.0:1616513089.469052:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.469054:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513089.469061:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.469064:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.469066:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513089.469071:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657618999488 00010000:00000200:4.0:1616513089.469082:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688657618999488/t0(0) o400->e1443226-e37b-8a91-735c-c68844805a04@10.31.138.190@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.469096:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657618999488, offset 192 00000400:00000200:4.0:1616513089.469103:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.190@tcp1 00000400:00000200:4.0:1616513089.469110:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.190@tcp1 routed destination 00000400:00000200:4.0:1616513089.469125:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.190@tcp1(10.31.138.190@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513089.469126:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657618999616 00000800:00000200:4.0:1616513089.469131:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.469137:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:0.0:1616513089.469139:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d200000 x1688657618999616/t0(0) o400->a6a9ca39-a5b9-b332-aa78-9e93ada5e07e@10.31.138.190@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:4.0:1616513089.469140:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.469142:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.469147:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.469152:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.469153:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:0.0:1616513089.469153:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657618999616, offset 192 00000400:00000200:0.0:1616513089.469159:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.190@tcp1 00000400:00000200:0.0:1616513089.469167:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.190@tcp1 routed destination 00000400:00000200:0.0:1616513089.469183:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.190@tcp1(10.31.138.190@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.469190:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.469196:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.469198:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.469201:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513089.469205:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513089.469205:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513089.469212:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.469214:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513089.469214:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513089.469215:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513089.469220:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.469225:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baee0 00000800:00000200:20.0:1616513089.469229:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513089.469268:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.469275:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.469277:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.469283:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.469287:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ba0 00000800:00000200:18.0:1616513089.469291:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.469294:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.477756:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513089.477812:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513089.477821:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.477830:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.16@tcp1 : PUT - for me 00000400:00000200:12.0:1616513089.477840:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.16@tcp1 of length 224 into portal 26 MB=0x6005eb2dd5880 00000400:00000200:12.0:1616513089.477846:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.16@tcp1 of length 224/224 into md 0xf97cd079 [1] + 0 00000400:00000200:12.0:1616513089.477851:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8cc7bfbed0 00000400:00000200:12.0:1616513089.477854:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513089.477856:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513089.477862:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfbed0 00000800:00000200:28.0:1616513089.477866:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513089.477866:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513089.477868:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513089.477870:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513089.477926:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689256588040320 00000800:00000200:7.2:1616513089.477930:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:4.0:1616513089.477937:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1689256588040320/t0(0) o400->82548cd6-e895-611d-b698-34def08e2e72@10.31.132.16@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.477951:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689256588040320, offset 192 00000400:00000200:4.0:1616513089.477957:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.16@tcp1 00000400:00000200:4.0:1616513089.477965:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.16@tcp1 routed destination 00000400:00000200:4.0:1616513089.477980:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.16@tcp1(10.31.132.16@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.477990:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513089.477993:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513089.477996:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.477999:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.478001:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513089.478003:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:4.0:1616513089.478006:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.478011:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513089.478012:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.16@tcp1 : PUT - for me 00000100:00000200:4.0:1616513089.478012:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513089.478020:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.16@tcp1 of length 224 into portal 12 MB=0x6005eb2dd5900 00000400:00000200:14.0:1616513089.478027:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.16@tcp1 of length 224/224 into md 0xf964d519 [1] + 80616 00000400:00000200:14.0:1616513089.478032:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.478035:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513089.478042:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.478046:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.478046:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.478050:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513089.478069:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.478076:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.478079:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.478086:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.478091:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb540 00000800:00000200:16.0:1616513089.478095:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.478098:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513089.478111:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689256588040448 00010000:00000200:22.0:1616513089.478124:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03180 x1689256588040448/t0(0) o400->3ce561ce-c53c-a1c9-ed89-76703fcf3cae@10.31.132.16@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.478138:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689256588040448, offset 192 00000400:00000200:22.0:1616513089.478144:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.16@tcp1 00000400:00000200:22.0:1616513089.478151:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.16@tcp1 routed destination 00000400:00000200:22.0:1616513089.478166:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.16@tcp1(10.31.132.16@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.478172:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.478178:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.478181:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.478183:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.478188:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.478194:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.478246:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.478253:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.478255:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.478262:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.478265:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b540 00000800:00000200:18.0:1616513089.478269:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.478272:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513089.498538:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927680, portal 4 00000100:00000200:23.0:1616513089.498544:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927680, offset 0 00000400:00000200:23.0:1616513089.498551:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:23.0:1616513089.498558:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:23.0:1616513089.498572:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513089.498579:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:23.0:1616513089.498586:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513089.498589:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:23.0:1616513089.498591:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:23.0:1616513089.498597:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513089.498644:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000100:00000200:15.0:1616513089.498698:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927696, portal 4 00000800:00000200:2.0:1616513089.498702:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000100:00000200:15.0:1616513089.498703:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927696, offset 0 00000800:00000200:2.0:1616513089.498709:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:15.0:1616513089.498710:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:2.0:1616513089.498713:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:15.0:1616513089.498716:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:2.0:1616513089.498720:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513089.498727:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63c380 x1693067725927680/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513089.498730:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513089.498737:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:2.0:1616513089.498739:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3ec38 00000800:00000200:15.0:1616513089.498743:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513089.498746:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:2.0:1616513089.498746:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:15.0:1616513089.498748:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:2.0:1616513089.498749:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:15.0:1616513089.498754:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513089.498763:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.498778:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513089.498784:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513089.498787:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513089.498794:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513089.498800:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600900 x1693067725927696/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513089.498812:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0a990 00000800:00000200:25.2:1616513089.498816:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:27.0:1616513089.498817:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.498819:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:18.0:1616513089.498873:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513089.498883:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513089.498891:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513089.498899:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627500 00000400:00000200:18.0:1616513089.498906:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cd769 [1] + 192 00000400:00000200:18.0:1616513089.498911:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.498917:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63c380 x1693067725927680/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513089.498926:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513089.498932:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513089.498935:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513089.498937:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:23.0:1616513089.498940:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63c380 x1693067725927680/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513089.498954:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63c380 x1693067725927680/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513089.498964:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f4b8 00000800:00000200:27.2:1616513089.498968:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513089.498977:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.498982:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513089.498986:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513089.498990:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513089.498998:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627510 00000400:00000200:27.0:1616513089.499004:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x37166ffd [1] + 192 00000400:00000200:27.0:1616513089.499009:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513089.499016:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600900 x1693067725927696/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513089.499030:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.499033:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.499035:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:15.0:1616513089.499037:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600900 x1693067725927696/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513089.499050:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600900 x1693067725927696/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513089.499070:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000100:00000200:6.0:1616513089.499468:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927712, portal 4 00000100:00000200:6.0:1616513089.499473:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927712, offset 0 00000400:00000200:6.0:1616513089.499478:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:6.0:1616513089.499482:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:6.0:1616513089.499491:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513089.499496:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:6.0:1616513089.499501:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513089.499503:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:6.0:1616513089.499505:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:6.0:1616513089.499510:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513089.499515:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.499523:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513089.499527:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513089.499530:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513089.499531:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513089.499535:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513089.499540:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bdec00 x1693067725927712/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513089.499550:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00ee0 00000800:00000200:27.0:1616513089.499553:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513089.499568:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513089.499576:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.499581:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513089.499583:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513089.499588:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513089.499593:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627520 00000400:00000200:27.0:1616513089.499599:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716701d [1] + 192 00000400:00000200:27.0:1616513089.499603:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513089.499606:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bdec00 x1693067725927712/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513089.499618:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.499621:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.499622:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:6.0:1616513089.499625:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bdec00 x1693067725927712/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:6.0:1616513089.499634:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bdec00 x1693067725927712/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513089.499641:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:11.2:1616513089.512080:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.512137:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513089.512141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513089.512146:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.512155:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.8@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.512165:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.8@tcp1 of length 224 into portal 26 MB=0x5ffdb1b602cc0 00000400:00000200:28.0:1616513089.512171:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.8@tcp1 of length 224/224 into md 0xf97cd0a9 [1] + 0 00000400:00000200:28.0:1616513089.512176:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8cc7bfbf58 00000400:00000200:28.0:1616513089.512186:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.512189:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513089.512190:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513089.512193:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfbf58 00000800:00000200:28.0:1616513089.512197:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513089.512200:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513089.512201:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.512202:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.512204:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.512210:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513089.512210:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.8@tcp1 : PUT - for me 00000800:00000200:2.0:1616513089.512211:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513089.512214:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691405761728 00000400:00000200:28.0:1616513089.512217:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.221@tcp1 : PUT - for me 00000400:00000200:12.0:1616513089.512218:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.8@tcp1 of length 224 into portal 12 MB=0x5ffdb1b602d80 00000800:00000200:2.0:1616513089.512220:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513089.512224:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.221@tcp1 of length 224 into portal 26 MB=0x601aca145c6c0 00000400:00000200:12.0:1616513089.512225:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.8@tcp1 of length 224/224 into md 0x865be291 [1] + 12208 00010000:00000200:4.0:1616513089.512226:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688691405761728/t0(0) o400->8cf5ca15-29cd-2f21-fe09-34d2143465a8@10.31.141.8@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513089.512229:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.221@tcp1 of length 224/224 into md 0xf97cd0b9 [1] + 0 00000400:00000200:12.0:1616513089.512230:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.512230:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.221@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.512233:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8cc7bfb320 00000100:00000200:12.0:1616513089.512233:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513089.512234:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.512237:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513089.512239:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.221@tcp1 of length 224 into portal 12 MB=0x601aca145c740 00000800:00000200:12.0:1616513089.512240:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:4.0:1616513089.512240:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688691405761728, offset 192 00000400:00000200:28.0:1616513089.512243:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfb320 00000800:00000200:12.0:1616513089.512243:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513089.512245:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513089.512245:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513089.512247:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:4.0:1616513089.512247:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.8@tcp1 00000400:00000200:2.0:1616513089.512248:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.221@tcp1 of length 224/224 into md 0xf964d519 [1] + 80840 00000800:00000200:12.0:1616513089.512249:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.512252:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513089.512253:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:4.0:1616513089.512254:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.8@tcp1 routed destination 00000100:00000200:2.0:1616513089.512256:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513089.512262:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:8.0:1616513089.512265:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690690811971264 00000800:00000200:2.0:1616513089.512265:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.512267:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513089.512269:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.8@tcp1(10.31.141.8@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513089.512270:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:8.0:1616513089.512276:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1690690811971264/t0(0) o400->27d1d6ec-8555-f2a6-97e7-b582071c1965@10.31.140.221@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:4.0:1616513089.512276:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.512281:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.512284:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.512286:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513089.512291:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690690811971264, offset 192 00000800:00000200:4.0:1616513089.512291:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.512296:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513089.512297:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.221@tcp1 00000100:00000200:4.0:1616513089.512298:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:8.0:1616513089.512305:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.221@tcp1 routed destination 00000800:00000200:16.0:1616513089.512307:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.512314:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513089.512315:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691405761920 00000400:00000200:16.0:1616513089.512316:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513089.512319:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513089.512320:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.221@tcp1(10.31.140.221@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513089.512323:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:8.0:1616513089.512327:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:0.0:1616513089.512328:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d207080 x1688691405761920/t0(0) o400->6d19681e-e7bb-d6f0-9b6d-feeff075f52f@10.31.141.8@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513089.512329:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baff0 00000100:00000200:22.0:1616513089.512332:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690690811971392 00000800:00000200:16.0:1616513089.512333:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513089.512333:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513089.512336:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513089.512338:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513089.512342:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688691405761920, offset 192 00000800:00000200:8.0:1616513089.512343:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:22.0:1616513089.512344:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53c9b00 x1690690811971392/t0(0) o400->6723e88b-d23c-bc66-cd73-97c7ba151fdd@10.31.140.221@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513089.512347:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513089.512348:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.8@tcp1 00000100:00000200:8.0:1616513089.512351:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513089.512356:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.8@tcp1 routed destination 00000100:00000200:22.0:1616513089.512359:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690690811971392, offset 192 00000400:00000200:22.0:1616513089.512365:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.221@tcp1 00000400:00000200:0.0:1616513089.512371:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.8@tcp1(10.31.141.8@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513089.512373:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.221@tcp1 routed destination 00000800:00000200:0.0:1616513089.512378:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513089.512384:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513089.512387:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.221@tcp1(10.31.140.221@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.512387:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.512389:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.512393:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513089.512394:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513089.512398:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513089.512400:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513089.512400:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.512402:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513089.512404:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (24)++ 00000800:00000200:28.0:1616513089.512408:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513089.512409:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:26.0:1616513089.512410:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (25)-- 00000800:00000200:28.0:1616513089.512411:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513089.512413:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.512414:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513089.512415:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513089.512419:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.512420:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.512422:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42880 00000800:00000200:28.0:1616513089.512425:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513089.512427:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:26.0:1616513089.512431:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.512452:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.512457:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.512458:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.512459:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.512464:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.512467:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a330 00000800:00000200:26.0:1616513089.512469:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.530011:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513089.530068:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.530077:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513089.530087:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.23@tcp1 : PUT - for me 00000400:00000200:20.0:1616513089.530098:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.23@tcp1 of length 224 into portal 26 MB=0x5ffe79e1ba1c0 00000400:00000200:20.0:1616513089.530104:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.23@tcp1 of length 224/224 into md 0xf97cd0c9 [1] + 0 00000400:00000200:20.0:1616513089.530109:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8cc7bfbdc0 00000400:00000200:20.0:1616513089.530112:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.530115:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513089.530121:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfbdc0 00000800:00000200:16.0:1616513089.530122:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.530124:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.530127:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513089.530130:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513089.530133:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.530134:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.530142:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.23@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.530152:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.23@tcp1 of length 224 into portal 12 MB=0x5ffe79e1ba3c0 00000400:00000200:16.0:1616513089.530159:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.23@tcp1 of length 224/224 into md 0x865be291 [1] + 12432 00000400:00000200:16.0:1616513089.530164:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.530167:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513089.530174:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.530187:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.530189:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.530206:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688745138693056 00000100:00000200:24.0:1616513089.530209:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688745138692544 00010000:00000200:0.0:1616513089.530220:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd3a80 x1688745138693056/t0(0) o400->652a9260-d388-f0e4-971b-db92022ebc9b@10.31.142.23@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513089.530221:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688745138692544/t0(0) o400->531c880f-d9fe-14bf-5dba-6f6f37dd1f13@10.31.142.23@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513089.530234:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688745138693056, offset 192 00000100:00000200:24.0:1616513089.530236:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688745138692544, offset 192 00000400:00000200:0.0:1616513089.530241:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.23@tcp1 00000400:00000200:24.0:1616513089.530246:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.23@tcp1 00000400:00000200:0.0:1616513089.530248:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.23@tcp1 routed destination 00000400:00000200:0.0:1616513089.530264:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.23@tcp1(10.31.142.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.530270:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513089.530273:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.23@tcp1 routed destination 00000800:00000200:0.0:1616513089.530281:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.530284:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.530286:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513089.530288:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.23@tcp1(10.31.142.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.530295:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.530297:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513089.530301:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:11.2:1616513089.530302:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.530304:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513089.530306:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513089.530311:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513089.530314:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513089.530318:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513089.530319:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.530321:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.530324:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513089.530325:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513089.530331:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.530335:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42bb0 00000800:00000200:28.0:1616513089.530339:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513089.530378:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.530385:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.530388:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.530394:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513089.530395:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.530400:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3430 00000800:00000200:14.0:1616513089.530404:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.530909:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513089.530933:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.530943:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.530949:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513089.530952:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513089.530958:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.41@tcp1 : PUT - for me 00000800:00000200:2.0:1616513089.530965:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513089.530972:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.41@tcp1 of length 224 into portal 12 MB=0x6001cbd839900 00000800:00000200:2.0:1616513089.530975:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513089.530979:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.41@tcp1 of length 224/224 into md 0x865be291 [1] + 12656 00000400:00000200:2.0:1616513089.530984:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.41@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.530985:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.530987:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.530994:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513089.530994:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.41@tcp1 of length 224 into portal 26 MB=0x6001cbd839880 00000800:00000200:28.0:1616513089.530996:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.530998:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513089.531000:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.41@tcp1 of length 224/224 into md 0xf97cd0d9 [1] + 0 00000100:00000200:0.0:1616513089.531006:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688973298866432 00000400:00000200:2.0:1616513089.531011:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8cc7bfa198 00010000:00000200:0.0:1616513089.531012:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5100 x1688973298866432/t0(0) o400->a806951b-11ad-5ff7-ec7f-57c80eb59af0@10.31.140.41@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513089.531014:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.531017:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513089.531021:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfa198 00000800:00000200:2.0:1616513089.531024:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:0.0:1616513089.531024:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688973298866432, offset 192 00000800:00000200:2.0:1616513089.531027:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.531028:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513089.531030:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.41@tcp1 00000800:00000200:2.0:1616513089.531031:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.531034:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688973298866304 00000400:00000200:0.0:1616513089.531036:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.41@tcp1 routed destination 00010000:00000200:24.0:1616513089.531040:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688973298866304/t0(0) o400->71681ea2-d7d4-5fce-80e9-578d31956e5d@10.31.140.41@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513089.531049:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.41@tcp1(10.31.140.41@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513089.531051:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688973298866304, offset 192 00000800:00000200:0.0:1616513089.531055:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513089.531056:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.41@tcp1 00000800:00000200:0.0:1616513089.531060:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513089.531062:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.41@tcp1 routed destination 00000800:00000200:0.0:1616513089.531062:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.531064:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.531067:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.531072:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513089.531076:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.41@tcp1(10.31.140.41@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.531081:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513089.531081:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.531085:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513089.531086:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.531088:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513089.531088:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513089.531090:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513089.531090:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513089.531093:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513089.531093:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.531096:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42110 00000800:00000200:7.2:1616513089.531098:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513089.531099:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513089.531099:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513089.531109:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.531114:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.531117:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.531118:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.531124:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.531129:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f36d8 00000800:00000200:18.0:1616513089.531132:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.535308:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.535362:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.535371:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.535380:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.6@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.535390:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.6@tcp1 of length 224 into portal 26 MB=0x5ffd3e8d1f680 00000400:00000200:26.0:1616513089.535397:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.6@tcp1 of length 224/224 into md 0xf97cd0f9 [1] + 0 00000400:00000200:26.0:1616513089.535401:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8cc7bfb650 00000400:00000200:26.0:1616513089.535404:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.535407:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513089.535411:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8cc7bfb650 00000800:00000200:26.0:1616513089.535414:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.535417:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.535419:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.535426:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.535429:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.535430:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513089.535451:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.6@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.535460:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.6@tcp1 of length 224 into portal 12 MB=0x5ffd3e8d1f840 00000400:00000200:2.0:1616513089.535467:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.6@tcp1 of length 224/224 into md 0xf964d519 [1] + 81064 00000400:00000200:2.0:1616513089.535472:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513089.535475:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660492809856 00000100:00000200:2.0:1616513089.535477:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513089.535483:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00010000:00000200:24.0:1616513089.535485:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688660492809856/t0(0) o400->189f41f5-fe32-7471-677d-d6bb50cfa4a9@10.31.138.6@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513089.535487:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.535488:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.535499:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660492809856, offset 192 00000400:00000200:24.0:1616513089.535505:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.6@tcp1 00000400:00000200:24.0:1616513089.535513:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.6@tcp1 routed destination 00000400:00000200:24.0:1616513089.535528:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.6@tcp1(10.31.138.6@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.535534:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.535540:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.535542:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.535545:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.535549:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.535556:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.535556:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513089.535557:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660492810304 00010000:00000200:22.0:1616513089.535570:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53cb600 x1688660492810304/t0(0) o400->d512406b-c10d-9aa3-d27f-cbea304fd0f5@10.31.138.6@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.535596:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660492810304, offset 192 00000400:00000200:22.0:1616513089.535602:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.6@tcp1 00000400:00000200:22.0:1616513089.535609:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.6@tcp1 routed destination 00000800:00000200:16.0:1616513089.535610:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.535617:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.535619:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513089.535624:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.6@tcp1(10.31.138.6@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513089.535626:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513089.535631:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513089.535632:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2dd0 00000800:00000200:16.0:1616513089.535635:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513089.535637:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513089.535638:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513089.535639:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.535642:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.535646:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.535651:0:50491:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.535705:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.535713:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.535715:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513089.535716:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.535722:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.535726:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b188 00000800:00000200:18.0:1616513089.535730:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.581511:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.581566:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.581575:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.581584:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.6@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.581593:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.6@tcp1 of length 224 into portal 26 MB=0x5ffd4158c2e00 00000400:00000200:28.0:1616513089.581600:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.6@tcp1 of length 224/224 into md 0xf97cd109 [1] + 0 00000400:00000200:28.0:1616513089.581604:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3760 00000400:00000200:28.0:1616513089.581607:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.581610:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.581615:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3760 00000800:00000200:28.0:1616513089.581618:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.581620:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.581621:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.581624:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.581682:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661243211264 00010000:00000200:24.0:1616513089.581693:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688661243211264/t0(0) o400->9ef89c70-e11a-978c-9b2c-fed5ea8a0000@10.31.133.6@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.581708:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661243211264, offset 192 00000400:00000200:24.0:1616513089.581715:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.6@tcp1 00000400:00000200:24.0:1616513089.581722:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.6@tcp1 routed destination 00000400:00000200:24.0:1616513089.581738:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.6@tcp1(10.31.133.6@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.581745:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.581751:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.581754:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.581756:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.581762:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.581766:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.581769:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513089.581821:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.581828:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.581831:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513089.581832:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513089.581838:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.581843:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3430 00000800:00000200:16.0:1616513089.581847:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.582206:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513089.582217:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.582225:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.582226:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513089.582234:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.6@tcp1 : PUT - for me 00000400:00000200:20.0:1616513089.582243:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.6@tcp1 of length 224 into portal 12 MB=0x5ffd4158c2f80 00000400:00000200:20.0:1616513089.582249:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.6@tcp1 of length 224/224 into md 0x865be291 [1] + 12880 00000400:00000200:20.0:1616513089.582254:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.582257:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513089.582265:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.582268:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.582270:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.582330:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661243211648 00010000:00000200:0.0:1616513089.582344:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad1680 x1688661243211648/t0(0) o400->0af5799a-f628-5aa9-431c-a434d404e964@10.31.133.6@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.582358:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661243211648, offset 192 00000400:00000200:0.0:1616513089.582364:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.6@tcp1 00000400:00000200:0.0:1616513089.582372:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.6@tcp1 routed destination 00000400:00000200:0.0:1616513089.582387:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.6@tcp1(10.31.133.6@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.582394:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.582400:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.582403:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.582405:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.582411:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.582418:0:50496:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.582482:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.582490:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.582493:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513089.582501:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.582506:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a423b8 00000800:00000200:14.0:1616513089.582510:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.582513:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.587145:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.587199:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.587208:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.587217:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.96@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.587226:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.96@tcp1 of length 224 into portal 26 MB=0x5ffd5b39c8c80 00000400:00000200:28.0:1616513089.587233:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.96@tcp1 of length 224/224 into md 0xf97cd149 [1] + 0 00000400:00000200:28.0:1616513089.587237:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2cc0 00000400:00000200:28.0:1616513089.587240:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.587243:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.587247:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2cc0 00000800:00000200:28.0:1616513089.587250:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.587252:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.587253:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.587257:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.587309:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668190051456 00010000:00000200:24.0:1616513089.587320:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688668190051456/t0(0) o400->e496895e-3985-1d60-4561-9c119a99162a@10.31.136.96@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.587334:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668190051456, offset 192 00000400:00000200:24.0:1616513089.587343:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.96@tcp1 00000400:00000200:24.0:1616513089.587351:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.96@tcp1 routed destination 00000400:00000200:24.0:1616513089.587365:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.96@tcp1(10.31.136.96@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.587372:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.587377:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.587380:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.587382:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.587387:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.587391:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.587394:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513089.587408:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.587412:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.587414:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.587420:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.587424:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3ba0 00000800:00000200:28.0:1616513089.587427:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.587429:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.587527:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.587583:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.587592:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.587602:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.96@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.587611:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.96@tcp1 of length 224 into portal 12 MB=0x5ffd5b39c8e00 00000400:00000200:26.0:1616513089.587619:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.96@tcp1 of length 224/224 into md 0xf964d519 [1] + 81288 00000400:00000200:26.0:1616513089.587624:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.587627:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513089.587634:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.587638:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.587638:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.587642:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513089.587703:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668190051840 00010000:00000200:22.0:1616513089.587716:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c694c80 x1688668190051840/t0(0) o400->ef61e6ed-4cc2-8162-43eb-1c499aef21a8@10.31.136.96@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.587730:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668190051840, offset 192 00000400:00000200:22.0:1616513089.587737:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.96@tcp1 00000400:00000200:22.0:1616513089.587744:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.96@tcp1 routed destination 00000400:00000200:22.0:1616513089.587759:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.96@tcp1(10.31.136.96@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.587765:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.587771:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.587774:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.587776:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.587781:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.587785:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.587837:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.587844:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.587847:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513089.587854:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.587858:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a990 00000800:00000200:14.0:1616513089.587862:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.587864:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.599776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.599832:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.599842:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.599851:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.70@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.599860:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.70@tcp1 of length 224 into portal 26 MB=0x5ffe301499f40 00000400:00000200:18.0:1616513089.599867:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.70@tcp1 of length 224/224 into md 0xf97cd169 [1] + 0 00000400:00000200:18.0:1616513089.599872:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3a08 00000400:00000200:18.0:1616513089.599875:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.599878:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513089.599882:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3a08 00000800:00000200:18.0:1616513089.599885:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.599886:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.599888:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.599893:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.599896:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.599897:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.599906:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.70@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.599914:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.70@tcp1 of length 224 into portal 12 MB=0x5ffe301499fc0 00000400:00000200:26.0:1616513089.599921:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.70@tcp1 of length 224/224 into md 0xf964d519 [1] + 81512 00000400:00000200:26.0:1616513089.599926:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.599929:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513089.599936:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.599939:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.599941:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.599947:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688725327814464 00010000:00000200:24.0:1616513089.599958:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688725327814464/t0(0) o400->d7aa4c68-4dff-31a4-9d59-2fca418817a3@10.31.143.70@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.599972:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688725327814464, offset 192 00000400:00000200:24.0:1616513089.599978:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.70@tcp1 00000400:00000200:24.0:1616513089.599986:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.70@tcp1 routed destination 00000400:00000200:24.0:1616513089.600001:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.70@tcp1(10.31.143.70@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513089.600003:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688725327814592 00000800:00000200:24.0:1616513089.600008:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.600013:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513089.600015:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c690d80 x1688725327814592/t0(0) o400->4cba191e-bdb3-be15-0523-e9b11fe55bc5@10.31.143.70@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513089.600016:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.600018:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.600023:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.600030:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513089.600030:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688725327814592, offset 192 00000800:00000200:11.2:1616513089.600030:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513089.600036:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.70@tcp1 00000400:00000200:22.0:1616513089.600044:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.70@tcp1 routed destination 00000400:00000200:22.0:1616513089.600058:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.70@tcp1(10.31.143.70@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.600067:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.600073:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.600076:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.600078:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.600083:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513089.600087:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513089.600088:0:50498:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513089.600094:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.600097:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.600103:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.600109:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f25d8 00000800:00000200:16.0:1616513089.600112:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.600115:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513089.600144:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.600151:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.600154:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.600155:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.600161:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.600165:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a3b8 00000800:00000200:2.0:1616513089.600169:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513089.610590:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927728, portal 4 00000100:00000200:28.0:1616513089.610596:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927728, offset 0 00000400:00000200:28.0:1616513089.610603:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:28.0:1616513089.610608:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:28.0:1616513089.610621:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513089.610627:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513089.610633:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513089.610636:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513089.610638:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513089.610643:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513089.610690:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513089.610704:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513089.610709:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513089.610712:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.610717:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513089.610722:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb239b00 x1693067725927728/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513089.610734:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc5d8 00000800:00000200:28.0:1616513089.610736:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513089.610739:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513089.610874:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513089.610886:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513089.610896:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:28.0:1616513089.610897:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:16.0:1616513089.610905:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513089.610913:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627530 00000400:00000200:16.0:1616513089.610919:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866da191 [1] + 192 00000400:00000200:16.0:1616513089.610925:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.610930:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb239b00 x1693067725927728/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513089.610944:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513089.610947:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:28.0:1616513089.610950:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb239b00 x1693067725927728/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513089.610951:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:28.0:1616513089.610962:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb239b00 x1693067725927728/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513089.610971:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc7f8 00000100:00000200:6.0:1616513089.642859:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927744, portal 4 00000100:00000200:6.0:1616513089.642864:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927744, offset 0 00000400:00000200:6.0:1616513089.642871:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:6.0:1616513089.642888:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:6.0:1616513089.642899:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513089.642905:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:6.0:1616513089.642910:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513089.642913:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:6.0:1616513089.642914:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:6.0:1616513089.642919:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513089.642966:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513089.642985:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513089.642992:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513089.642995:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513089.643002:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513089.643009:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bdc800 x1693067725927744/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513089.643022:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf018f8 00000800:00000200:1.0:1616513089.643027:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513089.643030:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513089.643142:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513089.643152:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513089.643159:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513089.643163:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513089.643167:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513089.643175:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627540 00000400:00000200:1.0:1616513089.643182:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436fff05 [1] + 192 00000400:00000200:1.0:1616513089.643188:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513089.643192:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bdc800 x1693067725927744/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513089.643207:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513089.643211:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513089.643212:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:6.0:1616513089.643216:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bdc800 x1693067725927744/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:6.0:1616513089.643226:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bdc800 x1693067725927744/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513089.643235:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:7.2:1616513089.651343:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.651399:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.651409:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.651419:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.94@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.651430:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.94@tcp1 of length 224 into portal 12 MB=0x6001c3d204340 00000400:00000200:26.0:1616513089.651450:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.94@tcp1 of length 224/224 into md 0xf964d519 [1] + 81736 00000800:00000200:14.0:1616513089.651453:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.651455:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.651458:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513089.651466:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.651469:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.651471:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.651484:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513089.651533:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688971144872768 00000800:00000200:28.0:1616513089.651543:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513089.651547:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c690480 x1688971144872768/t0(0) o400->df3622cc-07cb-42da-3a00-f4dda20605ff@10.31.138.94@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513089.651553:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.651562:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.94@tcp1 : PUT - for me 00000100:00000200:22.0:1616513089.651562:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688971144872768, offset 192 00000400:00000200:22.0:1616513089.651579:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.94@tcp1 00000400:00000200:28.0:1616513089.651581:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.94@tcp1 of length 224 into portal 26 MB=0x6001c3d204240 00000400:00000200:22.0:1616513089.651586:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.94@tcp1 routed destination 00000400:00000200:28.0:1616513089.651587:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.94@tcp1 of length 224/224 into md 0xf97cd189 [1] + 0 00000400:00000200:28.0:1616513089.651592:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3320 00000400:00000200:28.0:1616513089.651594:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.651597:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.651601:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3320 00000400:00000200:22.0:1616513089.651601:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.94@tcp1(10.31.138.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513089.651605:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513089.651606:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.651608:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.651610:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.651612:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513089.651612:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.651614:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513089.651616:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.651621:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.651625:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.651635:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.651639:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.651641:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.651646:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.651649:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4add0 00000800:00000200:28.0:1616513089.651651:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.651653:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.651665:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688971144872512 00010000:00000200:24.0:1616513089.651675:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688971144872512/t0(0) o400->8c10d0c8-80f5-ba9d-4878-b67106339956@10.31.138.94@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.651689:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688971144872512, offset 192 00000400:00000200:24.0:1616513089.651695:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.94@tcp1 00000400:00000200:24.0:1616513089.651702:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.94@tcp1 routed destination 00000400:00000200:24.0:1616513089.651716:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.94@tcp1(10.31.138.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.651722:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513089.651729:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.651732:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513089.651734:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513089.651739:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.651743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513089.651745:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513089.651795:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.651802:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.651805:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.651812:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.651817:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3b18 00000800:00000200:18.0:1616513089.651821:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.651824:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513089.658520:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927760, portal 4 00000100:00000200:14.0:1616513089.658526:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927760, offset 0 00000400:00000200:14.0:1616513089.658533:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:14.0:1616513089.658539:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:14.0:1616513089.658552:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513089.658558:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:14.0:1616513089.658565:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513089.658568:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:14.0:1616513089.658570:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:14.0:1616513089.658575:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513089.658623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.658644:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513089.658651:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513089.658653:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513089.658660:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513089.658666:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d747c695580 x1693067725927760/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513089.658678:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757e48 00000800:00000200:27.0:1616513089.658683:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.658688:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513089.658790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513089.658799:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.658805:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513089.658808:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513089.658812:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513089.658820:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627550 00000400:00000200:27.0:1616513089.658826:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716702d [1] + 192 00000400:00000200:27.0:1616513089.658831:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513089.658836:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d747c695580 x1693067725927760/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513089.658850:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513089.658853:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513089.658855:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:14.0:1616513089.658860:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d747c695580 x1693067725927760/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513089.658873:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d747c695580 x1693067725927760/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513089.658883:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757980 00000800:00000200:7.2:1616513089.659408:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.659421:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.659428:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513089.659432:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513089.659448:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.250@tcp1 : PUT - for me 00000800:00000200:26.0:1616513089.659451:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.659456:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.250@tcp1 of length 224 into portal 26 MB=0x5ffdda46928c0 00000400:00000200:26.0:1616513089.659460:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.250@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.659463:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.250@tcp1 of length 224/224 into md 0xf97cd1a9 [1] + 0 00000400:00000200:26.0:1616513089.659469:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.250@tcp1 of length 224 into portal 12 MB=0x5ffdda4692940 00000400:00000200:14.0:1616513089.659469:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3980 00000400:00000200:14.0:1616513089.659471:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.659475:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.250@tcp1 of length 224/224 into md 0xf964d519 [1] + 81960 00000400:00000200:26.0:1616513089.659482:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.659482:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513089.659485:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.659486:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3980 00000100:00000200:26.0:1616513089.659488:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513089.659489:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.659491:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.659493:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.659494:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.659498:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.659500:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.659556:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702294763712 00010000:00000200:24.0:1616513089.659567:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688702294763712/t0(0) o400->6a4a7f07-726c-28cf-5f52-5fb95e0862db@10.31.141.250@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513089.659580:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702294763840 00000100:00000200:24.0:1616513089.659581:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702294763712, offset 192 00000400:00000200:24.0:1616513089.659588:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.250@tcp1 00010000:00000200:22.0:1616513089.659593:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695100 x1688702294763840/t0(0) o400->d7e73f64-e9f9-598b-bafe-750c85124fc5@10.31.141.250@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513089.659595:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.250@tcp1 routed destination 00000100:00000200:22.0:1616513089.659608:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702294763840, offset 192 00000400:00000200:24.0:1616513089.659610:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.250@tcp1(10.31.141.250@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513089.659615:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.250@tcp1 00000800:00000200:24.0:1616513089.659617:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.659622:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513089.659622:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.250@tcp1 routed destination 00000800:00000200:24.0:1616513089.659625:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.659627:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.659632:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513089.659637:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.250@tcp1(10.31.141.250@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513089.659639:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.659639:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.659643:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.659649:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.659651:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.659653:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.659658:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.659669:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.659694:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.659701:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.659704:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.659710:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.659716:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2e58 00000800:00000200:28.0:1616513089.659719:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.659722:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513089.659744:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.659751:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.659754:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.659754:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.659760:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.659765:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4acc0 00000800:00000200:2.0:1616513089.659769:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.669850:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.669904:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.669914:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.669923:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.206@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.669933:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.206@tcp1 of length 224 into portal 26 MB=0x5ffd517aa5d80 00000400:00000200:28.0:1616513089.669939:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.206@tcp1 of length 224/224 into md 0xf97cd1b9 [1] + 0 00000400:00000200:28.0:1616513089.669944:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3ed0 00000400:00000200:28.0:1616513089.669949:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.669952:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.669956:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3ed0 00000800:00000200:28.0:1616513089.669959:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.669959:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.669962:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.669964:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.670019:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665573711232 00010000:00000200:24.0:1616513089.670030:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688665573711232/t0(0) o400->647fae42-421d-cfb9-0c26-11626fba7810@10.31.132.206@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.670045:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665573711232, offset 192 00000400:00000200:24.0:1616513089.670051:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.206@tcp1 00000400:00000200:24.0:1616513089.670059:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.206@tcp1 routed destination 00000400:00000200:24.0:1616513089.670074:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.206@tcp1(10.31.132.206@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.670080:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.670086:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.670089:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.670097:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.670101:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.670106:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.670110:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513089.670115:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.670118:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.670120:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.670126:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.670130:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2f68 00000800:00000200:28.0:1616513089.670132:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.670135:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.670617:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.670628:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.670636:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.670638:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513089.670646:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.206@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.670655:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.206@tcp1 of length 224 into portal 12 MB=0x5ffd517aa5f00 00000400:00000200:16.0:1616513089.670661:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.206@tcp1 of length 224/224 into md 0x865be291 [1] + 13104 00000400:00000200:16.0:1616513089.670667:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.670669:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513089.670676:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.670679:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.670681:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.670741:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665573711616 00010000:00000200:0.0:1616513089.670753:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1200 x1688665573711616/t0(0) o400->073ac25c-cafe-a2ab-717f-0af86e264ca6@10.31.132.206@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.670768:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665573711616, offset 192 00000400:00000200:0.0:1616513089.670774:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.206@tcp1 00000400:00000200:0.0:1616513089.670783:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.206@tcp1 routed destination 00000400:00000200:0.0:1616513089.670798:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.206@tcp1(10.31.132.206@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.670805:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.670810:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.670813:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.670816:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.670820:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.670827:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.670881:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.670888:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.670891:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513089.670898:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.670902:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a426e8 00000800:00000200:14.0:1616513089.670906:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.670909:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:13.0:1616513089.675737:0:6491:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927776, portal 4 00000100:00000200:13.0:1616513089.675743:0:6491:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927776, offset 0 00000400:00000200:13.0:1616513089.675749:0:6491:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:13.0:1616513089.675755:0:6491:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:13.0:1616513089.675769:0:6491:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:13.0:1616513089.675776:0:6491:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:13.0:1616513089.675783:0:6491:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513089.675786:0:6491:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:13.0:1616513089.675788:0:6491:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:13.0:1616513089.675793:0:6491:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513089.675838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513089.675895:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513089.675902:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513089.675905:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.675911:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513089.675918:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4558ad1680 x1693067725927776/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513089.675930:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345760 00000800:00000200:28.0:1616513089.675935:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513089.675938:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513089.676023:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513089.676034:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513089.676039:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:28.0:1616513089.676047:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513089.676054:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627560 00000400:00000200:28.0:1616513089.676061:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866da1f1 [1] + 192 00000400:00000200:28.0:1616513089.676065:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.676070:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4558ad1680 x1693067725927776/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513089.676084:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:16.0:1616513089.676085:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513089.676087:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:28.0:1616513089.676090:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:13.0:1616513089.676094:0:6491:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4558ad1680 x1693067725927776/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:13.0:1616513089.676107:0:6491:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4558ad1680 x1693067725927776/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:13.0:1616513089.676117:0:6491:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345078 00000800:00000200:7.2:1616513089.693134:0:50509:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.693190:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.693200:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.693210:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.111@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.693220:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.111@tcp1 of length 224 into portal 26 MB=0x5ffdeebc91c00 00000400:00000200:18.0:1616513089.693228:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.111@tcp1 of length 224/224 into md 0xf97cd1c9 [1] + 0 00000400:00000200:18.0:1616513089.693233:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f34b8 00000400:00000200:18.0:1616513089.693236:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.693239:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513089.693243:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f34b8 00000800:00000200:18.0:1616513089.693247:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.693248:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.693249:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.693252:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.693306:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707787201536 00010000:00000200:24.0:1616513089.693317:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688707787201536/t0(0) o400->d2f4efc2-9101-f126-bb32-07d83172a652@10.31.141.111@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.693330:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688707787201536, offset 192 00000400:00000200:24.0:1616513089.693336:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.111@tcp1 00000400:00000200:24.0:1616513089.693343:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.111@tcp1 routed destination 00000400:00000200:24.0:1616513089.693358:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.111@tcp1(10.31.141.111@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.693364:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.693369:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.693372:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.693374:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.693379:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.693385:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.693386:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.693449:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.693456:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.693458:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.693465:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.693470:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2660 00000800:00000200:28.0:1616513089.693473:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.693476:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.693610:0:50509:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.693665:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.693674:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.693682:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.111@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.693690:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.111@tcp1 of length 224 into portal 12 MB=0x5ffdeebc92000 00000400:00000200:14.0:1616513089.693696:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.111@tcp1 of length 224/224 into md 0xf964d519 [1] + 82184 00000400:00000200:14.0:1616513089.693701:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.693704:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513089.693711:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.693714:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.693716:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.693719:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513089.693783:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707787202560 00010000:00000200:22.0:1616513089.693796:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e00000 x1688707787202560/t0(0) o400->5b3acff3-fd4b-bef7-1771-3a6d0546b55d@10.31.141.111@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.693810:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688707787202560, offset 192 00000400:00000200:22.0:1616513089.693817:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.111@tcp1 00000400:00000200:22.0:1616513089.693824:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.111@tcp1 routed destination 00000400:00000200:22.0:1616513089.693839:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.111@tcp1(10.31.141.111@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.693845:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.693851:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.693854:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.693856:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.693861:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.693866:0:50509:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.693877:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.693883:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.693886:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.693888:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.693892:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.693896:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bd38 00000800:00000200:2.0:1616513089.693900:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.696643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.696697:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.696706:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.696715:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.103@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.696724:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.103@tcp1 of length 224 into portal 26 MB=0x5ffe1a0875940 00000400:00000200:26.0:1616513089.696730:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.103@tcp1 of length 224/224 into md 0xf97cd1d9 [1] + 0 00000400:00000200:26.0:1616513089.696735:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2550 00000400:00000200:26.0:1616513089.696737:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.696740:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513089.696744:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2550 00000800:00000200:26.0:1616513089.696747:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.696750:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.696750:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513089.696757:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.696760:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.696761:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.696769:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.103@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.696778:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.103@tcp1 of length 224 into portal 12 MB=0x5ffe1a08759c0 00000400:00000200:14.0:1616513089.696784:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.103@tcp1 of length 224/224 into md 0xf964d519 [1] + 82408 00000400:00000200:14.0:1616513089.696789:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.696792:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513089.696800:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.696803:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.696805:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.696818:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688719409502528 00010000:00000200:24.0:1616513089.696829:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688719409502528/t0(0) o400->6350d6a3-75d8-93d7-c0b4-30d25cb5bf09@10.31.141.103@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.696843:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688719409502528, offset 192 00000400:00000200:24.0:1616513089.696849:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.103@tcp1 00000400:00000200:24.0:1616513089.696856:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.103@tcp1 routed destination 00000100:00000200:22.0:1616513089.696864:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688719409502656 00000400:00000200:24.0:1616513089.696871:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.103@tcp1(10.31.141.103@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:22.0:1616513089.696876:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07500 x1688719409502656/t0(0) o400->22aaa4b0-41fa-072c-fab2-6b4e42c5b17b@10.31.141.103@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513089.696877:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.696883:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.696886:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.696888:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513089.696890:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688719409502656, offset 192 00000800:00000200:24.0:1616513089.696893:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513089.696898:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.103@tcp1 00000800:00000200:11.2:1616513089.696898:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.696899:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513089.696905:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.103@tcp1 routed destination 00000400:00000200:22.0:1616513089.696920:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.103@tcp1(10.31.141.103@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.696927:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.696932:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.696935:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.696937:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.696942:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.696947:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513089.696952:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.696959:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.696962:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.696969:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.696974:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2f68 00000800:00000200:16.0:1616513089.696977:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.696980:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513089.697001:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.697008:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.697011:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513089.697012:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.697017:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.697021:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bba0 00000800:00000200:2.0:1616513089.697025:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.700388:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.700452:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.700461:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.700470:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.112@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.700479:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.112@tcp1 of length 224 into portal 12 MB=0x6007b4fb64900 00000400:00000200:14.0:1616513089.700486:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.112@tcp1 of length 224/224 into md 0xf964d519 [1] + 82632 00000400:00000200:14.0:1616513089.700491:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.700494:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513089.700500:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.700503:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.700505:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.700510:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.700516:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513089.700563:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689379478587648 00000800:00000200:28.0:1616513089.700572:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513089.700575:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07980 x1689379478587648/t0(0) o400->e414d7be-7865-acd1-1d1a-fbf6e46447a0@10.31.132.112@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513089.700580:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.700589:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.112@tcp1 : PUT - for me 00000100:00000200:22.0:1616513089.700590:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689379478587648, offset 192 00000400:00000200:22.0:1616513089.700596:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.112@tcp1 00000400:00000200:28.0:1616513089.700597:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.112@tcp1 of length 224 into portal 26 MB=0x6007b4fb64440 00000400:00000200:28.0:1616513089.700603:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.112@tcp1 of length 224/224 into md 0xf97cd1e9 [1] + 0 00000400:00000200:22.0:1616513089.700603:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.112@tcp1 routed destination 00000400:00000200:28.0:1616513089.700607:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2bb0 00000400:00000200:28.0:1616513089.700610:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.700612:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.700616:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2bb0 00000400:00000200:22.0:1616513089.700621:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.112@tcp1(10.31.132.112@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513089.700625:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.700626:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513089.700627:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.700629:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.700630:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513089.700633:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.700636:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513089.700638:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.700642:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.700647:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.700680:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689379478586432 00010000:00000200:24.0:1616513089.700690:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1689379478586432/t0(0) o400->8e942892-5d0e-2216-473c-582ced77c7a8@10.31.132.112@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513089.700700:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513089.700705:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689379478586432, offset 192 00000800:00000200:16.0:1616513089.700708:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.700710:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513089.700712:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513089.700713:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.112@tcp1 00000400:00000200:16.0:1616513089.700717:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513089.700720:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.112@tcp1 routed destination 00000400:00000200:16.0:1616513089.700721:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b5c8 00000800:00000200:16.0:1616513089.700726:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513089.700735:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.112@tcp1(10.31.132.112@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.700742:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513089.700748:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.700751:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513089.700753:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513089.700758:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.700763:0:50510:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513089.700764:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513089.700817:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.700823:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.700826:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.700832:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.700837:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2e58 00000800:00000200:26.0:1616513089.700840:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.700843:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.705988:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513089.705999:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513089.706044:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.706053:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.706062:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.242@tcp1 : PUT - for me 00000800:00000200:20.0:1616513089.706063:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513089.706071:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.242@tcp1 of length 224 into portal 26 MB=0x5fff7c4168740 00000800:00000200:20.0:1616513089.706072:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:18.0:1616513089.706078:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.242@tcp1 of length 224/224 into md 0xf97cd209 [1] + 0 00000400:00000200:20.0:1616513089.706081:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.242@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.706083:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3100 00000400:00000200:18.0:1616513089.706085:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.706088:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513089.706090:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.242@tcp1 of length 224 into portal 12 MB=0x5fff7c4168a00 00000400:00000200:18.0:1616513089.706092:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3100 00000800:00000200:18.0:1616513089.706095:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513089.706097:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.242@tcp1 of length 224/224 into md 0x865be291 [1] + 13328 00000800:00000200:14.0:1616513089.706097:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.706098:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.706101:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513089.706102:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.706105:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513089.706112:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.706115:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513089.706117:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.706119:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.706169:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688814495369024 00000100:00000200:0.0:1616513089.706178:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688814495369728 00010000:00000200:24.0:1616513089.706180:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688814495369024/t0(0) o400->a97538c5-2765-e8e8-002e-b3c5070c2766@10.31.141.242@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:0.0:1616513089.706190:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad1f80 x1688814495369728/t0(0) o400->45ee2d6d-6f02-dae7-8a0f-38ce03f9208e@10.31.141.242@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513089.706194:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688814495369024, offset 192 00000400:00000200:24.0:1616513089.706201:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.242@tcp1 00000100:00000200:0.0:1616513089.706204:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688814495369728, offset 192 00000400:00000200:24.0:1616513089.706208:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.242@tcp1 routed destination 00000400:00000200:0.0:1616513089.706211:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.242@tcp1 00000400:00000200:24.0:1616513089.706222:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.242@tcp1(10.31.141.242@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.706229:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513089.706239:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.242@tcp1 routed destination 00000800:00000200:24.0:1616513089.706247:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.706250:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.706252:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513089.706254:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.242@tcp1(10.31.141.242@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.706257:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.706261:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.706261:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:24.0:1616513089.706263:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513089.706267:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.706270:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513089.706271:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513089.706272:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.706276:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513089.706277:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513089.706278:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513089.706281:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513089.706281:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513089.706284:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.706289:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3b18 00000800:00000200:28.0:1616513089.706292:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513089.706334:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.706340:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.706343:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.706349:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.706353:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42908 00000800:00000200:26.0:1616513089.706356:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.706359:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.718212:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.718267:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.718276:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.718284:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.127@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.718293:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.127@tcp1 of length 224 into portal 26 MB=0x5ffd3d6636c00 00000400:00000200:14.0:1616513089.718299:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.127@tcp1 of length 224/224 into md 0xf97cd219 [1] + 0 00000400:00000200:14.0:1616513089.718303:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2330 00000400:00000200:14.0:1616513089.718306:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.718308:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.718312:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2330 00000800:00000200:14.0:1616513089.718315:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.718318:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513089.718319:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.718322:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.718373:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660183575552 00010000:00000200:24.0:1616513089.718384:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688660183575552/t0(0) o400->3fe497c5-844c-e87e-658e-76a3eb231264@10.31.129.127@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513089.718396:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513089.718397:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660183575552, offset 192 00000400:00000200:24.0:1616513089.718403:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.127@tcp1 00000400:00000200:24.0:1616513089.718410:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.127@tcp1 routed destination 00000400:00000200:24.0:1616513089.718424:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.127@tcp1(10.31.129.127@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513089.718454:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.718456:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.718462:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513089.718463:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.718464:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.718466:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513089.718471:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:20.0:1616513089.718472:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.127@tcp1 : PUT - for me 00000100:00000200:24.0:1616513089.718478:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513089.718481:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.127@tcp1 of length 224 into portal 12 MB=0x5ffd3d6636f00 00000400:00000200:20.0:1616513089.718490:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.127@tcp1 of length 224/224 into md 0x865be291 [1] + 13552 00000400:00000200:20.0:1616513089.718495:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.718497:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513089.718504:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513089.718508:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513089.718509:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513089.718512:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.718515:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.718518:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513089.718523:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.718527:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f25d8 00000800:00000200:20.0:1616513089.718530:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.718533:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.718568:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660183576320 00010000:00000200:0.0:1616513089.718581:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad6300 x1688660183576320/t0(0) o400->c309ce47-9cad-fbbb-c870-c93dca6593f1@10.31.129.127@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.718595:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660183576320, offset 192 00000400:00000200:0.0:1616513089.718601:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.127@tcp1 00000400:00000200:0.0:1616513089.718609:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.127@tcp1 routed destination 00000400:00000200:0.0:1616513089.718624:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.127@tcp1(10.31.129.127@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.718631:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.718636:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.718639:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.718643:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.718648:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.718654:0:50512:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.718717:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.718725:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.718728:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513089.718734:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.718738:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42440 00000800:00000200:26.0:1616513089.718742:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513089.718744:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.734355:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513089.734409:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.734419:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513089.734428:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.99@tcp1 : PUT - for me 00000400:00000200:14.0:1616513089.734449:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.99@tcp1 of length 224 into portal 26 MB=0x5ffd400197380 00000400:00000200:14.0:1616513089.734456:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.99@tcp1 of length 224/224 into md 0xf97cd249 [1] + 0 00000400:00000200:14.0:1616513089.734461:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3210 00000800:00000200:18.0:1616513089.734463:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513089.734465:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.734468:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513089.734472:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3210 00000800:00000200:14.0:1616513089.734475:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.734478:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513089.734480:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.734540:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660883370880 00010000:00000200:24.0:1616513089.734551:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688660883370880/t0(0) o400->63382f0a-7eff-2a55-2e32-97ad3219705a@10.31.128.99@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513089.734559:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513089.734565:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660883370880, offset 192 00000400:00000200:24.0:1616513089.734571:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.99@tcp1 00000400:00000200:24.0:1616513089.734579:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.99@tcp1 routed destination 00000400:00000200:24.0:1616513089.734594:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.99@tcp1(10.31.128.99@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.734601:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.734606:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.734609:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.734611:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513089.734614:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513089.734616:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513089.734622:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:24.0:1616513089.734623:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.734623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513089.734630:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.189@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.734639:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.189@tcp1 of length 224 into portal 26 MB=0x5ffdc444d06c0 00000400:00000200:26.0:1616513089.734645:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.189@tcp1 of length 224/224 into md 0xf97cd269 [1] + 0 00000400:00000200:26.0:1616513089.734650:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2908 00000400:00000200:26.0:1616513089.734652:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.734655:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513089.734659:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2908 00000800:00000200:26.0:1616513089.734662:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.734665:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.734666:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.734669:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513089.734672:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696387339968 00010000:00000200:24.0:1616513089.734677:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688696387339968/t0(0) o400->8de49965-e62d-872a-2b29-574a87e46130@10.31.141.189@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513089.734680:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513089.734682:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513089.734688:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513089.734688:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.189@tcp1 : PUT - for me 00000100:00000200:24.0:1616513089.734689:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696387339968, offset 192 00000800:00000200:18.0:1616513089.734690:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513089.734692:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:24.0:1616513089.734694:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.189@tcp1 00000800:00000200:28.0:1616513089.734696:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513089.734697:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.189@tcp1 of length 224 into portal 12 MB=0x5ffdc444d08c0 00000400:00000200:28.0:1616513089.734698:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513089.734700:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.189@tcp1 routed destination 00000400:00000200:18.0:1616513089.734700:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.99@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.734704:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.189@tcp1 of length 224/224 into md 0xf964d519 [1] + 82856 00000400:00000200:28.0:1616513089.734705:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.734711:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3ba0 00000400:00000200:2.0:1616513089.734711:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513089.734713:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.189@tcp1(10.31.141.189@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513089.734714:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:2.0:1616513089.734714:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.734719:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513089.734719:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513089.734721:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513089.734722:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.99@tcp1 of length 224 into portal 12 MB=0x5ffd4001976c0 00000800:00000200:24.0:1616513089.734724:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513089.734724:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513089.734726:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.734726:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.734729:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.99@tcp1 of length 224/224 into md 0xf964d519 [1] + 83080 00000800:00000200:24.0:1616513089.734731:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513089.734734:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.734734:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.734737:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:24.0:1616513089.734739:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513089.734739:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.734745:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513089.734748:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513089.734750:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:26.0:1616513089.734752:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513089.734752:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513089.734754:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.734755:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.734759:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.734763:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3430 00000800:00000200:26.0:1616513089.734766:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513089.734774:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696387340480 00000100:00000200:22.0:1616513089.734777:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660883371712 00010000:00000200:30.0:1616513089.734786:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc53ca880 x1688696387340480/t0(0) o400->69218bf9-fe32-61fd-ca80-bf143c510219@10.31.141.189@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:22.0:1616513089.734790:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120ec00 x1688660883371712/t0(0) o400->9b6f64ce-2bd9-d561-1efb-fc879e932b45@10.31.128.99@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513089.734800:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696387340480, offset 192 00000400:00000200:30.0:1616513089.734806:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.189@tcp1 00000100:00000200:22.0:1616513089.734809:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660883371712, offset 192 00000400:00000200:30.0:1616513089.734813:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.189@tcp1 routed destination 00000400:00000200:22.0:1616513089.734815:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.99@tcp1 00000400:00000200:22.0:1616513089.734823:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.99@tcp1 routed destination 00000400:00000200:30.0:1616513089.734828:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.189@tcp1(10.31.141.189@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513089.734834:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513089.734838:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.99@tcp1(10.31.128.99@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513089.734845:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.734845:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513089.734848:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513089.734850:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.734851:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.734853:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513089.734854:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513089.734855:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.734860:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513089.734865:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513089.734866:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.734876:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.734880:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.734882:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513089.734884:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.734888:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.734891:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b760 00000800:00000200:2.0:1616513089.734895:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513089.734921:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.734927:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.734930:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513089.734937:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.734941:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efcb0 00000800:00000200:20.0:1616513089.734945:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.734948:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.763693:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.763750:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.763760:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.763770:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.195@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.763779:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.195@tcp1 of length 224 into portal 26 MB=0x5ffd3640f3cc0 00000400:00000200:18.0:1616513089.763786:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.195@tcp1 of length 224/224 into md 0xf97cd279 [1] + 0 00000400:00000200:18.0:1616513089.763791:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2198 00000400:00000200:18.0:1616513089.763794:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.763796:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513089.763801:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2198 00000800:00000200:18.0:1616513089.763804:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.763807:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.763808:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.763811:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.763816:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.763822:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.195@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.763828:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.195@tcp1 of length 224 into portal 12 MB=0x5ffd3640f3d40 00000400:00000200:18.0:1616513089.763833:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.195@tcp1 of length 224/224 into md 0xf964d519 [1] + 83304 00000400:00000200:18.0:1616513089.763837:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.763839:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.763845:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.763847:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.763848:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.763851:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.763865:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658265455808 00010000:00000200:24.0:1616513089.763875:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688658265455808/t0(0) o400->0b7bce4f-509b-dbf1-34ee-5cb1242ddb73@10.31.138.195@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.763889:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658265455808, offset 192 00000400:00000200:24.0:1616513089.763895:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.195@tcp1 00000400:00000200:24.0:1616513089.763902:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.195@tcp1 routed destination 00000100:00000200:22.0:1616513089.763914:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658265455936 00000400:00000200:24.0:1616513089.763916:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.195@tcp1(10.31.138.195@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.763923:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:22.0:1616513089.763927:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120c380 x1688658265455936/t0(0) o400->bd37682f-d1d5-0757-2288-bba63adc6dda@10.31.138.195@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513089.763928:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.763931:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.763933:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.763938:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513089.763942:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658265455936, offset 192 00000100:00000200:24.0:1616513089.763944:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.763944:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513089.763948:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.195@tcp1 00000400:00000200:22.0:1616513089.763955:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.195@tcp1 routed destination 00000400:00000200:22.0:1616513089.763970:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.195@tcp1(10.31.138.195@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.763976:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.763982:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.763985:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.763990:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.763995:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.763999:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.764000:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.764007:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.764010:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.764016:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.764022:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2dd0 00000800:00000200:28.0:1616513089.764025:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.764028:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.764034:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.764036:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.764038:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.764040:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.764042:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513089.764055:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.764062:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.764065:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513089.764067:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.764072:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.764076:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bc28 00000800:00000200:2.0:1616513089.764080:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513089.770520:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927792, portal 4 00000100:00000200:10.0:1616513089.770525:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725927792, offset 0 00000400:00000200:10.0:1616513089.770532:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:10.0:1616513089.770540:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:10.0:1616513089.770552:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513089.770559:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:10.0:1616513089.770565:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513089.770568:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:10.0:1616513089.770570:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:10.0:1616513089.770574:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513089.770621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513089.770678:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:16.0:1616513089.770685:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:16.0:1616513089.770688:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.770695:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:16.0:1616513089.770701:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf87500 x1693067725927792/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513089.770713:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82b28 00000800:00000200:16.0:1616513089.770719:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513089.770722:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513089.770824:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:0.0:1616513089.770889:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513089.770899:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:0.0:1616513089.770908:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:0.0:1616513089.770916:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627570 00000400:00000200:0.0:1616513089.770923:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866da2a1 [1] + 192 00000400:00000200:0.0:1616513089.770928:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513089.770933:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf87500 x1693067725927792/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513089.770942:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513089.770948:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513089.770951:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513089.770952:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:10.0:1616513089.770956:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf87500 x1693067725927792/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513089.770969:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf87500 x1693067725927792/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513089.770980:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83540 00000800:00000200:7.2:1616513089.773485:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.773541:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.773551:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.773559:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.194@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.773568:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.194@tcp1 of length 224 into portal 26 MB=0x60182c09c9c80 00000400:00000200:18.0:1616513089.773575:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.194@tcp1 of length 224/224 into md 0xf97cd299 [1] + 0 00000400:00000200:18.0:1616513089.773580:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f23b8 00000400:00000200:18.0:1616513089.773584:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.773587:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513089.773591:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f23b8 00000800:00000200:18.0:1616513089.773594:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.773595:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.773597:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.773599:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.773655:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510949129344 00010000:00000200:24.0:1616513089.773666:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1690510949129344/t0(0) o400->4b1cc56a-9fa1-fff7-b276-6890dca46fa2@10.31.136.194@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.773680:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510949129344, offset 192 00000400:00000200:24.0:1616513089.773687:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.194@tcp1 00000400:00000200:24.0:1616513089.773694:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.194@tcp1 routed destination 00000400:00000200:24.0:1616513089.773709:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.194@tcp1(10.31.136.194@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.773716:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.773722:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.773724:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.773727:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.773731:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513089.773738:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.773738:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.773791:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.773798:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.773801:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.773807:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.773813:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3430 00000800:00000200:16.0:1616513089.773816:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.773819:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.773874:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.773928:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.773938:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.773946:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.194@tcp1 : PUT - for me 00000400:00000200:26.0:1616513089.773955:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.194@tcp1 of length 224 into portal 12 MB=0x60182c09c9e00 00000400:00000200:26.0:1616513089.773962:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.194@tcp1 of length 224/224 into md 0xf964d519 [1] + 83528 00000400:00000200:26.0:1616513089.773967:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.773970:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513089.773977:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.773983:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.773983:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513089.773987:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513089.774041:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510949129728 00010000:00000200:22.0:1616513089.774054:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695580 x1690510949129728/t0(0) o400->706db541-2a00-0f18-68fd-b5bc88b6ae3e@10.31.136.194@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.774068:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510949129728, offset 192 00000400:00000200:22.0:1616513089.774074:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.194@tcp1 00000400:00000200:22.0:1616513089.774082:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.194@tcp1 routed destination 00000400:00000200:22.0:1616513089.774096:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.194@tcp1(10.31.136.194@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.774103:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.774108:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.774111:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.774113:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.774118:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.774123:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513089.774133:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513089.774139:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513089.774141:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.774143:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513089.774148:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513089.774152:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b078 00000800:00000200:14.0:1616513089.774155:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.776664:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513089.776720:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.776729:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513089.776738:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.240@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.776747:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.240@tcp1 of length 224 into portal 26 MB=0x604f5fa14c240 00000400:00000200:2.0:1616513089.776753:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.240@tcp1 of length 224/224 into md 0xf97cd2a9 [1] + 0 00000400:00000200:2.0:1616513089.776758:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f27f8 00000400:00000200:2.0:1616513089.776761:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.776764:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513089.776768:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f27f8 00000800:00000200:2.0:1616513089.776771:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.776773:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.776774:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.776777:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.776831:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304369427008 00010000:00000200:24.0:1616513089.776842:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1694304369427008/t0(0) o400->8e115b9d-a3ce-2547-0ef3-b435e07fd322@10.31.140.240@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.776858:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694304369427008, offset 192 00000400:00000200:24.0:1616513089.776864:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.240@tcp1 00000800:00000200:7.2:1616513089.776864:0:50518:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513089.776872:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.240@tcp1 routed destination 00000400:00000200:24.0:1616513089.776886:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.240@tcp1(10.31.140.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.776893:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.776898:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.776901:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.776903:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.776908:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.776914:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.776915:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513089.776917:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.776925:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.776933:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.240@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.776941:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.240@tcp1 of length 224 into portal 12 MB=0x604f5fa14c340 00000400:00000200:18.0:1616513089.776947:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.240@tcp1 of length 224/224 into md 0xf964d519 [1] + 83752 00000400:00000200:18.0:1616513089.776952:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.776954:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.776960:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.776963:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.776965:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513089.776968:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513089.776968:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513089.776975:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.776978:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513089.776984:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.776990:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2908 00000800:00000200:20.0:1616513089.776993:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.776996:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513089.777039:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304369427264 00010000:00000200:22.0:1616513089.777050:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461209f80 x1694304369427264/t0(0) o400->a2976fff-6c72-613e-e4a6-50cdf4069ecd@10.31.140.240@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513089.777063:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694304369427264, offset 192 00000400:00000200:22.0:1616513089.777069:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.240@tcp1 00000400:00000200:22.0:1616513089.777075:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.240@tcp1 routed destination 00000400:00000200:22.0:1616513089.777089:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.240@tcp1(10.31.140.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.777095:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.777101:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.777103:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.777105:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.777109:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.777114:0:50518:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.777124:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513089.777129:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513089.777131:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513089.777133:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.777137:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513089.777141:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a5d8 00000800:00000200:26.0:1616513089.777145:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513089.786676:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927840, portal 10 00000100:00000200:30.0:1616513089.786681:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 456 bytes to portal 23, xid 1693067725927840, offset 0 00000100:00000200:10.0:1616513089.786681:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927808, portal 10 00000400:00000200:30.0:1616513089.786688:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000100:00000200:10.0:1616513089.786690:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 456 bytes to portal 23, xid 1693067725927808, offset 0 00000400:00000200:30.0:1616513089.786694:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:10.0:1616513089.786697:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:30.0:1616513089.786706:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513089.786713:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000100:00000200:6.0:1616513089.786713:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927824, portal 10 00000400:00000200:10.0:1616513089.786714:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000100:00000200:6.0:1616513089.786718:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 456 bytes to portal 23, xid 1693067725927824, offset 0 00000400:00000200:6.0:1616513089.786725:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:10.0:1616513089.786727:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513089.786733:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:30.0:1616513089.786734:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000100:00000200:18.0:1616513089.786735:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927872, portal 10 00000400:00000200:6.0:1616513089.786735:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000100:00000200:22.0:1616513089.786736:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927888, portal 10 00000800:00000200:30.0:1616513089.786737:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000100:00000200:14.0:1616513089.786737:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725927856, portal 10 00000800:00000200:30.0:1616513089.786739:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:18.0:1616513089.786740:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 456 bytes to portal 23, xid 1693067725927872, offset 0 00000100:00000200:22.0:1616513089.786741:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 456 bytes to portal 23, xid 1693067725927888, offset 0 00000100:00000200:14.0:1616513089.786743:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 456 bytes to portal 23, xid 1693067725927856, offset 0 00000800:00000200:30.0:1616513089.786744:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:18.0:1616513089.786747:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:6.0:1616513089.786748:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513089.786753:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000800:00000200:6.0:1616513089.786754:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000400:00000200:18.0:1616513089.786755:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000800:00000200:10.0:1616513089.786755:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513089.786758:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000800:00000200:10.0:1616513089.786758:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:10.0:1616513089.786760:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:10.0:1616513089.786765:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:18.0:1616513089.786768:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000100:00000200:30.0:1616513089.786773:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1693067725927968, portal 18 00000800:00000200:18.0:1616513089.786774:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000400:00000200:22.0:1616513089.786775:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000100:00000200:30.0:1616513089.786776:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1693067725927968, offset 0 00000400:00000200:30.0:1616513089.786780:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000800:00000200:31.2:1616513089.786790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (22)++ 00000400:00000200:22.0:1616513089.786790:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513089.786796:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000400:00000200:14.0:1616513089.786797:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000800:00000200:6.0:1616513089.786797:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513089.786800:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (23)++ 00000800:00000200:6.0:1616513089.786803:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (24)++ 00000800:00000200:6.0:1616513089.786809:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (25)-- 00000400:00000200:14.0:1616513089.786810:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513089.786817:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000800:00000200:14.0:1616513089.786817:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 456 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:18.0:1616513089.786818:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000100:00000200:2.0:1616513089.786818:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1693067725927920, portal 18 00000100:00000200:10.0:1616513089.786819:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1693067725927936, portal 18 00000100:00000200:26.0:1616513089.786820:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1693067725927904, portal 18 00000800:00000200:18.0:1616513089.786821:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (24)++ 00000100:00000200:10.0:1616513089.786822:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1693067725927936, offset 0 00000800:00000200:18.0:1616513089.786823:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (25)++ 00000100:00000200:2.0:1616513089.786823:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1693067725927920, offset 0 00000400:00000200:10.0:1616513089.786827:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000800:00000200:18.0:1616513089.786828:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (26)-- 00000400:00000200:30.0:1616513089.786829:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513089.786829:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1693067725927904, offset 0 00000400:00000200:2.0:1616513089.786833:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000800:00000200:30.0:1616513089.786834:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000400:00000200:10.0:1616513089.786835:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000800:00000200:22.0:1616513089.786836:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513089.786836:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513089.786847:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (25)++ 00000800:00000200:14.0:1616513089.786850:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (26)++ 00000800:00000200:18.0:1616513089.786851:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (26)++ 00000800:00000200:22.0:1616513089.786852:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (27)++ 00000400:00000200:26.0:1616513089.786853:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000800:00000200:22.0:1616513089.786854:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (29)++ 00000800:00000200:14.0:1616513089.786854:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (29)-- 00000100:00000200:6.0:1616513089.786855:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1693067725927952, portal 18 00000400:00000200:10.0:1616513089.786856:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000100:00000200:6.0:1616513089.786857:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1693067725927952, offset 0 00000800:00000200:22.0:1616513089.786859:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (29)-- 00000800:00000200:18.0:1616513089.786859:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (29)-- 00000400:00000200:18.0:1616513089.786861:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513089.786861:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000400:00000200:6.0:1616513089.786861:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000800:00000200:30.0:1616513089.786862:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513089.786863:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000800:00000200:30.0:1616513089.786864:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (27)++ 00000800:00000200:30.0:1616513089.786865:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (28)++ 00000400:00000200:18.0:1616513089.786866:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000800:00000200:30.0:1616513089.786868:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (29)-- 00000400:00000200:26.0:1616513089.786871:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000100:00000200:18.0:1616513089.786871:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc53ca400 x1693067725927840/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513089.786876:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (28)++ 00000400:00000200:6.0:1616513089.786877:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000800:00000200:22.0:1616513089.786882:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (29)-- 00000800:00000200:10.0:1616513089.786882:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513089.786884:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.786884:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee4c8 00000800:00000200:10.0:1616513089.786884:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (28)++ 00000100:00000200:14.0:1616513089.786887:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1693067725927984, portal 18 00000800:00000200:2.0:1616513089.786887:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (28)++ 00000800:00000200:10.0:1616513089.786888:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (29)++ 00000400:00000200:22.0:1616513089.786889:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000800:00000200:18.0:1616513089.786889:0:6471:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000100:00000200:14.0:1616513089.786890:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1693067725927984, offset 0 00000800:00000200:10.0:1616513089.786892:0:6498:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000800:00000200:10.0:1616513089.786893:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (31)-- 00000100:00000200:22.0:1616513089.786894:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc53cd580 x1693067725927808/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513089.786894:0:6471:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513089.786896:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (30)++ 00000800:00000200:18.0:1616513089.786896:0:6471:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (30)++ 00000800:00000200:2.0:1616513089.786896:0:6472:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000800:00000200:2.0:1616513089.786897:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (32)-- 00000800:00000200:18.0:1616513089.786898:0:6471:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (32)++ 00000400:00000200:2.0:1616513089.786900:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.786901:0:6471:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000800:00000200:18.0:1616513089.786902:0:6471:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (32)-- 00000800:00000200:30.0:1616513089.786904:0:6469:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000400:00000200:22.0:1616513089.786904:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83100 00000800:00000200:18.0:1616513089.786904:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (31)-- 00000400:00000200:2.0:1616513089.786905:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000800:00000200:30.0:1616513089.786906:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (30)-- 00000800:00000200:18.0:1616513089.786906:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (30)++ 00000400:00000200:30.0:1616513089.786908:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.786909:0:6471:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000400:00000200:14.0:1616513089.786909:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000800:00000200:22.0:1616513089.786910:0:6470:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:18.0:1616513089.786910:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (30)-- 00000400:00000200:18.0:1616513089.786912:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513089.786912:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:30.0:1616513089.786913:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000100:00000200:2.0:1616513089.786913:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc53c9200 x1693067725927824/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513089.786916:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000800:00000200:22.0:1616513089.786918:0:6470:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513089.786921:0:6470:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (29)++ 00000800:00000200:22.0:1616513089.786922:0:6470:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (30)++ 00000800:00000200:22.0:1616513089.786925:0:6470:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000400:00000200:2.0:1616513089.786925:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00220 00000800:00000200:22.0:1616513089.786926:0:6470:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (31)-- 00000100:00000200:30.0:1616513089.786929:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc53cc380 x1693067725927872/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513089.786929:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (30)-- 00000800:00000200:22.0:1616513089.786931:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (29)++ 00000800:00000200:2.0:1616513089.786931:0:6472:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:22.0:1616513089.786934:0:6470:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000800:00000200:22.0:1616513089.786935:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (30)-- 00000400:00000200:22.0:1616513089.786937:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513089.786937:0:6472:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513089.786939:0:6472:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (29)++ 00000400:00000200:22.0:1616513089.786941:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000800:00000200:2.0:1616513089.786941:0:6472:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (30)++ 00000800:00000200:2.0:1616513089.786945:0:6472:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000800:00000200:2.0:1616513089.786946:0:6472:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (31)-- 00000800:00000200:2.0:1616513089.786948:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (30)-- 00000400:00000200:30.0:1616513089.786949:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc908 00000800:00000200:2.0:1616513089.786950:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (29)++ 00000100:00000200:18.0:1616513089.786953:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc53cd100 x1693067725927856/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513089.786953:0:6472:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000800:00000200:30.0:1616513089.786954:0:6469:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 328 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:2.0:1616513089.786954:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (30)-- 00000400:00000200:2.0:1616513089.786956:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513089.786958:0:6469:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513089.786961:0:6469:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (29)++ 00000400:00000200:2.0:1616513089.786961:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000800:00000200:30.0:1616513089.786962:0:6469:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (30)++ 00000400:00000200:18.0:1616513089.786962:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47566e8 00000800:00000200:30.0:1616513089.786965:0:6469:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000800:00000200:18.0:1616513089.786965:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (31)-- 00000800:00000200:30.0:1616513089.786966:0:6469:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (30)-- 00000100:00000200:22.0:1616513089.786966:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc53ce780 x1693067725927888/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 2 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513089.786967:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (30)++ 00000800:00000200:30.0:1616513089.786968:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (30)-- 00000800:00000200:30.0:1616513089.786971:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (29)++ 00000800:00000200:18.0:1616513089.786971:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.234@o2ib1 00000800:00000200:30.0:1616513089.786975:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.234@o2ib1 00000400:00000200:22.0:1616513089.786975:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4abb0 00000800:00000200:22.0:1616513089.786977:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (30)-- 00000800:00000200:18.0:1616513089.786977:0:6471:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.234@o2ib1: no credits 00000800:00000200:18.0:1616513089.786978:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (30)++ 00000800:00000200:30.0:1616513089.786979:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (30)++ 00000100:00000200:2.0:1616513089.786979:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d6c00 x1693067725927968/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513089.786980:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (30)++ 00000800:00000200:18.0:1616513089.786980:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (30)-- 00000800:00000200:30.0:1616513089.786981:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (31)-- 00000800:00000200:18.0:1616513089.786981:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (31)-- 00000800:00000200:30.0:1616513089.786983:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (29)-- 00000800:00000200:22.0:1616513089.786983:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000800:00000200:18.0:1616513089.786985:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (28)++ 00000800:00000200:18.0:1616513089.786989:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (29)-- 00000400:00000200:2.0:1616513089.786989:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee660 00000400:00000200:22.0:1616513089.786990:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513089.786991:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513089.786991:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (28)-- 00000800:00000200:30.0:1616513089.786993:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (27)++ 00000400:00000200:18.0:1616513089.786995:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:22.0:1616513089.786996:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 192 into portal 18 MB=0x603d60c627620 00000800:00000200:30.0:1616513089.786998:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (28)-- 00000100:00000200:18.0:1616513089.786999:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d4380 x1693067725927936/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513089.787000:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513089.787003:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (27)++ 00000400:00000200:30.0:1616513089.787004:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000800:00000200:2.0:1616513089.787005:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (28)-- 00000400:00000200:18.0:1616513089.787007:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82110 00000400:00000200:2.0:1616513089.787007:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.787010:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (27)-- 00000800:00000200:18.0:1616513089.787011:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (26)++ 00000400:00000200:22.0:1616513089.787012:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 12 from 12345-10.242.46.234@o2ib1 of length 192/192 into md 0xf97cdaa9 [1] + 192 00000400:00000200:2.0:1616513089.787012:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000800:00000200:18.0:1616513089.787013:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (27)-- 00000400:00000200:18.0:1616513089.787015:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513089.787016:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513089.787018:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d2400 x1693067725927920/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513089.787020:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:30.0:1616513089.787026:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed210 00000800:00000200:30.0:1616513089.787029:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (26)-- 00000100:00000200:2.0:1616513089.787030:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d3a80 x1693067725927904/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513089.787031:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)++ 00000800:00000200:30.0:1616513089.787032:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (26)-- 00000400:00000200:30.0:1616513089.787034:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513089.787038:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.787039:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af3a8 00000800:00000200:2.0:1616513089.787042:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)-- 00000100:00000200:22.0:1616513089.787043:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d6c00 x1693067725927968/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513089.787045:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (24)++ 00000800:00000200:2.0:1616513089.787048:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:2.0:1616513089.787055:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000800:00000200:22.0:1616513089.787056:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (25)++ 00000100:00000200:18.0:1616513089.787057:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d0480 x1693067725927952/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513089.787058:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (26)-- 00000800:00000200:22.0:1616513089.787060:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)-- 00000800:00000200:22.0:1616513089.787061:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (24)++ 00000400:00000200:2.0:1616513089.787061:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 464 into portal 10 MB=0x603d60c6275a0 00000800:00000200:22.0:1616513089.787065:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.234@o2ib1 00000400:00000200:18.0:1616513089.787065:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00110 00000800:00000200:18.0:1616513089.787067:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)-- 00000100:00000200:30.0:1616513089.787068:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d3600 x1693067725927984/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513089.787069:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (24)++ 00000400:00000200:22.0:1616513089.787071:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000800:00000200:18.0:1616513089.787072:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:30.0:1616513089.787076:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47576d8 00000400:00000200:22.0:1616513089.787077:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 464 into portal 10 MB=0x603d60c627590 00000800:00000200:30.0:1616513089.787078:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)-- 00000400:00000200:18.0:1616513089.787079:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000800:00000200:30.0:1616513089.787080:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (24)++ 00000100:00000200:6.0:1616513089.787080:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d6c00 x1693067725927968/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513089.787084:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:18.0:1616513089.787085:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 464 into portal 10 MB=0x603d60c627580 00000100:00000200:6.0:1616513089.787089:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d6c00 x1693067725927968/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513089.787090:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:6.0:1616513089.787095:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efba0 00000400:00000200:30.0:1616513089.787096:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 464 into portal 10 MB=0x603d60c6275b0 00000400:00000200:2.0:1616513089.787099:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.242.46.234@o2ib1 of length 464/464 into md 0xf97cd9e9 [1] + 192 00000400:00000200:2.0:1616513089.787104:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513089.787106:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.242.46.234@o2ib1 of length 464/464 into md 0xf97cda29 [1] + 192 00000400:00000200:22.0:1616513089.787109:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.787111:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.242.46.234@o2ib1 of length 464/464 into md 0xf97cd9f9 [1] + 192 00000400:00000200:18.0:1616513089.787116:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513089.787117:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.242.46.234@o2ib1 of length 464/464 into md 0xf97cda69 [1] + 192 00000400:00000200:30.0:1616513089.787122:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.787123:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc53ca400 x1693067725927840/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513089.787135:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (25)++ 00000100:00000200:22.0:1616513089.787136:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc53c9200 x1693067725927824/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513089.787138:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (26)-- 00000800:00000200:2.0:1616513089.787139:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)-- 00000800:00000200:2.0:1616513089.787141:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (24)++ 00000800:00000200:2.0:1616513089.787145:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000800:00000200:22.0:1616513089.787147:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (25)++ 00000800:00000200:22.0:1616513089.787149:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (26)-- 00000100:00000200:18.0:1616513089.787149:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc53cd580 x1693067725927808/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513089.787150:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)-- 00000800:00000200:22.0:1616513089.787152:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (24)++ 00000400:00000200:2.0:1616513089.787152:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000800:00000200:22.0:1616513089.787155:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:2.0:1616513089.787158:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 464 into portal 10 MB=0x603d60c6275c0 00000400:00000200:22.0:1616513089.787161:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000800:00000200:18.0:1616513089.787161:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (25)++ 00000100:00000200:30.0:1616513089.787162:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc53cd100 x1693067725927856/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513089.787164:0:6471:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (26)++ 00000400:00000200:22.0:1616513089.787166:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 464 into portal 10 MB=0x603d60c6275d0 00000800:00000200:18.0:1616513089.787166:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (27)-- 00000800:00000200:18.0:1616513089.787168:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (26)-- 00000800:00000200:18.0:1616513089.787170:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)++ 00000800:00000200:18.0:1616513089.787173:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000800:00000200:30.0:1616513089.787175:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (26)++ 00000100:00000200:6.0:1616513089.787176:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc53ca400 x1693067725927840/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513089.787177:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (27)-- 00000800:00000200:30.0:1616513089.787178:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (26)-- 00000400:00000200:18.0:1616513089.787179:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000800:00000200:30.0:1616513089.787180:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)++ 00000800:00000200:30.0:1616513089.787182:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (26)-- 00000800:00000200:30.0:1616513089.787183:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)-- 00000100:00000200:6.0:1616513089.787183:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc53ca400 x1693067725927840/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513089.787184:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 192 into portal 18 MB=0x603d60c6275f0 00000800:00000200:30.0:1616513089.787185:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (24)++ 00000800:00000200:30.0:1616513089.787188:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.234@o2ib1 00000400:00000200:6.0:1616513089.787189:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef760 00000100:00000200:10.0:1616513089.787193:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc53c9200 x1693067725927824/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513089.787194:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513089.787199:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 192 into portal 18 MB=0x603d60c627600 00000100:00000200:10.0:1616513089.787206:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc53c9200 x1693067725927824/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513089.787217:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000400:00000200:2.0:1616513089.787221:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.242.46.234@o2ib1 of length 464/464 into md 0xf97cda49 [1] + 192 00000400:00000200:2.0:1616513089.787225:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.787228:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc53cd580 x1693067725927808/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513089.787241:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc53cd580 x1693067725927808/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513089.787262:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83540 00000400:00000200:22.0:1616513089.787267:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index a from 12345-10.242.46.234@o2ib1 of length 464/464 into md 0xf97cda59 [1] + 192 00000400:00000200:22.0:1616513089.787270:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513089.787273:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc53cd100 x1693067725927856/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513089.787285:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc53cd100 x1693067725927856/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513089.787295:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757980 00000400:00000200:18.0:1616513089.787299:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 12 from 12345-10.242.46.234@o2ib1 of length 192/192 into md 0xf97cdac9 [1] + 192 00000400:00000200:18.0:1616513089.787302:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513089.787305:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 12 from 12345-10.242.46.234@o2ib1 of length 192/192 into md 0xf97cdad9 [1] + 192 00000400:00000200:30.0:1616513089.787308:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.787310:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc53cc380 x1693067725927872/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513089.787322:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (25)++ 00000100:00000200:22.0:1616513089.787323:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc53ce780 x1693067725927888/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:N/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513089.787324:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (26)-- 00000800:00000200:2.0:1616513089.787326:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)-- 00000800:00000200:2.0:1616513089.787328:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (24)++ 00000800:00000200:2.0:1616513089.787331:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000800:00000200:22.0:1616513089.787334:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (25)++ 00000800:00000200:22.0:1616513089.787335:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (26)-- 00000100:00000200:18.0:1616513089.787335:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d2400 x1693067725927920/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513089.787337:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)-- 00000400:00000200:2.0:1616513089.787338:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000800:00000200:22.0:1616513089.787339:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (24)++ 00000800:00000200:22.0:1616513089.787342:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:2.0:1616513089.787343:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 192 into portal 18 MB=0x603d60c627630 00000800:00000200:18.0:1616513089.787346:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (25)++ 00000100:00000200:30.0:1616513089.787347:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d4380 x1693067725927936/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513089.787348:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000800:00000200:18.0:1616513089.787348:0:6471:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (26)++ 00000800:00000200:18.0:1616513089.787350:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (27)-- 00000800:00000200:18.0:1616513089.787352:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (26)-- 00000400:00000200:22.0:1616513089.787353:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 192 into portal 18 MB=0x603d60c627610 00000800:00000200:18.0:1616513089.787354:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)++ 00000800:00000200:18.0:1616513089.787357:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000800:00000200:30.0:1616513089.787358:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (26)++ 00000800:00000200:30.0:1616513089.787360:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (27)-- 00000100:00000200:26.0:1616513089.787360:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc53cc380 x1693067725927872/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513089.787361:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (26)-- 00000800:00000200:30.0:1616513089.787363:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)++ 00000400:00000200:18.0:1616513089.787363:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000800:00000200:30.0:1616513089.787365:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (26)-- 00000800:00000200:30.0:1616513089.787366:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (25)-- 00000400:00000200:18.0:1616513089.787368:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 192 into portal 18 MB=0x603d60c6275e0 00000100:00000200:26.0:1616513089.787370:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc53cc380 x1693067725927872/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513089.787374:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (24)-- 00000400:00000200:26.0:1616513089.787378:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccd48 00000100:00000200:6.0:1616513089.787382:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc53ce780 x1693067725927888/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000100:00000200:6.0:1616513089.787389:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc53ce780 x1693067725927888/t0(0) o101->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:23/10 lens 456/496 e 0 to 0 dl 1616513096 ref 1 fl Rpc:RN/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513089.787394:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a2a8 00000400:00000200:2.0:1616513089.787398:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 12 from 12345-10.242.46.234@o2ib1 of length 192/192 into md 0xf97cdb49 [1] + 192 00000400:00000200:2.0:1616513089.787401:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513089.787404:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d2400 x1693067725927920/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513089.787414:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d2400 x1693067725927920/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513089.787422:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eec440 00000400:00000200:22.0:1616513089.787426:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 12 from 12345-10.242.46.234@o2ib1 of length 192/192 into md 0xf97cdb29 [1] + 192 00000400:00000200:22.0:1616513089.787429:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513089.787431:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d4380 x1693067725927936/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513089.787452:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d4380 x1693067725927936/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513089.787460:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83e48 00000400:00000200:18.0:1616513089.787464:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 12 from 12345-10.242.46.234@o2ib1 of length 192/192 into md 0xf97cdae9 [1] + 192 00000400:00000200:18.0:1616513089.787467:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.787469:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d3600 x1693067725927984/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513089.787479:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)++ 00000100:00000200:22.0:1616513089.787481:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d0480 x1693067725927952/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513089.787482:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (24)-- 00000800:00000200:2.0:1616513089.787483:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:22.0:1616513089.787491:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:22.0:1616513089.787493:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000100:00000200:18.0:1616513089.787493:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d3a80 x1693067725927904/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513089.787495:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:18.0:1616513089.787504:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:26.0:1616513089.787506:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d3600 x1693067725927984/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513089.787506:0:6471:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:18.0:1616513089.787508:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:18.0:1616513089.787510:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513089.787513:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:26.0:1616513089.787515:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d3600 x1693067725927984/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513089.787523:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47564c8 00000100:00000200:10.0:1616513089.787527:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d0480 x1693067725927952/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513089.787536:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d0480 x1693067725927952/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513089.787544:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf007f8 00000100:00000200:30.0:1616513089.787548:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d3a80 x1693067725927904/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513089.787557:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d3a80 x1693067725927904/t0(0) o103->boslfs02-MDT0000-lwp-MDT0001@10.242.46.234@o2ib1:17/18 lens 328/224 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513089.787565:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae088 00000800:00000200:30.0:1616513089.787579:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:30.0:1616513089.787583:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:30.0:1616513089.787585:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:2.0:1616513089.787587:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:11.2:1616513089.800081:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.800136:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.800146:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.800155:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.90@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.800164:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.90@tcp1 of length 224 into portal 26 MB=0x5ffd5b18fc280 00000400:00000200:16.0:1616513089.800171:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.90@tcp1 of length 224/224 into md 0xf97cd2c9 [1] + 0 00000400:00000200:16.0:1616513089.800176:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3a90 00000400:00000200:16.0:1616513089.800179:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.800181:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513089.800186:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3a90 00000800:00000200:16.0:1616513089.800189:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.800190:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.800192:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.800197:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.800200:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.800201:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.800209:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.90@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.800219:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.90@tcp1 of length 224 into portal 12 MB=0x5ffd5b18fc300 00000400:00000200:28.0:1616513089.800225:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.90@tcp1 of length 224/224 into md 0x865be291 [1] + 13776 00000400:00000200:28.0:1616513089.800230:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.800233:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.800240:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.800243:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.800245:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513089.800247:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668155658880 00010000:00000200:24.0:1616513089.800257:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688668155658880/t0(0) o400->0cdb757a-f2b8-66cd-cc4a-1da3de4d670e@10.31.128.90@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513089.800270:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668155658880, offset 192 00000400:00000200:24.0:1616513089.800276:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.90@tcp1 00000400:00000200:24.0:1616513089.800283:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.90@tcp1 routed destination 00000400:00000200:24.0:1616513089.800297:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.90@tcp1(10.31.128.90@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.800303:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.800308:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.800310:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.800313:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.800317:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513089.800318:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668155659008 00000800:00000200:11.2:1616513089.800322:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.800323:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:0.0:1616513089.800336:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa2400 x1688668155659008/t0(0) o400->a974c68a-7103-1fec-14a7-a472cf8adb49@10.31.128.90@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513089.800343:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.800347:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513089.800349:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1616513089.800350:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668155659008, offset 192 00000800:00000200:16.0:1616513089.800351:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513089.800354:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513089.800356:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.90@tcp1 00000400:00000200:24.0:1616513089.800360:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f25d8 00000400:00000200:0.0:1616513089.800364:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.90@tcp1 routed destination 00000800:00000200:24.0:1616513089.800369:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513089.800379:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.90@tcp1(10.31.128.90@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.800386:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.800402:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.800404:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.800406:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.800411:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.800419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.800474:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513089.800480:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513089.800482:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513089.800489:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513089.800492:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42f68 00000800:00000200:22.0:1616513089.800496:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513089.800499:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.807338:0:50521:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513089.807351:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513089.807359:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513089.807368:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.201@tcp1 : PUT - for me 00000800:00000200:11.2:1616513089.807375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:30.0:1616513089.807377:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.201@tcp1 of length 224 into portal 26 MB=0x5ffdcbab79940 00000400:00000200:30.0:1616513089.807383:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.201@tcp1 of length 224/224 into md 0xf97cd2d9 [1] + 0 00000400:00000200:30.0:1616513089.807388:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2a18 00000400:00000200:30.0:1616513089.807390:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513089.807393:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513089.807396:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2a18 00000800:00000200:30.0:1616513089.807399:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513089.807402:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.807404:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513089.807408:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513089.807432:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.807453:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.807462:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.201@tcp1 : PUT - for me 00000100:00000200:24.0:1616513089.807473:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698374035776 00000400:00000200:28.0:1616513089.807478:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.201@tcp1 of length 224 into portal 12 MB=0x5ffdcbab799c0 00000400:00000200:28.0:1616513089.807484:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.201@tcp1 of length 224/224 into md 0x865be291 [1] + 14000 00010000:00000200:24.0:1616513089.807485:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688698374035776/t0(0) o400->ae98b9b9-3435-6f55-e7be-ad07496ef5a1@10.31.141.201@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513089.807489:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.807492:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.807499:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513089.807499:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698374035776, offset 192 00000800:00000200:28.0:1616513089.807502:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513089.807504:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513089.807506:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.201@tcp1 00000800:00000200:28.0:1616513089.807508:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513089.807513:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.201@tcp1 routed destination 00000100:00000200:0.0:1616513089.807522:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698374035904 00000400:00000200:24.0:1616513089.807528:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.201@tcp1(10.31.141.201@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.807534:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:0.0:1616513089.807534:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa0900 x1688698374035904/t0(0) o400->c4f604b8-d87b-0fdd-1c04-c67ab15fc9ae@10.31.141.201@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513089.807540:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513089.807543:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.807545:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513089.807547:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698374035904, offset 192 00000800:00000200:24.0:1616513089.807550:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513089.807553:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.201@tcp1 00000800:00000200:11.2:1616513089.807554:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513089.807556:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513089.807560:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.201@tcp1 routed destination 00000400:00000200:0.0:1616513089.807575:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.201@tcp1(10.31.141.201@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.807582:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.807587:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.807590:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.807592:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.807596:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.807601:0:50521:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513089.807607:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.807614:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.807617:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513089.807619:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513089.807624:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.807629:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f25d8 00000800:00000200:16.0:1616513089.807632:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513089.807654:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513089.807660:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513089.807663:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513089.807669:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513089.807673:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43b18 00000800:00000200:22.0:1616513089.807677:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513089.807680:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.811529:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513089.811584:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.811594:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513089.811596:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513089.811603:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.82@tcp1 : PUT - for me 00000800:00000200:30.0:1616513089.811604:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513089.811614:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.82@tcp1 of length 224 into portal 26 MB=0x6018460148100 00000400:00000200:30.0:1616513089.811615:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.82@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.811620:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.82@tcp1 of length 224/224 into md 0xf97cd2e9 [1] + 0 00000400:00000200:30.0:1616513089.811624:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.82@tcp1 of length 224 into portal 12 MB=0x60184601481c0 00000400:00000200:2.0:1616513089.811625:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f35c8 00000400:00000200:2.0:1616513089.811628:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:30.0:1616513089.811630:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.82@tcp1 of length 224/224 into md 0xf964d519 [1] + 83976 00000400:00000200:30.0:1616513089.811635:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.811635:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513089.811639:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f35c8 00000100:00000200:30.0:1616513089.811642:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513089.811642:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513089.811645:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513089.811647:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513089.811650:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.811650:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513089.811653:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513089.811655:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513089.811719:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517919531264 00000100:00000200:26.0:1616513089.811722:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517919531456 00010000:00000200:24.0:1616513089.811730:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1690517919531264/t0(0) o400->6e53bef3-c635-df5c-973e-b21104b4ac17@10.31.131.82@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:26.0:1616513089.811734:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf85580 x1690517919531456/t0(0) o400->40ba642c-4807-1d2f-4d66-3f833482584b@10.31.131.82@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513089.811744:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690517919531264, offset 192 00000100:00000200:26.0:1616513089.811748:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690517919531456, offset 192 00000400:00000200:24.0:1616513089.811750:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.82@tcp1 00000400:00000200:26.0:1616513089.811755:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.82@tcp1 00000400:00000200:24.0:1616513089.811758:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.82@tcp1 routed destination 00000400:00000200:26.0:1616513089.811764:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.82@tcp1 routed destination 00000400:00000200:24.0:1616513089.811773:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.82@tcp1(10.31.131.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513089.811779:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.82@tcp1(10.31.131.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.811780:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.811785:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513089.811786:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513089.811788:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.811790:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513089.811792:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513089.811795:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513089.811795:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513089.811797:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.811802:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513089.811802:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.811802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513089.811806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513089.811858:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.811875:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.811878:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513089.811884:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.811890:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3100 00000800:00000200:20.0:1616513089.811895:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513089.811897:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513089.811898:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513089.811903:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.811906:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513089.811908:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.811912:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.811916:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af6d8 00000800:00000200:18.0:1616513089.811920:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.825784:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.825842:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.825852:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513089.825873:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.240@tcp1 : PUT - for me 00000400:00000200:24.0:1616513089.825881:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.240@tcp1 of length 224 into portal 26 MB=0x603e0eb5f5880 00000400:00000200:24.0:1616513089.825888:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.240@tcp1 of length 224/224 into md 0xf97cd319 [1] + 0 00000400:00000200:24.0:1616513089.825893:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f26e8 00000400:00000200:24.0:1616513089.825896:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513089.825898:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513089.825904:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f26e8 00000800:00000200:16.0:1616513089.825906:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.825907:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.825909:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513089.825912:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513089.825915:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.825916:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.825925:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.240@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.825933:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.240@tcp1 of length 224 into portal 12 MB=0x603e0eb5f5940 00000400:00000200:16.0:1616513089.825940:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.240@tcp1 of length 224/224 into md 0x865be291 [1] + 14224 00000400:00000200:16.0:1616513089.825947:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.825949:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513089.825956:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.825959:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.825961:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513089.825967:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114416715904 00010000:00000200:28.0:1616513089.825978:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1693114416715904/t0(0) o400->03a0f3dc-213c-bb93-9886-a892b1c8a3c0@10.31.136.240@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513089.825992:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114416715904, offset 192 00000400:00000200:28.0:1616513089.825999:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.240@tcp1 00000400:00000200:28.0:1616513089.826006:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.240@tcp1 routed destination 00000100:00000200:0.0:1616513089.826018:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114416716096 00000400:00000200:28.0:1616513089.826020:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.240@tcp1(10.31.136.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513089.826027:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:0.0:1616513089.826029:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1680 x1693114416716096/t0(0) o400->2cb189d7-4461-2d12-36c9-2571e00ccd86@10.31.136.240@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513089.826033:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513089.826035:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.826037:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.826042:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513089.826043:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114416716096, offset 192 00000800:00000200:11.2:1616513089.826047:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513089.826048:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.240@tcp1 00000100:00000200:28.0:1616513089.826049:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513089.826055:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.240@tcp1 routed destination 00000800:00000200:28.0:1616513089.826062:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.826067:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.826069:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513089.826069:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.240@tcp1(10.31.136.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513089.826070:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513089.826074:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513089.826075:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513089.826077:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc7f8 00000800:00000200:28.0:1616513089.826080:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513089.826081:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.826083:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.826085:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.826090:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.826097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513089.826108:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513089.826112:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513089.826114:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513089.826119:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513089.826122:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42660 00000800:00000200:30.0:1616513089.826125:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513089.826127:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.841503:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513089.841521:0:50525:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513089.841558:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.841567:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.841576:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.143@tcp1 : PUT - for me 00000800:00000200:2.0:1616513089.841583:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513089.841586:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.143@tcp1 of length 224 into portal 12 MB=0x5f93af5303700 00000400:00000200:16.0:1616513089.841592:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.143@tcp1 of length 224/224 into md 0x865be291 [1] + 14448 00000800:00000200:2.0:1616513089.841593:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513089.841597:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.841600:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:2.0:1616513089.841603:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.143@tcp1 : PUT - for me 00000800:00000200:16.0:1616513089.841607:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.841610:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513089.841612:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513089.841612:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.143@tcp1 of length 224 into portal 26 MB=0x5f93af5303680 00000800:00000200:16.0:1616513089.841616:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513089.841619:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.143@tcp1 of length 224/224 into md 0xf97cd349 [1] + 0 00000400:00000200:2.0:1616513089.841623:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f24c8 00000400:00000200:2.0:1616513089.841626:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.841629:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513089.841633:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f24c8 00000800:00000200:2.0:1616513089.841636:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.841639:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.841640:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.841644:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513089.841676:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681406500550400 00010000:00000200:0.0:1616513089.841689:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd6780 x1681406500550400/t0(0) o400->1d26707d-9b49-846c-7f19-143a4a1fd4d0@10.31.143.143@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513089.841702:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681406500550272 00000100:00000200:0.0:1616513089.841703:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681406500550400, offset 192 00000400:00000200:0.0:1616513089.841710:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.143@tcp1 00010000:00000200:28.0:1616513089.841713:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1681406500550272/t0(0) o400->2a178445-6cc6-d1af-db55-7d8256d8cf86@10.31.143.143@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513089.841717:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.143@tcp1 routed destination 00000100:00000200:28.0:1616513089.841727:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681406500550272, offset 192 00000400:00000200:0.0:1616513089.841732:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.143@tcp1(10.31.143.143@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513089.841733:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.143@tcp1 00000800:00000200:0.0:1616513089.841739:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.841741:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.143@tcp1 routed destination 00000800:00000200:0.0:1616513089.841749:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.841752:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:28.0:1616513089.841757:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.143@tcp1(10.31.143.143@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.841757:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.841762:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.841763:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513089.841766:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.841769:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513089.841772:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513089.841774:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.841781:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.841785:0:50525:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513089.841787:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513089.841796:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513089.841801:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513089.841803:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513089.841810:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513089.841815:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc7f8 00000800:00000200:30.0:1616513089.841818:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513089.841819:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513089.841820:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513089.841826:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513089.841829:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513089.841832:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513089.841836:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513089.841840:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a437e8 00000800:00000200:24.0:1616513089.841843:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.842057:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513089.842068:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.842076:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513089.842077:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513089.842085:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.29@tcp1 : PUT - for me 00000400:00000200:20.0:1616513089.842095:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.29@tcp1 of length 224 into portal 12 MB=0x601b389a0f500 00000400:00000200:20.0:1616513089.842101:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.29@tcp1 of length 224/224 into md 0x865be291 [1] + 14672 00000400:00000200:20.0:1616513089.842106:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.842109:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513089.842116:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.842119:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.842121:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.842128:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690720480064768 00010000:00000200:0.0:1616513089.842134:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5100 x1690720480064768/t0(0) o400->1ec29bf0-783d-edbf-2d68-ff382723d95b@10.31.132.29@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.842146:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690720480064768, offset 192 00000400:00000200:0.0:1616513089.842151:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.29@tcp1 00000400:00000200:0.0:1616513089.842157:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.29@tcp1 routed destination 00000400:00000200:0.0:1616513089.842170:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.29@tcp1(10.31.132.29@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.842176:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513089.842181:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.842183:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.842185:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.842188:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.842192:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.842202:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.842206:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.842208:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513089.842210:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513089.842214:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.842217:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43650 00000800:00000200:28.0:1616513089.842220:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.842273:0:50525:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513089.842284:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.842292:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513089.842294:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.842301:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.29@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.842310:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.29@tcp1 of length 224 into portal 26 MB=0x601b389a0f480 00000400:00000200:18.0:1616513089.842316:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.29@tcp1 of length 224/224 into md 0xf97cd369 [1] + 0 00000400:00000200:18.0:1616513089.842321:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f22a8 00000400:00000200:18.0:1616513089.842324:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.842327:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513089.842331:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f22a8 00000800:00000200:18.0:1616513089.842334:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.842337:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.842338:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513089.842343:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690720480064640 00010000:00000200:28.0:1616513089.842350:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1690720480064640/t0(0) o400->8c967bb4-f62f-4574-0f0f-40fd78b583b7@10.31.132.29@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513089.842361:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690720480064640, offset 192 00000400:00000200:28.0:1616513089.842366:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.29@tcp1 00000400:00000200:28.0:1616513089.842371:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.29@tcp1 routed destination 00000400:00000200:28.0:1616513089.842384:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.29@tcp1(10.31.132.29@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513089.842390:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513089.842395:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513089.842397:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513089.842399:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.842402:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:28.0:1616513089.842406:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513089.842406:0:50525:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513089.842414:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513089.842417:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513089.842419:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513089.842423:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513089.842424:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513089.842427:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc3b8 00000800:00000200:30.0:1616513089.842430:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:31.0:1616513089.871544:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513089.871552:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513089.874319:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.874374:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.874384:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.874394:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.127@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.874403:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.127@tcp1 of length 224 into portal 26 MB=0x5ffd4d4bd8a40 00000400:00000200:16.0:1616513089.874410:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.127@tcp1 of length 224/224 into md 0xf97cd379 [1] + 0 00000400:00000200:16.0:1616513089.874414:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3078 00000400:00000200:16.0:1616513089.874417:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.874420:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513089.874424:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3078 00000800:00000200:20.0:1616513089.874427:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.874428:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.874431:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.874433:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513089.874497:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664450894400 00010000:00000200:28.0:1616513089.874508:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688664450894400/t0(0) o400->08ad17dd-21be-1c05-c473-e8f8ab52177a@10.31.136.127@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513089.874521:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664450894400, offset 192 00000400:00000200:28.0:1616513089.874527:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.127@tcp1 00000400:00000200:28.0:1616513089.874534:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.127@tcp1 routed destination 00000400:00000200:28.0:1616513089.874549:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.127@tcp1(10.31.136.127@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513089.874554:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.874560:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513089.874564:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.874566:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.874571:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.874576:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:28.0:1616513089.874578:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513089.874629:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.874636:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513089.874639:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513089.874641:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513089.874646:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513089.874651:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc198 00000800:00000200:24.0:1616513089.874654:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.875686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513089.875741:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.875750:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513089.875759:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.127@tcp1 : PUT - for me 00000400:00000200:20.0:1616513089.875767:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.127@tcp1 of length 224 into portal 12 MB=0x5ffd4d4bd8bc0 00000400:00000200:20.0:1616513089.875774:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.127@tcp1 of length 224/224 into md 0x865be291 [1] + 14896 00000400:00000200:20.0:1616513089.875779:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.875782:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513089.875789:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.875792:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.875794:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513089.875798:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.875867:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664450894784 00010000:00000200:0.0:1616513089.875879:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0d80 x1688664450894784/t0(0) o400->26b1326f-902b-7efd-5789-f6a61618bbd0@10.31.136.127@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.875893:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664450894784, offset 192 00000400:00000200:0.0:1616513089.875900:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.127@tcp1 00000400:00000200:0.0:1616513089.875907:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.127@tcp1 routed destination 00000400:00000200:0.0:1616513089.875922:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.127@tcp1(10.31.136.127@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.875928:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.875934:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.875937:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.875939:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.875944:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.875952:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.876009:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.876016:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.876019:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.876028:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.876032:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a90 00000800:00000200:2.0:1616513089.876036:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.876039:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513089.882597:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928000, portal 4 00000100:00000200:15.0:1616513089.882603:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928000, offset 0 00000400:00000200:15.0:1616513089.882610:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:15.0:1616513089.882616:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:15.0:1616513089.882630:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513089.882636:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:15.0:1616513089.882643:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513089.882647:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:15.0:1616513089.882649:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:15.0:1616513089.882654:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513089.882700:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513089.882757:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513089.882765:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513089.882768:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513089.882769:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513089.882775:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513089.882782:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc566300 x1693067725928000/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513089.882794:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0bba0 00000800:00000200:22.0:1616513089.882801:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513089.882871:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513089.882926:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513089.882945:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:2.0:1616513089.882953:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513089.882962:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627640 00000400:00000200:2.0:1616513089.882968:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cdbe9 [1] + 192 00000400:00000200:2.0:1616513089.882974:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.882979:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc566300 x1693067725928000/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513089.882992:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513089.882995:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513089.882998:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513089.882999:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:15.0:1616513089.883004:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc566300 x1693067725928000/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513089.883017:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc566300 x1693067725928000/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513089.883028:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000800:00000200:7.2:1616513089.887541:0:50530:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513089.887551:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513089.887558:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513089.887567:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.46@tcp1 : PUT - for me 00000400:00000200:30.0:1616513089.887576:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.46@tcp1 of length 224 into portal 26 MB=0x60015fe0f2d80 00000400:00000200:30.0:1616513089.887583:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.46@tcp1 of length 224/224 into md 0xf97cd389 [1] + 0 00000400:00000200:30.0:1616513089.887587:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2c38 00000400:00000200:30.0:1616513089.887590:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513089.887593:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513089.887596:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2c38 00000800:00000200:30.0:1616513089.887599:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513089.887602:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513089.887603:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:30.0:1616513089.887610:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513089.887612:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513089.887614:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513089.887622:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.46@tcp1 : PUT - for me 00000400:00000200:22.0:1616513089.887631:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.46@tcp1 of length 224 into portal 12 MB=0x60015fe0f30c0 00000400:00000200:22.0:1616513089.887638:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.46@tcp1 of length 224/224 into md 0xf964d519 [1] + 84200 00000400:00000200:22.0:1616513089.887643:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513089.887646:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513089.887654:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.887657:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513089.887659:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513089.887662:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688944316984704 00010000:00000200:28.0:1616513089.887673:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688944316984704/t0(0) o400->fcc482ae-ecf4-643c-87ac-25c382f7c74e@10.31.140.46@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513089.887688:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688944316984704, offset 192 00000400:00000200:28.0:1616513089.887694:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.46@tcp1 00000400:00000200:28.0:1616513089.887701:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.46@tcp1 routed destination 00000400:00000200:28.0:1616513089.887716:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.46@tcp1(10.31.140.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513089.887720:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688944316985536 00000800:00000200:28.0:1616513089.887723:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.887729:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513089.887732:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:26.0:1616513089.887732:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a29b00 x1688944316985536/t0(0) o400->f27bf265-2977-853a-7643-0d03081ab855@10.31.140.46@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513089.887734:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.887739:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513089.887746:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.887746:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513089.887747:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688944316985536, offset 192 00000400:00000200:26.0:1616513089.887759:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.46@tcp1 00000800:00000200:28.0:1616513089.887762:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.887765:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513089.887766:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.46@tcp1 routed destination 00000400:00000200:28.0:1616513089.887767:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.887773:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.887776:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc198 00000800:00000200:28.0:1616513089.887779:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.887781:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513089.887781:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.46@tcp1(10.31.140.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513089.887788:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513089.887794:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513089.887796:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.887798:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.887803:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.887809:0:50530:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.887863:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.887870:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513089.887873:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.887873:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.887880:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.887884:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae880 00000800:00000200:2.0:1616513089.887888:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.888054:0:50530:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513089.888064:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.888069:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513089.888073:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513089.888076:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.239@tcp1 : PUT - for me 00000400:00000200:22.0:1616513089.888084:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.239@tcp1 of length 224 into portal 26 MB=0x604f68c30dc40 00000800:00000200:7.2:1616513089.888084:0:50530:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:22.0:1616513089.888089:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.239@tcp1 of length 224/224 into md 0xf97cd399 [1] + 0 00000800:00000200:30.0:1616513089.888091:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513089.888093:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f38f8 00000400:00000200:22.0:1616513089.888095:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513089.888097:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:22.0:1616513089.888098:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513089.888100:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513089.888101:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f38f8 00000400:00000200:30.0:1616513089.888104:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.239@tcp1 : PUT - for me 00000800:00000200:22.0:1616513089.888104:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513089.888106:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513089.888107:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513089.888110:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.239@tcp1 of length 224 into portal 12 MB=0x604f68c30dfc0 00000100:00000200:28.0:1616513089.888115:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694306820742208 00000400:00000200:30.0:1616513089.888116:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.239@tcp1 of length 224/224 into md 0xf964d519 [1] + 84424 00000400:00000200:30.0:1616513089.888120:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:28.0:1616513089.888121:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1694306820742208/t0(0) o400->9977ef97-1a85-f939-c9a1-c5429ed9798d@10.31.140.239@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513089.888122:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513089.888127:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513089.888131:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513089.888132:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513089.888132:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694306820742208, offset 192 00000400:00000200:28.0:1616513089.888137:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.239@tcp1 00000100:00000200:26.0:1616513089.888140:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694306820743104 00000400:00000200:28.0:1616513089.888143:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.239@tcp1 routed destination 00010000:00000200:26.0:1616513089.888146:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf80d80 x1694306820743104/t0(0) o400->54b7ab96-7e67-0e0a-5aeb-9be61bfb11aa@10.31.140.239@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513089.888156:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.239@tcp1(10.31.140.239@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513089.888157:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694306820743104, offset 192 00000400:00000200:26.0:1616513089.888163:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.239@tcp1 00000800:00000200:28.0:1616513089.888164:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.888168:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:26.0:1616513089.888169:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.239@tcp1 routed destination 00000800:00000200:28.0:1616513089.888170:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.888172:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.888175:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513089.888180:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.888180:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513089.888182:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.239@tcp1(10.31.140.239@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513089.888187:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513089.888192:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513089.888195:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.888198:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.888201:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.888205:0:50530:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.888216:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.888223:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.888226:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513089.888227:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.888232:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513089.888233:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513089.888236:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afed0 00000800:00000200:16.0:1616513089.888240:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513089.888241:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513089.888243:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.888249:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.888257:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc198 00000800:00000200:16.0:1616513089.888260:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.888263:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:27.0:1616513089.898610:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928016, portal 4 00000100:00000200:27.0:1616513089.898615:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928016, offset 0 00000400:00000200:27.0:1616513089.898622:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:27.0:1616513089.898628:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:27.0:1616513089.898641:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513089.898647:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:27.0:1616513089.898654:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513089.898656:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:27.0:1616513089.898659:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:27.0:1616513089.898663:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513089.898710:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513089.898730:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513089.898735:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513089.898738:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513089.898745:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513089.898752:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456ec800 x1693067725928016/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513089.898764:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d2482a8 00000800:00000200:1.0:1616513089.898768:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513089.898771:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513089.898859:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513089.898868:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513089.898874:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513089.898878:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513089.898885:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513089.898893:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627650 00000400:00000200:1.0:1616513089.898899:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436fff25 [1] + 192 00000400:00000200:1.0:1616513089.898905:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513089.898909:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456ec800 x1693067725928016/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513089.898923:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513089.898925:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513089.898927:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:27.0:1616513089.898930:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456ec800 x1693067725928016/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513089.898943:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456ec800 x1693067725928016/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513089.898952:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248660 00000800:00000200:11.2:1616513089.909409:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.909476:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.909485:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.909494:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.33@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.909503:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.33@tcp1 of length 224 into portal 12 MB=0x601ac269a85c0 00000400:00000200:28.0:1616513089.909508:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.33@tcp1 of length 224/224 into md 0x865be291 [1] + 15120 00000400:00000200:28.0:1616513089.909513:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.909516:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.909523:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.909525:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.909527:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.909530:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.909588:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690688753927616 00000800:00000200:7.2:1616513089.909597:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:0.0:1616513089.909602:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa0000 x1690688753927616/t0(0) o400->2cbfb84a-1852-08e7-2f65-bce8f3f7f4d9@10.31.132.33@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.909616:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690688753927616, offset 192 00000400:00000200:0.0:1616513089.909624:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.33@tcp1 00000400:00000200:0.0:1616513089.909632:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.33@tcp1 routed destination 00000400:00000200:0.0:1616513089.909647:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.33@tcp1(10.31.132.33@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.909654:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513089.909656:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.909660:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.909663:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.909665:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.909666:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513089.909667:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513089.909670:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513089.909675:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.33@tcp1 : PUT - for me 00000800:00000200:11.2:1616513089.909675:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513089.909683:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.33@tcp1 of length 224 into portal 26 MB=0x601ac269a8540 00000400:00000200:22.0:1616513089.909690:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.33@tcp1 of length 224/224 into md 0xf97cd3b9 [1] + 0 00000400:00000200:22.0:1616513089.909695:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2000 00000400:00000200:22.0:1616513089.909698:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513089.909701:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513089.909705:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2000 00000800:00000200:22.0:1616513089.909710:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.909714:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513089.909715:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513089.909724:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690688753927488 00000800:00000200:16.0:1616513089.909729:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:28.0:1616513089.909733:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1690688753927488/t0(0) o400->db743e5d-d130-14a0-ac63-4737c3afd5ef@10.31.132.33@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513089.909736:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.909739:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513089.909742:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513089.909745:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690688753927488, offset 192 00000400:00000200:16.0:1616513089.909746:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.909751:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.33@tcp1 00000400:00000200:16.0:1616513089.909754:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42990 00000400:00000200:28.0:1616513089.909757:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.33@tcp1 routed destination 00000800:00000200:16.0:1616513089.909762:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:28.0:1616513089.909770:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.33@tcp1(10.31.132.33@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513089.909776:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513089.909782:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513089.909784:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513089.909786:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513089.909790:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.909795:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513089.909796:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513089.909847:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.909853:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.909856:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.909862:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.909867:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc6e8 00000800:00000200:2.0:1616513089.909871:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.909873:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:15.0:1616513089.914504:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928032, portal 4 00000100:00000200:15.0:1616513089.914509:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928032, offset 0 00000400:00000200:15.0:1616513089.914516:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:15.0:1616513089.914521:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:15.0:1616513089.914533:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513089.914539:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:15.0:1616513089.914545:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513089.914548:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:15.0:1616513089.914550:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:15.0:1616513089.914555:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513089.914602:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513089.914658:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513089.914665:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:24.0:1616513089.914669:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513089.914675:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:24.0:1616513089.914682:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd63b180 x1693067725928032/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513089.914695:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b7e8 00000800:00000200:24.0:1616513089.914699:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513089.914702:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513089.914789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513089.914843:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513089.914853:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513089.914861:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513089.914869:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627660 00000400:00000200:20.0:1616513089.914876:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866da3e1 [1] + 192 00000400:00000200:20.0:1616513089.914881:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513089.914887:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd63b180 x1693067725928032/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513089.914897:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513089.914908:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513089.914911:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513089.914913:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:15.0:1616513089.914917:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd63b180 x1693067725928032/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513089.914929:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd63b180 x1693067725928032/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513089.914948:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000800:00000200:11.2:1616513089.918021:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.918077:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.918086:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513089.918097:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.211@tcp1 : PUT - for me 00000400:00000200:24.0:1616513089.918106:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.211@tcp1 of length 224 into portal 26 MB=0x604a4acda6a00 00000400:00000200:24.0:1616513089.918112:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.211@tcp1 of length 224/224 into md 0xf97cd3d9 [1] + 0 00000400:00000200:24.0:1616513089.918117:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2aa0 00000400:00000200:24.0:1616513089.918120:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513089.918123:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513089.918129:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2aa0 00000800:00000200:28.0:1616513089.918131:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.918132:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.918135:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513089.918138:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513089.918141:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.918142:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.918151:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.211@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.918159:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.211@tcp1 of length 224 into portal 12 MB=0x604a4acda6c00 00000400:00000200:28.0:1616513089.918166:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.211@tcp1 of length 224/224 into md 0x865be291 [1] + 15344 00000400:00000200:28.0:1616513089.918171:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.918174:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.918198:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.918201:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.918202:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.918203:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693955181406720 00010000:00000200:0.0:1616513089.918214:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1693955181406720/t0(0) o400->b9bf8679-187f-2a2a-57e7-0b7790ff0c49@10.31.140.211@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513089.918228:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693955181406720, offset 192 00000400:00000200:0.0:1616513089.918235:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.211@tcp1 00000400:00000200:0.0:1616513089.918242:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.211@tcp1 routed destination 00000100:00000200:4.0:1616513089.918251:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693955181407232 00000400:00000200:0.0:1616513089.918258:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.211@tcp1(10.31.140.211@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:4.0:1616513089.918263:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5a00 x1693955181407232/t0(0) o400->78db329b-923c-4784-8732-76b0a9bf6575@10.31.140.211@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513089.918264:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513089.918270:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.918273:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.918275:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513089.918278:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693955181407232, offset 192 00000800:00000200:0.0:1616513089.918280:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513089.918284:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.211@tcp1 00000800:00000200:11.2:1616513089.918285:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513089.918287:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:4.0:1616513089.918292:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.211@tcp1 routed destination 00000800:00000200:20.0:1616513089.918301:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:4.0:1616513089.918307:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.211@tcp1(10.31.140.211@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513089.918309:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.918311:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513089.918313:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513089.918313:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513089.918318:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:4.0:1616513089.918319:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.918322:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513089.918324:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a422a8 00000800:00000200:4.0:1616513089.918326:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513089.918327:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:4.0:1616513089.918331:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.918338:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513089.918349:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513089.918354:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513089.918356:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513089.918362:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513089.918366:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baee0 00000800:00000200:30.0:1616513089.918369:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513089.918371:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513089.919454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513089.919522:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.919532:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513089.919540:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.8.173@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.919550:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.8.173@tcp1 of length 224 into portal 26 MB=0x5ff2d25349140 00000400:00000200:2.0:1616513089.919558:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.8.173@tcp1 of length 224/224 into md 0xf97cd3e9 [1] + 0 00000400:00000200:2.0:1616513089.919563:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2990 00000400:00000200:2.0:1616513089.919566:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.919569:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513089.919573:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2990 00000800:00000200:2.0:1616513089.919576:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.919579:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.919580:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513089.919587:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513089.919590:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513089.919590:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513089.919599:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.8.173@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.919608:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.8.173@tcp1 of length 224 into portal 12 MB=0x5ff2d253492c0 00000400:00000200:18.0:1616513089.919615:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.8.173@tcp1 of length 224/224 into md 0xf964d519 [1] + 84648 00000400:00000200:18.0:1616513089.919620:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.919622:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.919629:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.919632:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.919634:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513089.919637:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1687944246366528 00010000:00000200:0.0:1616513089.919647:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1687944246366528/t0(0) o400->4414238e-ead9-a8ff-94f1-f4b7fece98a0@10.31.8.173@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513089.919662:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1687944246366528, offset 192 00000400:00000200:0.0:1616513089.919668:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.8.173@tcp1 00000400:00000200:0.0:1616513089.919676:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.8.173@tcp1 routed destination 00000400:00000200:0.0:1616513089.919690:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.8.173@tcp1(10.31.8.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.919697:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513089.919705:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.919708:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.919710:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513089.919711:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1687944246366912 00000800:00000200:0.0:1616513089.919715:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.919721:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513089.919722:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:26.0:1616513089.919723:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120b600 x1687944246366912/t0(0) o400->8f097719-2337-0ef6-17ed-675d50a7d832@10.31.8.173@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513089.919738:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1687944246366912, offset 192 00000400:00000200:26.0:1616513089.919744:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.8.173@tcp1 00000400:00000200:26.0:1616513089.919751:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.8.173@tcp1 routed destination 00000400:00000200:26.0:1616513089.919766:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.8.173@tcp1(10.31.8.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513089.919772:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513089.919773:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513089.919778:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513089.919780:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513089.919781:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513089.919782:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513089.919783:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513089.919788:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513089.919788:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.919792:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513089.919793:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42aa0 00000800:00000200:28.0:1616513089.919796:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.919799:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513089.919849:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513089.919856:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513089.919858:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513089.919859:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513089.919866:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513089.919870:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa90 00000800:00000200:22.0:1616513089.919874:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.925648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.925702:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.925711:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513089.925720:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.82@tcp1 : PUT - for me 00000400:00000200:24.0:1616513089.925730:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.82@tcp1 of length 224 into portal 26 MB=0x5ffdc75b23d80 00000400:00000200:24.0:1616513089.925736:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.82@tcp1 of length 224/224 into md 0xf97cd3f9 [1] + 0 00000400:00000200:24.0:1616513089.925741:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2220 00000400:00000200:24.0:1616513089.925744:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513089.925747:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513089.925751:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2220 00000800:00000200:24.0:1616513089.925754:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.925755:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513089.925757:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513089.925760:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.925812:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697216056704 00000800:00000200:7.2:1616513089.925818:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:0.0:1616513089.925823:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688697216056704/t0(0) o400->0461c591-a753-1da7-2f9f-a642ec1c6abc@10.31.141.82@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513089.925836:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697216056704, offset 192 00000400:00000200:0.0:1616513089.925842:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.82@tcp1 00000400:00000200:0.0:1616513089.925849:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.82@tcp1 routed destination 00000400:00000200:0.0:1616513089.925863:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.82@tcp1(10.31.141.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.925869:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513089.925874:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.925874:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.925877:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.925879:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513089.925883:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513089.925883:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.925889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513089.925889:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513089.925893:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.82@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.925901:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.82@tcp1 of length 224 into portal 12 MB=0x5ffdc75b240c0 00000400:00000200:2.0:1616513089.925908:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.82@tcp1 of length 224/224 into md 0xf964d519 [1] + 84872 00000400:00000200:2.0:1616513089.925913:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.925915:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513089.925922:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.925925:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.925927:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.925929:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513089.925949:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.925956:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.925958:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513089.925964:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513089.925970:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43870 00000800:00000200:28.0:1616513089.925973:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.925975:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513089.925988:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697216057536 00010000:00000200:26.0:1616513089.926001:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5100 x1688697216057536/t0(0) o400->a0612bf1-85aa-efc6-597d-c5541ea50a62@10.31.141.82@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513089.926015:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697216057536, offset 192 00000400:00000200:26.0:1616513089.926021:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.82@tcp1 00000400:00000200:26.0:1616513089.926028:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.82@tcp1 routed destination 00000400:00000200:26.0:1616513089.926043:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.82@tcp1(10.31.141.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513089.926050:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513089.926055:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513089.926058:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.926060:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.926065:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.926070:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513089.926078:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513089.926082:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513089.926083:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513089.926088:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513089.926091:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afba0 00000800:00000200:30.0:1616513089.926093:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513089.926095:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513089.962578:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928048, portal 4 00000100:00000200:30.0:1616513089.962595:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928048, offset 0 00000400:00000200:30.0:1616513089.962601:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513089.962606:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513089.962617:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513089.962622:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513089.962627:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513089.962630:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513089.962631:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513089.962636:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513089.962684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513089.962739:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513089.962746:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513089.962748:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513089.962754:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513089.962760:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d747c697080 x1693067725928048/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513089.962771:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efed0 00000800:00000200:20.0:1616513089.962775:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513089.962778:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513089.962843:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513089.962898:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513089.962907:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:28.0:1616513089.962915:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513089.962923:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627670 00000400:00000200:28.0:1616513089.962929:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866da431 [1] + 192 00000400:00000200:28.0:1616513089.962933:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.962938:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d747c697080 x1693067725928048/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513089.962952:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:16.0:1616513089.962952:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513089.962956:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:30.0:1616513089.962957:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d747c697080 x1693067725928048/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513089.962958:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513089.962968:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d747c697080 x1693067725928048/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513096 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513089.962976:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eebb0 00000800:00000200:7.2:1616513089.963261:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513089.963319:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513089.963329:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513089.963338:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.78@tcp1 : PUT - for me 00000400:00000200:2.0:1616513089.963349:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.78@tcp1 of length 224 into portal 26 MB=0x5ffd44e7a9640 00000400:00000200:2.0:1616513089.963355:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.78@tcp1 of length 224/224 into md 0xf97cd419 [1] + 0 00000400:00000200:2.0:1616513089.963360:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3540 00000400:00000200:2.0:1616513089.963363:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513089.963365:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513089.963369:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3540 00000800:00000200:2.0:1616513089.963372:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.963374:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.963375:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.963381:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513089.963384:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.963385:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513089.963393:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.78@tcp1 : PUT - for me 00000400:00000200:18.0:1616513089.963402:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.78@tcp1 of length 224 into portal 12 MB=0x5ffd44e7a96c0 00000400:00000200:18.0:1616513089.963409:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.78@tcp1 of length 224/224 into md 0xf964d519 [1] + 85096 00000400:00000200:18.0:1616513089.963414:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513089.963417:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513089.963424:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513089.963427:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.963429:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513089.963449:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662198359744 00000100:00000200:0.0:1616513089.963452:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662198359616 00010000:00000200:26.0:1616513089.963461:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461208480 x1688662198359744/t0(0) o400->e5694448-7f28-cc41-96d8-553b0e25ab41@10.31.133.78@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:0.0:1616513089.963464:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688662198359616/t0(0) o400->820fb76d-edb9-ab23-3699-ac5cb89da9eb@10.31.133.78@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513089.963476:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662198359744, offset 192 00000100:00000200:0.0:1616513089.963478:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662198359616, offset 192 00000400:00000200:26.0:1616513089.963484:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.78@tcp1 00000400:00000200:0.0:1616513089.963485:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.78@tcp1 00000400:00000200:26.0:1616513089.963492:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.78@tcp1 routed destination 00000400:00000200:0.0:1616513089.963493:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.78@tcp1 routed destination 00000400:00000200:26.0:1616513089.963509:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.78@tcp1(10.31.133.78@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513089.963509:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.78@tcp1(10.31.133.78@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513089.963516:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513089.963516:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513089.963522:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.963523:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513089.963525:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.963525:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513089.963527:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.963527:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.963532:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513089.963532:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.963537:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513089.963541:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513089.963578:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513089.963591:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513089.963591:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.963595:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.963597:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513089.963599:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513089.963601:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513089.963602:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513089.963602:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513089.963603:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.963606:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aea18 00000400:00000200:22.0:1616513089.963608:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513089.963608:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513089.963614:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42000 00000800:00000200:22.0:1616513089.963618:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.973775:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513089.973831:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.973841:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513089.973850:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.11@tcp1 : PUT - for me 00000400:00000200:16.0:1616513089.973860:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.11@tcp1 of length 224 into portal 26 MB=0x60074994e8200 00000400:00000200:16.0:1616513089.973867:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.11@tcp1 of length 224/224 into md 0xf97cd439 [1] + 0 00000400:00000200:16.0:1616513089.973871:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f37e8 00000400:00000200:16.0:1616513089.973874:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.973877:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513089.973881:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f37e8 00000800:00000200:16.0:1616513089.973884:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.973886:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.973887:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.973893:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513089.973896:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.973897:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.973905:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.11@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.973914:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.11@tcp1 of length 224 into portal 12 MB=0x60074994e8280 00000400:00000200:28.0:1616513089.973921:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.11@tcp1 of length 224/224 into md 0x865be291 [1] + 15568 00000400:00000200:28.0:1616513089.973926:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.973928:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.973936:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.973939:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.973941:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513089.973946:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689350648529408 00010000:00000200:0.0:1616513089.973957:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1689350648529408/t0(0) o400->bfbc3aec-ba95-ba43-f8b0-05080e8985a3@10.31.136.11@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.973959:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689350648529536 00010000:00000200:4.0:1616513089.973971:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa6c00 x1689350648529536/t0(0) o400->3799c868-e4f6-5b6b-3649-ace35c874880@10.31.136.11@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.973971:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689350648529408, offset 192 00000400:00000200:0.0:1616513089.973978:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.11@tcp1 00000100:00000200:4.0:1616513089.973984:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689350648529536, offset 192 00000400:00000200:0.0:1616513089.973985:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.11@tcp1 routed destination 00000400:00000200:4.0:1616513089.973990:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.11@tcp1 00000400:00000200:0.0:1616513089.974000:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.11@tcp1(10.31.136.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.974006:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:4.0:1616513089.974009:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.11@tcp1 routed destination 00000800:00000200:0.0:1616513089.974016:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.974019:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.974021:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:4.0:1616513089.974023:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.11@tcp1(10.31.136.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.974026:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513089.974029:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513089.974031:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513089.974032:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513089.974034:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.974037:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513089.974039:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513089.974043:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513089.974043:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513089.974050:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513089.974051:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513089.974052:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513089.974054:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513089.974059:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.974065:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ee0 00000800:00000200:20.0:1616513089.974068:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513089.974107:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513089.974114:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513089.974117:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513089.974124:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.974128:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba330 00000800:00000200:2.0:1616513089.974132:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513089.974134:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513089.975600:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.975658:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.975668:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.975677:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.975687:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d0322c1300 00000400:00000200:28.0:1616513089.975693:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0x865be291 [1] + 15792 00000400:00000200:28.0:1616513089.975698:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.975701:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.975707:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.975710:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513089.975711:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.975715:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513089.975772:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042590094080 00010000:00000200:4.0:1616513089.975858:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa3f00 x1693042590094080/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:715/0 lens 440/368 e 0 to 0 dl 1616513095 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513089.975873:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042590094080, offset 192 00000400:00000200:4.0:1616513089.975879:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:4.0:1616513089.975887:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:4.0:1616513089.975902:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.975908:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.975914:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.975916:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.975919:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.975923:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.975929:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.975939:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513089.975944:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.975946:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513089.975949:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513089.975952:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.975956:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb7e8 00000800:00000200:16.0:1616513089.975959:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513089.976348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513089.976358:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513089.976364:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513089.976369:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513089.976372:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.233@tcp1 : PUT - for me 00000400:00000200:30.0:1616513089.976380:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.233@tcp1 of length 224 into portal 26 MB=0x5ffddd9676200 00000400:00000200:30.0:1616513089.976386:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.233@tcp1 of length 224/224 into md 0xf97cd459 [1] + 0 00000400:00000200:30.0:1616513089.976390:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3e48 00000400:00000200:30.0:1616513089.976393:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513089.976395:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513089.976398:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3e48 00000800:00000200:30.0:1616513089.976401:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513089.976403:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513089.976405:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513089.976422:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703183839744 00010000:00000200:0.0:1616513089.976432:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688703183839744/t0(0) o400->08d984c2-154b-fbce-c6c9-f90d820a8b03@10.31.140.233@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513089.976456:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703183839744, offset 192 00000400:00000200:0.0:1616513089.976462:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.233@tcp1 00000400:00000200:0.0:1616513089.976470:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.233@tcp1 routed destination 00000400:00000200:0.0:1616513089.976484:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.233@tcp1(10.31.140.233@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.976490:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513089.976493:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.976496:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.976498:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.976500:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513089.976504:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.976505:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513089.976510:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513089.976511:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513089.976512:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513089.976514:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513089.976521:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.233@tcp1 : PUT - for me 00000400:00000200:24.0:1616513089.976531:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.233@tcp1 of length 224 into portal 12 MB=0x5ffddd9676280 00000400:00000200:24.0:1616513089.976537:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.233@tcp1 of length 224/224 into md 0x865be291 [1] + 16232 00000400:00000200:24.0:1616513089.976543:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513089.976547:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513089.976554:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.976557:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513089.976559:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513089.976563:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:4.0:1616513089.976566:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703183839872 00000800:00000200:2.0:1616513089.976570:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:4.0:1616513089.976573:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e5a00 x1688703183839872/t0(0) o400->253cd5a3-ed9d-e8a3-d197-cd92dda003b9@10.31.140.233@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513089.976573:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513089.976576:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513089.976580:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513089.976585:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ed0 00000100:00000200:4.0:1616513089.976588:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703183839872, offset 192 00000800:00000200:2.0:1616513089.976589:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513089.976595:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.233@tcp1 00000400:00000200:4.0:1616513089.976601:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.233@tcp1 routed destination 00000400:00000200:4.0:1616513089.976614:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.233@tcp1(10.31.140.233@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513089.976619:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513089.976624:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513089.976626:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513089.976628:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513089.976631:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.976636:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.976646:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.976651:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.976655:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513089.976655:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513089.976662:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.976666:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb298 00000800:00000200:20.0:1616513089.976670:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.981746:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513089.981755:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513089.981802:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.981812:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.981821:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.63@tcp1 : PUT - for me 00000800:00000200:22.0:1616513089.981821:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513089.981831:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513089.981831:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.63@tcp1 of length 224 into portal 26 MB=0x5ffdc8a9dab00 00000800:00000200:22.0:1616513089.981831:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513089.981838:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.63@tcp1 of length 224/224 into md 0xf97cd469 [1] + 0 00000400:00000200:22.0:1616513089.981841:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.63@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.981843:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3c28 00000400:00000200:28.0:1616513089.981846:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.981848:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513089.981850:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.63@tcp1 of length 224 into portal 12 MB=0x5ffdc8a9dae40 00000400:00000200:28.0:1616513089.981853:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3c28 00000800:00000200:24.0:1616513089.981855:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.981858:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513089.981859:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.63@tcp1 of length 224/224 into md 0xf964d519 [1] + 85320 00000800:00000200:28.0:1616513089.981860:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513089.981862:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513089.981864:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513089.981867:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513089.981874:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513089.981877:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513089.981879:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513089.981914:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697567030016 00010000:00000200:0.0:1616513089.981924:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688697567030016/t0(0) o400->48478625-af15-b077-b795-b0730249728a@10.31.142.63@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513089.981938:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697567030848 00000100:00000200:0.0:1616513089.981938:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697567030016, offset 192 00000400:00000200:0.0:1616513089.981945:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.63@tcp1 00010000:00000200:26.0:1616513089.981949:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2d100 x1688697567030848/t0(0) o400->be0d7fb8-b128-9084-528b-9ea0c2a04ffa@10.31.142.63@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513089.981952:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.63@tcp1 routed destination 00000100:00000200:26.0:1616513089.981962:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697567030848, offset 192 00000400:00000200:0.0:1616513089.981966:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.63@tcp1(10.31.142.63@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513089.981968:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.63@tcp1 00000800:00000200:0.0:1616513089.981972:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513089.981975:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.63@tcp1 routed destination 00000800:00000200:0.0:1616513089.981977:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.981980:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513089.981982:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513089.981986:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513089.981989:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.63@tcp1(10.31.142.63@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513089.981993:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513089.981995:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513089.982000:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:7.2:1616513089.982001:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513089.982002:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513089.982004:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513089.982008:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513089.982024:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513089.982057:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.982064:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.982066:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.982073:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.982079:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a438f8 00000800:00000200:16.0:1616513089.982081:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513089.982082:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.982087:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513089.982088:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513089.982092:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.982099:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513089.982103:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af650 00000800:00000200:16.0:1616513089.982107:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.982109:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.998793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513089.998850:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513089.998860:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513089.998869:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.7@tcp1 : PUT - for me 00000400:00000200:24.0:1616513089.998877:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.7@tcp1 of length 224 into portal 26 MB=0x6013701186940 00000400:00000200:24.0:1616513089.998884:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.7@tcp1 of length 224/224 into md 0xf97cd489 [1] + 0 00000400:00000200:24.0:1616513089.998889:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2110 00000400:00000200:24.0:1616513089.998891:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513089.998894:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513089.998898:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2110 00000800:00000200:24.0:1616513089.998901:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513089.998904:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.998906:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513089.998913:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.998916:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.998916:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513089.998925:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.7@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.998933:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.7@tcp1 of length 224 into portal 12 MB=0x60137011869c0 00000400:00000200:28.0:1616513089.998940:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.7@tcp1 of length 224/224 into md 0x865be291 [1] + 16456 00000400:00000200:28.0:1616513089.998945:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513089.998948:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513089.998956:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513089.998959:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513089.998960:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690185613470016 00000800:00000200:28.0:1616513089.998961:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513089.998970:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1690185613470016/t0(0) o400->56ac16b4-29c5-ff30-5edd-daaf71cce722@10.31.136.7@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513089.998974:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690185613470144 00000100:00000200:0.0:1616513089.998984:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690185613470016, offset 192 00010000:00000200:8.0:1616513089.998985:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa5e80 x1690185613470144/t0(0) o400->913dfa1c-5e46-39cb-5577-94f4f5f916cf@10.31.136.7@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513089.998990:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.7@tcp1 00000400:00000200:0.0:1616513089.998996:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.7@tcp1 routed destination 00000100:00000200:8.0:1616513089.998997:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690185613470144, offset 192 00000400:00000200:8.0:1616513089.999003:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.7@tcp1 00000400:00000200:8.0:1616513089.999010:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.7@tcp1 routed destination 00000400:00000200:0.0:1616513089.999011:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.7@tcp1(10.31.136.7@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.999017:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513089.999023:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.7@tcp1(10.31.136.7@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513089.999029:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513089.999030:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.999032:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513089.999034:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.999034:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513089.999036:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513089.999038:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513089.999039:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513089.999042:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513089.999045:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513089.999046:0:50541:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513089.999048:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513089.999060:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513089.999060:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513089.999065:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513089.999067:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513089.999067:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.999070:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513089.999071:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513089.999073:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.999076:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:30.0:1616513089.999077:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a427f8 00000800:00000200:30.0:1616513089.999080:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513089.999081:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000800:00000200:30.0:1616513089.999082:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513089.999085:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513089.999419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513089.999428:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.999458:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.999459:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513089.999466:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.21@tcp1 : PUT - for me 00000800:00000200:16.0:1616513089.999468:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513089.999469:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513089.999473:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.21@tcp1 of length 224 into portal 26 MB=0x5ffd4f0d62940 00000400:00000200:16.0:1616513089.999477:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.21@tcp1 : PUT - for me 00000400:00000200:28.0:1616513089.999478:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.21@tcp1 of length 224/224 into md 0xf97cd499 [1] + 0 00000400:00000200:28.0:1616513089.999483:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3f58 00000400:00000200:28.0:1616513089.999485:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513089.999486:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.21@tcp1 of length 224 into portal 12 MB=0x5ffd4f0d62a80 00000100:00000200:28.0:1616513089.999487:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513089.999492:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3f58 00000400:00000200:16.0:1616513089.999492:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.21@tcp1 of length 224/224 into md 0x865be291 [1] + 16680 00000800:00000200:28.0:1616513089.999494:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513089.999496:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513089.999497:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513089.999498:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513089.999500:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513089.999506:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664922270016 00000800:00000200:16.0:1616513089.999507:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513089.999510:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513089.999512:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513089.999512:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688664922270016/t0(0) o400->086d1d37-5a14-8a93-2296-f93ab44fd2ce@10.31.136.21@tcp1:46/0 lens 224/224 e 0 to 0 dl 1616513181 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513089.999515:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664922270336 00010000:00000200:8.0:1616513089.999521:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5e80 x1688664922270336/t0(0) o400->db00b1a1-46c6-3199-ecd8-6130e67b974c@10.31.136.21@tcp1:715/0 lens 224/224 e 0 to 0 dl 1616513095 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513089.999522:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664922270016, offset 192 00000400:00000200:0.0:1616513089.999526:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.21@tcp1 00000100:00000200:8.0:1616513089.999531:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664922270336, offset 192 00000400:00000200:0.0:1616513089.999532:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.21@tcp1 routed destination 00000400:00000200:8.0:1616513089.999535:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.21@tcp1 00000400:00000200:8.0:1616513089.999541:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.21@tcp1 routed destination 00000400:00000200:0.0:1616513089.999543:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.21@tcp1(10.31.136.21@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513089.999548:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513089.999552:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.21@tcp1(10.31.136.21@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513089.999557:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513089.999557:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.999559:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513089.999561:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513089.999561:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513089.999563:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513089.999564:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513089.999565:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513089.999567:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513089.999568:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513089.999570:0:50541:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513089.999573:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513089.999581:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513089.999584:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513089.999586:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513089.999588:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513089.999592:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513089.999594:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:20.0:1616513089.999597:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513089.999625:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513089.999632:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513089.999634:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513089.999641:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513089.999647:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42220 00000800:00000200:18.0:1616513089.999650:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513089.999653:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.013514:0:50542:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513090.013526:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513090.013533:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513090.013542:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.70@tcp1 : PUT - for me 00000400:00000200:30.0:1616513090.013551:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.70@tcp1 of length 224 into portal 12 MB=0x5ffd46ea20340 00000400:00000200:30.0:1616513090.013557:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.70@tcp1 of length 224/224 into md 0xf964d519 [1] + 85544 00000400:00000200:30.0:1616513090.013562:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513090.013565:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513090.013571:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.013574:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513090.013575:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513090.013578:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.013621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513090.013636:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662737814336 00010000:00000200:26.0:1616513090.013648:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf80000 x1688662737814336/t0(0) o400->6e2e4a07-8313-a6eb-c823-1bc6f49e45cf@10.31.138.70@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513090.013662:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662737814336, offset 192 00000400:00000200:26.0:1616513090.013669:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.70@tcp1 00000400:00000200:26.0:1616513090.013676:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.70@tcp1 routed destination 00000800:00000200:28.0:1616513090.013677:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.013686:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513090.013691:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.70@tcp1(10.31.138.70@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513090.013695:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.70@tcp1 : PUT - for me 00000800:00000200:26.0:1616513090.013698:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513090.013704:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.70@tcp1 of length 224 into portal 26 MB=0x5ffd46ea201c0 00000800:00000200:26.0:1616513090.013704:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513090.013708:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513090.013711:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.70@tcp1 of length 224/224 into md 0xf97cd4a9 [1] + 0 00000800:00000200:26.0:1616513090.013711:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513090.013715:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513090.013716:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3298 00000400:00000200:28.0:1616513090.013719:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:7.2:1616513090.013722:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513090.013723:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513090.013728:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3298 00000800:00000200:16.0:1616513090.013730:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.013731:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.013734:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.013736:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513090.013792:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.013799:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.013802:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1616513090.013806:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662737813952 00000400:00000200:18.0:1616513090.013809:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513090.013812:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeff0 00000800:00000200:18.0:1616513090.013816:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513090.013817:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688662737813952/t0(0) o400->148442f2-a60c-a8f0-338b-4eafb134cbee@10.31.138.70@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513090.013819:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.013831:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662737813952, offset 192 00000400:00000200:0.0:1616513090.013838:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.70@tcp1 00000400:00000200:0.0:1616513090.013845:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.70@tcp1 routed destination 00000400:00000200:0.0:1616513090.013860:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.70@tcp1(10.31.138.70@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.013866:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.013872:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.013875:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.013877:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.013882:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.013887:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.013888:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513090.013941:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.013947:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513090.013950:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513090.013957:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513090.013962:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a435c8 00000800:00000200:24.0:1616513090.013966:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.013969:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.018153:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.018209:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.018219:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.018228:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.184@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.018237:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.184@tcp1 of length 224 into portal 26 MB=0x5ffd535354fc0 00000400:00000200:16.0:1616513090.018244:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.184@tcp1 of length 224/224 into md 0xf97cd4b9 [1] + 0 00000400:00000200:16.0:1616513090.018249:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2770 00000400:00000200:16.0:1616513090.018252:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.018254:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.018259:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2770 00000800:00000200:28.0:1616513090.018261:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.018262:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.018265:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513090.018267:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513090.018270:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.018271:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513090.018279:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.184@tcp1 : PUT - for me 00000400:00000200:28.0:1616513090.018287:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.184@tcp1 of length 224 into portal 12 MB=0x5ffd535355040 00000400:00000200:28.0:1616513090.018293:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.184@tcp1 of length 224/224 into md 0x865be291 [1] + 16904 00000400:00000200:28.0:1616513090.018298:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.018300:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513090.018307:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.018310:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.018312:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.018320:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666069356480 00000100:00000200:8.0:1616513090.018324:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666069356608 00010000:00000200:0.0:1616513090.018330:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688666069356480/t0(0) o400->4054c251-84a7-0a31-daba-b9a1a301de7a@10.31.138.184@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:8.0:1616513090.018334:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa1f80 x1688666069356608/t0(0) o400->22473b26-dda0-e899-1f78-68c971bafc1f@10.31.138.184@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513090.018343:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666069356480, offset 192 00000100:00000200:8.0:1616513090.018347:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666069356608, offset 192 00000400:00000200:0.0:1616513090.018350:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.184@tcp1 00000400:00000200:8.0:1616513090.018355:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.184@tcp1 00000400:00000200:0.0:1616513090.018357:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.184@tcp1 routed destination 00000400:00000200:8.0:1616513090.018366:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.184@tcp1 routed destination 00000400:00000200:0.0:1616513090.018371:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.184@tcp1(10.31.138.184@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.018377:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513090.018379:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.184@tcp1(10.31.138.184@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513090.018385:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.018386:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.018388:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513090.018390:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.018390:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513090.018392:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513090.018394:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.018395:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513090.018398:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.018403:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513090.018408:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513090.018409:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.018420:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.018427:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.018430:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513090.018431:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513090.018449:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.018453:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000800:00000200:20.0:1616513090.018457:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513090.018463:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.018470:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513090.018472:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513090.018473:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.018479:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.018485:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a424c8 00000800:00000200:22.0:1616513090.018488:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.029915:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513090.029970:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.029980:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.029989:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.47@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.029998:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.47@tcp1 of length 224 into portal 26 MB=0x5ffdb617fc0c0 00000400:00000200:18.0:1616513090.030005:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.47@tcp1 of length 224/224 into md 0xf97cd4e9 [1] + 0 00000400:00000200:18.0:1616513090.030009:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2b28 00000400:00000200:18.0:1616513090.030012:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.030015:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513090.030019:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2b28 00000800:00000200:18.0:1616513090.030022:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.030025:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.030027:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513090.030034:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.030036:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.030038:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:30.0:1616513090.030042:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.030042:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.030047:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.47@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.030049:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.030057:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x6039913411000 00000400:00000200:2.0:1616513090.030057:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.47@tcp1 of length 224 into portal 12 MB=0x5ffdb617fc140 00000400:00000200:18.0:1616513090.030062:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0xf964d519 [1] + 85768 00000400:00000200:18.0:1616513090.030066:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.030069:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.47@tcp1 of length 224/224 into md 0xf964d519 [1] + 86104 00000100:00000200:18.0:1616513090.030074:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:2.0:1616513090.030074:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513090.030080:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:2.0:1616513090.030080:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513090.030081:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692582236352 00000800:00000200:18.0:1616513090.030082:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.030083:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.030088:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.030091:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513090.030091:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688692582236352/t0(0) o400->7cca6abc-5ee7-8104-6f3a-bed6a3a3197d@10.31.141.47@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513090.030093:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.030104:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692582236352, offset 192 00000400:00000200:0.0:1616513090.030110:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.47@tcp1 00000100:00000200:30.0:1616513090.030114:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692582236480 00000400:00000200:0.0:1616513090.030116:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.47@tcp1 routed destination 00000100:00000200:2.0:1616513090.030122:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805848174592 00010000:00000200:30.0:1616513090.030125:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0000 x1688692582236480/t0(0) o400->257feac5-efe8-55fe-67ac-8d7ebec07017@10.31.141.47@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513090.030131:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.47@tcp1(10.31.141.47@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.030137:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:30.0:1616513090.030138:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692582236480, offset 192 00000800:00000200:0.0:1616513090.030143:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513090.030144:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.47@tcp1 00000800:00000200:0.0:1616513090.030146:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.030148:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:30.0:1616513090.030151:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.47@tcp1 routed destination 00000800:00000200:0.0:1616513090.030152:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.030158:0:50544:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513090.030158:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:30.0:1616513090.030168:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.47@tcp1(10.31.141.47@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513090.030174:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513090.030179:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513090.030182:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513090.030184:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513090.030188:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.030195:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513090.030199:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.030203:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513090.030205:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:2.0:1616513090.030206:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120cc80 x1692805848174592/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:716/0 lens 336/336 e 0 to 0 dl 1616513096 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513090.030208:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513090.030210:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513090.030214:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43f58 00000800:00000200:30.0:1616513090.030216:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513090.030221:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805848174592, offset 192 00000400:00000200:2.0:1616513090.030226:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:2.0:1616513090.030233:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:2.0:1616513090.030247:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513090.030251:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513090.030253:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513090.030257:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513090.030259:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513090.030260:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513090.030264:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513090.030266:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513090.030266:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513090.030270:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eebb0 00000800:00000200:2.0:1616513090.030270:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513090.030273:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513090.030275:0:50544:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513090.030276:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513090.030287:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.030294:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.030298:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513090.030299:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513090.030305:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.030309:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed100 00000800:00000200:22.0:1616513090.030313:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000001:00080000:1.0:1616513090.031027:0:173083:0:(tgt_lastrcvd.c:902:tgt_cb_last_committed()) boslfs02-MDT0001: transno 77331494633 is committed 00000800:00000200:7.2:1616513090.032576:0:50544:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513090.032632:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513090.032642:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.032642:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.032651:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.106@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.032659:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.106@tcp1 of length 224 into portal 26 MB=0x6001caceeca00 00000400:00000200:18.0:1616513090.032666:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.106@tcp1 of length 224/224 into md 0xf97cd4f9 [1] + 0 00000400:00000200:18.0:1616513090.032671:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3870 00000400:00000200:18.0:1616513090.032673:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.032676:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513090.032680:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3870 00000800:00000200:18.0:1616513090.032683:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.032686:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.032688:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.032745:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688973020678656 00010000:00000200:0.0:1616513090.032756:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688973020678656/t0(0) o400->ccfbc50c-6353-4891-7d33-f6fbad715648@10.31.128.106@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.032771:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688973020678656, offset 192 00000400:00000200:0.0:1616513090.032777:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.106@tcp1 00000400:00000200:0.0:1616513090.032785:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.106@tcp1 routed destination 00000400:00000200:0.0:1616513090.032800:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.106@tcp1(10.31.128.106@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.032806:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.032824:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.032827:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.032828:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.032833:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.032838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.032838:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513090.032905:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.032912:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513090.032914:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513090.032923:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.032928:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43f58 00000800:00000200:16.0:1616513090.032932:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.032935:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.032941:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.032949:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.106@tcp1 : PUT - for me 00000800:00000200:20.0:1616513090.032957:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513090.032957:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.106@tcp1 of length 224 into portal 12 MB=0x6001caceecc00 00000400:00000200:16.0:1616513090.032963:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.106@tcp1 of length 224/224 into md 0x865be291 [1] + 17128 00000400:00000200:16.0:1616513090.032969:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.032971:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513090.032977:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.032980:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.032981:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513090.032993:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688973020679168 00010000:00000200:8.0:1616513090.033002:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1b00 x1688973020679168/t0(0) o400->79cbbef2-79df-af54-cd7f-01446766250f@10.31.128.106@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513090.033014:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688973020679168, offset 192 00000400:00000200:8.0:1616513090.033019:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.106@tcp1 00000400:00000200:8.0:1616513090.033025:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.106@tcp1 routed destination 00000400:00000200:8.0:1616513090.033038:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.106@tcp1(10.31.128.106@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513090.033044:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513090.033049:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513090.033051:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513090.033053:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513090.033057:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.033062:0:50544:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.033119:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.033126:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.033128:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.033135:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.033139:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:2.0:1616513090.033143:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.033146:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.038448:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513090.038459:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513090.038466:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513090.038473:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.39@tcp1 : PUT - for me 00000400:00000200:30.0:1616513090.038482:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.39@tcp1 of length 224 into portal 26 MB=0x6018390c39080 00000400:00000200:30.0:1616513090.038488:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.39@tcp1 of length 224/224 into md 0xf97cd529 [1] + 0 00000400:00000200:30.0:1616513090.038492:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3650 00000400:00000200:30.0:1616513090.038494:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513090.038497:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513090.038501:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3650 00000800:00000200:30.0:1616513090.038503:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.038505:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513090.038507:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.038510:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.038560:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514441343104 00010000:00000200:0.0:1616513090.038569:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1690514441343104/t0(0) o400->e7ff9c89-5f7b-fdcd-36a7-d7b27570cf53@10.31.136.39@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.038582:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690514441343104, offset 192 00000400:00000200:0.0:1616513090.038589:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.39@tcp1 00000400:00000200:0.0:1616513090.038596:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.39@tcp1 routed destination 00000400:00000200:0.0:1616513090.038608:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.39@tcp1(10.31.136.39@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.038614:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513090.038616:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.038619:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.038621:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.038623:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.038627:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.038633:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513090.038635:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513090.038677:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.038687:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513090.038694:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513090.038695:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.39@tcp1 : PUT - for me 00000800:00000200:24.0:1616513090.038700:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513090.038703:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.038704:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.39@tcp1 of length 224 into portal 12 MB=0x6018390c39200 00000400:00000200:24.0:1616513090.038709:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.038711:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.39@tcp1 of length 224/224 into md 0xf964d519 [1] + 86328 00000400:00000200:24.0:1616513090.038715:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a435c8 00000400:00000200:2.0:1616513090.038716:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513090.038718:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:2.0:1616513090.038719:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513090.038723:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513090.038726:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.038729:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513090.038731:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.038735:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513090.038789:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690514441343488 00010000:00000200:10.0:1616513090.038800:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3f00 x1690514441343488/t0(0) o400->2071d204-e1ea-c065-8a44-4819644dcd07@10.31.136.39@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513090.038812:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690514441343488, offset 192 00000400:00000200:10.0:1616513090.038818:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.39@tcp1 00000400:00000200:10.0:1616513090.038824:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.39@tcp1 routed destination 00000400:00000200:10.0:1616513090.038836:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.39@tcp1(10.31.136.39@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513090.038842:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513090.038847:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.038849:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.038851:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.038855:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.038859:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513090.038867:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.038870:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513090.038872:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513090.038874:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513090.038877:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513090.038880:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82440 00000800:00000200:30.0:1616513090.038882:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513090.042541:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928064, portal 4 00000100:00000200:6.0:1616513090.042547:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928064, offset 0 00000400:00000200:6.0:1616513090.042554:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:6.0:1616513090.042560:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:6.0:1616513090.042573:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513090.042579:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:6.0:1616513090.042586:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513090.042589:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:6.0:1616513090.042592:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:6.0:1616513090.042597:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513090.042645:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513090.042665:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513090.042671:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513090.042674:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513090.042681:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513090.042690:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d2d00 x1693067725928064/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513090.042703:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf004c8 00000800:00000200:27.0:1616513090.042708:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513090.042710:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513090.042817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513090.042828:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513090.042834:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513090.042838:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513090.042842:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513090.042850:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627680 00000400:00000200:27.0:1616513090.042857:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716706d [1] + 192 00000400:00000200:27.0:1616513090.042863:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513090.042867:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d2d00 x1693067725928064/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513090.042882:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513090.042885:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513090.042887:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:6.0:1616513090.042890:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d2d00 x1693067725928064/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:6.0:1616513090.042904:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d2d00 x1693067725928064/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513090.042914:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01650 00000800:00000200:7.2:1616513090.044886:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513090.044940:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.044949:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513090.044959:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.123@tcp1 : PUT - for me 00000400:00000200:22.0:1616513090.044969:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.123@tcp1 of length 224 into portal 26 MB=0x603fc0d6db640 00000400:00000200:22.0:1616513090.044975:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.123@tcp1 of length 224/224 into md 0xf97cd539 [1] + 0 00000400:00000200:22.0:1616513090.044981:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f33a8 00000400:00000200:22.0:1616513090.044983:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513090.044986:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513090.044991:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f33a8 00000800:00000200:2.0:1616513090.044992:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.044994:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.044997:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513090.044999:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513090.045002:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.045002:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.045010:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.123@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.045019:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.123@tcp1 of length 224 into portal 12 MB=0x603fc0d6db6c0 00000400:00000200:2.0:1616513090.045025:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.123@tcp1 of length 224/224 into md 0xf964d519 [1] + 86552 00000400:00000200:2.0:1616513090.045030:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.045032:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513090.045038:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.045041:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.045043:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.045050:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230952199744 00010000:00000200:0.0:1616513090.045060:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1693230952199744/t0(0) o400->e90f30d4-c8ff-54e7-615d-fd52d5d38bb6@10.31.138.123@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.045072:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230952199744, offset 192 00000400:00000200:0.0:1616513090.045077:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.123@tcp1 00000400:00000200:0.0:1616513090.045084:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.123@tcp1 routed destination 00000400:00000200:0.0:1616513090.045097:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.123@tcp1(10.31.138.123@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513090.045103:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230952199872 00000800:00000200:0.0:1616513090.045103:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.045108:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.045110:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.045112:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:10.0:1616513090.045116:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6300 x1693230952199872/t0(0) o400->cc48007b-cf98-608f-76e9-cbd28dd486fe@10.31.138.123@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513090.045116:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.045122:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.045122:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:10.0:1616513090.045131:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230952199872, offset 192 00000400:00000200:10.0:1616513090.045137:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.123@tcp1 00000400:00000200:10.0:1616513090.045145:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.123@tcp1 routed destination 00000400:00000200:10.0:1616513090.045159:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.123@tcp1(10.31.138.123@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513090.045166:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513090.045172:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.045174:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513090.045176:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513090.045176:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.045181:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513090.045182:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513090.045185:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513090.045186:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513090.045191:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:30.0:1616513090.045194:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513090.045196:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a427f8 00000800:00000200:30.0:1616513090.045199:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513090.045199:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:30.0:1616513090.045201:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513090.045202:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513090.045203:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513090.045206:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513090.045209:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83a08 00000800:00000200:30.0:1616513090.045212:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513090.074521:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928080, portal 4 00000100:00000200:30.0:1616513090.074526:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928080, offset 0 00000400:00000200:30.0:1616513090.074532:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:30.0:1616513090.074537:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:30.0:1616513090.074549:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513090.074556:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513090.074561:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513090.074564:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513090.074566:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513090.074571:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513090.074618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513090.074633:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513090.074639:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513090.074643:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.074650:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513090.074655:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d4800 x1693067725928080/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513090.074666:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efa08 00000800:00000200:2.0:1616513090.074670:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513090.074673:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513090.074795:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513090.074851:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513090.074860:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:30.0:1616513090.074861:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513090.074869:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513090.074876:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627690 00000400:00000200:22.0:1616513090.074883:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cdde9 [1] + 192 00000400:00000200:22.0:1616513090.074887:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513090.074892:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d4800 x1693067725928080/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513090.074906:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513090.074909:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:30.0:1616513090.074911:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d4800 x1693067725928080/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513090.074911:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:30.0:1616513090.074933:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d4800 x1693067725928080/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513090.074941:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeaa0 00000800:00000200:11.2:1616513090.132426:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513090.132483:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.132493:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.132503:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.80@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.132514:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.80@tcp1 of length 224 into portal 12 MB=0x5ffdda4e30e00 00000400:00000200:24.0:1616513090.132520:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.80@tcp1 of length 224/224 into md 0x865be291 [1] + 17352 00000400:00000200:24.0:1616513090.132526:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.132529:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513090.132537:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.132538:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.132541:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.132542:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513090.132563:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702302752256 00010000:00000200:8.0:1616513090.132578:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e2d00 x1688702302752256/t0(0) o400->b99b322f-3344-92c6-e69c-bd0caba8f79e@10.31.136.80@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513090.132594:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702302752256, offset 192 00000800:00000200:7.2:1616513090.132599:0:50554:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513090.132600:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.80@tcp1 00000400:00000200:8.0:1616513090.132610:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.80@tcp1 routed destination 00000400:00000200:8.0:1616513090.132625:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.80@tcp1(10.31.136.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513090.132632:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513090.132639:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513090.132641:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513090.132645:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513090.132651:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.132655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513090.132656:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.132666:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.132676:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.80@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.132686:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.80@tcp1 of length 224 into portal 26 MB=0x5ffdda4e30cc0 00000400:00000200:2.0:1616513090.132693:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.80@tcp1 of length 224/224 into md 0xf97cd549 [1] + 0 00000400:00000200:2.0:1616513090.132698:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3188 00000400:00000200:2.0:1616513090.132701:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.132704:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513090.132709:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3188 00000800:00000200:16.0:1616513090.132710:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513090.132712:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.132715:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513090.132717:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513090.132717:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513090.132720:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513090.132720:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513090.132727:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.132731:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:16.0:1616513090.132735:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.132738:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.132792:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702302751936 00010000:00000200:0.0:1616513090.132805:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688702302751936/t0(0) o400->05e0593d-20dc-8b60-f0bb-ee1fa3d30f8d@10.31.136.80@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.132820:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702302751936, offset 192 00000400:00000200:0.0:1616513090.132826:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.80@tcp1 00000400:00000200:0.0:1616513090.132834:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.80@tcp1 routed destination 00000400:00000200:0.0:1616513090.132850:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.80@tcp1(10.31.136.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.132857:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513090.132865:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.132868:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.132870:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513090.132876:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.132881:0:50554:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513090.132883:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513090.132892:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.132898:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513090.132901:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513090.132906:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513090.132911:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ed0 00000800:00000200:30.0:1616513090.132915:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513090.132917:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513090.138545:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928096, portal 4 00000100:00000200:30.0:1616513090.138550:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928096, offset 0 00000400:00000200:30.0:1616513090.138555:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:30.0:1616513090.138560:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:30.0:1616513090.138571:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513090.138577:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513090.138582:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513090.138584:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513090.138586:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513090.138590:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513090.138636:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513090.138696:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513090.138703:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513090.138706:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513090.138709:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513090.138713:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513090.138719:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d7461208d80 x1693067725928096/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513090.138732:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efa08 00000800:00000200:2.0:1616513090.138736:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513090.138819:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513090.138829:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513090.138834:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:30.0:1616513090.138840:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:30.0:1616513090.138846:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6276a0 00000400:00000200:30.0:1616513090.138851:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cde19 [1] + 192 00000400:00000200:30.0:1616513090.138855:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513090.138859:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d7461208d80 x1693067725928096/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:30.0:1616513090.138871:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:30.0:1616513090.138873:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:30.0:1616513090.138875:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:30.0:1616513090.138877:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:2.0:1616513090.138880:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d7461208d80 x1693067725928096/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513090.138892:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d7461208d80 x1693067725928096/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513090.138901:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeaa0 00000800:00000200:7.2:1616513090.150721:0:50556:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513090.150777:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.150787:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.150796:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.109@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.150806:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.109@tcp1 of length 224 into portal 26 MB=0x60000979ec380 00000400:00000200:18.0:1616513090.150813:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.109@tcp1 of length 224/224 into md 0xf97cd559 [1] + 0 00000400:00000200:18.0:1616513090.150818:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba770 00000400:00000200:18.0:1616513090.150833:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.150836:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513090.150840:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba770 00000800:00000200:2.0:1616513090.150840:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.150843:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.150846:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.150847:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.150868:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688852404028288 00010000:00000200:0.0:1616513090.150881:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688852404028288/t0(0) o400->bdb03bcd-1c26-4e69-2558-cf5f1c412867@10.31.138.109@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.150895:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688852404028288, offset 192 00000400:00000200:0.0:1616513090.150902:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.109@tcp1 00000400:00000200:0.0:1616513090.150910:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.109@tcp1 routed destination 00000400:00000200:0.0:1616513090.150926:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.109@tcp1(10.31.138.109@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.150932:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.150938:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.150941:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.150943:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.150948:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.150955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.150955:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513090.151010:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.151017:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.151020:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.151026:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.151032:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42000 00000800:00000200:20.0:1616513090.151035:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.151038:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.151044:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513090.151052:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.109@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.151061:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.109@tcp1 of length 224 into portal 12 MB=0x60000979ec400 00000800:00000200:24.0:1616513090.151064:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513090.151067:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.109@tcp1 of length 224/224 into md 0x865be291 [1] + 17576 00000400:00000200:20.0:1616513090.151074:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.151076:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.151082:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.151085:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.151087:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513090.151098:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688852404028416 00010000:00000200:8.0:1616513090.151108:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad3180 x1688852404028416/t0(0) o400->117a078c-5ccd-aa2c-233c-689d51715455@10.31.138.109@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513090.151119:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688852404028416, offset 192 00000400:00000200:8.0:1616513090.151124:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.109@tcp1 00000400:00000200:8.0:1616513090.151131:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.109@tcp1 routed destination 00000400:00000200:8.0:1616513090.151143:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.109@tcp1(10.31.138.109@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513090.151148:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513090.151153:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513090.151155:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513090.151157:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513090.151161:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.151167:0:50556:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.151221:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.151229:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.151231:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513090.151232:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513090.151240:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.151245:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:22.0:1616513090.151249:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.154937:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513090.154991:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.155001:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513090.155010:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.201@tcp1 : PUT - for me 00000400:00000200:28.0:1616513090.155019:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.201@tcp1 of length 224 into portal 26 MB=0x60527a16081c0 00000400:00000200:28.0:1616513090.155026:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.201@tcp1 of length 224/224 into md 0xf97cd589 [1] + 0 00000400:00000200:28.0:1616513090.155030:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba908 00000400:00000200:28.0:1616513090.155033:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.155036:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513090.155040:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba908 00000800:00000200:28.0:1616513090.155043:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.155045:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.155046:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.155052:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.155055:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.155055:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.155064:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.201@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.155073:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.201@tcp1 of length 224 into portal 12 MB=0x60527a1608400 00000400:00000200:24.0:1616513090.155079:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.201@tcp1 of length 224/224 into md 0x865be291 [1] + 17800 00000400:00000200:24.0:1616513090.155084:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.155087:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513090.155094:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.155097:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.155099:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513090.155112:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694517629584384 00000100:00000200:0.0:1616513090.155114:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694517629583808 00010000:00000200:8.0:1616513090.155123:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e0900 x1694517629584384/t0(0) o400->de7fbc53-bb90-1ff3-58b4-b0a9d37148ab@10.31.142.201@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:0.0:1616513090.155125:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1694517629583808/t0(0) o400->64c90382-4b3b-dd66-a81d-e34fac50a7c8@10.31.142.201@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513090.155136:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694517629584384, offset 192 00000100:00000200:0.0:1616513090.155139:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694517629583808, offset 192 00000400:00000200:8.0:1616513090.155142:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.201@tcp1 00000400:00000200:0.0:1616513090.155148:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.201@tcp1 00000400:00000200:8.0:1616513090.155149:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.201@tcp1 routed destination 00000400:00000200:8.0:1616513090.155164:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.201@tcp1(10.31.142.201@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513090.155170:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513090.155172:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.201@tcp1 routed destination 00000800:00000200:8.0:1616513090.155180:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513090.155182:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513090.155184:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513090.155187:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.201@tcp1(10.31.142.201@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513090.155189:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.155193:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.155194:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513090.155200:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.155203:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.155205:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513090.155209:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.155216:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513090.155218:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513090.155246:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.155253:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513090.155256:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513090.155257:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513090.155276:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.155280:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2000 00000800:00000200:16.0:1616513090.155284:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513090.155295:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.155302:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.155304:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.155311:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.155317:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42aa0 00000800:00000200:2.0:1616513090.155320:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.155323:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.178782:0:50559:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513090.178795:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513090.178804:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513090.178813:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.161@tcp1 : PUT - for me 00000400:00000200:30.0:1616513090.178823:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.161@tcp1 of length 224 into portal 26 MB=0x603e12d508640 00000400:00000200:30.0:1616513090.178830:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.161@tcp1 of length 224/224 into md 0xf97cd5a9 [1] + 0 00000400:00000200:30.0:1616513090.178835:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb4b8 00000400:00000200:30.0:1616513090.178839:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513090.178842:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513090.178846:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb4b8 00000800:00000200:30.0:1616513090.178849:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.178852:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513090.178854:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513090.178857:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.178914:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115523040832 00010000:00000200:0.0:1616513090.178925:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1693115523040832/t0(0) o400->da018e2f-82d0-5775-9ac8-a1b335505c69@10.31.136.161@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.178939:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693115523040832, offset 192 00000400:00000200:0.0:1616513090.178944:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.161@tcp1 00000400:00000200:0.0:1616513090.178952:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.161@tcp1 routed destination 00000400:00000200:0.0:1616513090.178966:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.161@tcp1(10.31.136.161@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.178972:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.178978:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.178980:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.178982:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.178987:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.178994:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.178994:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513090.179048:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.179054:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513090.179057:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513090.179064:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513090.179070:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42990 00000800:00000200:24.0:1616513090.179073:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.179076:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513090.179121:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513090.179176:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.179186:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.179195:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.161@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.179203:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.161@tcp1 of length 224 into portal 12 MB=0x603e12d5086c0 00000400:00000200:2.0:1616513090.179210:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.161@tcp1 of length 224/224 into md 0xf964d519 [1] + 86776 00000400:00000200:2.0:1616513090.179215:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.179218:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513090.179225:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.179229:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.179230:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.179233:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513090.179250:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115523040960 00010000:00000200:10.0:1616513090.179262:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3180 x1693115523040960/t0(0) o400->93425747-0d21-f029-a294-8bbd4ca7396e@10.31.136.161@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513090.179276:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693115523040960, offset 192 00000400:00000200:10.0:1616513090.179281:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.161@tcp1 00000400:00000200:10.0:1616513090.179288:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.161@tcp1 routed destination 00000400:00000200:10.0:1616513090.179302:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.161@tcp1(10.31.136.161@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513090.179308:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513090.179314:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.179317:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.179319:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.179324:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.179329:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513090.179338:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.179342:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513090.179344:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513090.179347:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:30.0:1616513090.179349:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513090.179352:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e835c8 00000800:00000200:30.0:1616513090.179355:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:3.0:1616513090.186576:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928112, portal 4 00000100:00000200:3.0:1616513090.186581:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928112, offset 0 00000400:00000200:3.0:1616513090.186588:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:3.0:1616513090.186595:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:3.0:1616513090.186608:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513090.186615:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000100:00000200:19.0:1616513090.186616:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928128, portal 4 00000100:00000200:19.0:1616513090.186622:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928128, offset 0 00000800:00000200:3.0:1616513090.186622:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513090.186627:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000400:00000200:19.0:1616513090.186629:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513090.186629:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:19.0:1616513090.186635:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:3.0:1616513090.186635:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:19.0:1616513090.186649:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513090.186655:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:19.0:1616513090.186662:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513090.186665:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:19.0:1616513090.186667:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:19.0:1616513090.186672:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:27.2:1616513090.186681:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513090.186700:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513090.186705:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513090.186707:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513090.186714:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:5.2:1616513090.186719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:27.0:1616513090.186720:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be43600 x1693067725928112/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513090.186732:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2ff0 00000800:00000200:27.0:1616513090.186736:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513090.186739:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:28.0:1616513090.186776:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513090.186783:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513090.186786:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513090.186793:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513090.186799:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be40000 x1693067725928128/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513090.186812:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707980 00000800:00000200:28.0:1616513090.186816:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513090.186819:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.2:1616513090.186852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513090.186862:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513090.186868:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513090.186872:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513090.186876:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513090.186884:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6276b0 00000400:00000200:27.0:1616513090.186890:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716707d [1] + 192 00000400:00000200:27.0:1616513090.186895:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513090.186902:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be43600 x1693067725928112/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.2:1616513090.186912:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:27.0:1616513090.186916:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513090.186919:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513090.186921:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:3.0:1616513090.186924:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be43600 x1693067725928112/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513090.186937:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be43600 x1693067725928112/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513090.186946:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:24.0:1616513090.186964:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513090.186974:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:24.0:1616513090.186983:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:24.0:1616513090.186991:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6276c0 00000400:00000200:24.0:1616513090.187000:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866da5e1 [1] + 192 00000400:00000200:24.0:1616513090.187005:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.187010:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be40000 x1693067725928128/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513090.187017:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513090.187026:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513090.187029:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513090.187031:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:19.0:1616513090.187035:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be40000 x1693067725928128/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513090.187048:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be40000 x1693067725928128/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513090.187058:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000100:00000200:14.0:1616513090.202538:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928144, portal 4 00000100:00000200:14.0:1616513090.202544:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928144, offset 0 00000400:00000200:14.0:1616513090.202551:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:14.0:1616513090.202557:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:14.0:1616513090.202570:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513090.202576:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513090.202583:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.202586:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513090.202588:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513090.202593:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513090.202640:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513090.202656:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513090.202665:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513090.202668:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513090.202674:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513090.202681:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bd9b00 x1693067725928144/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513090.202693:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756d48 00000800:00000200:18.0:1616513090.202697:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513090.202700:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513090.202809:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513090.202863:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513090.202873:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:30.0:1616513090.202875:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513090.202882:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513090.202890:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6276d0 00000400:00000200:22.0:1616513090.202897:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cde79 [1] + 192 00000400:00000200:22.0:1616513090.202902:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513090.202907:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bd9b00 x1693067725928144/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513090.202922:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513090.202925:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513090.202927:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:14.0:1616513090.202929:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bd9b00 x1693067725928144/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513090.202942:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bd9b00 x1693067725928144/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513090.202952:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757cb0 00000800:00000200:7.2:1616513090.214765:0:50563:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513090.214790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513090.214822:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.214831:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.214840:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.200@tcp1 : PUT - for me 00000800:00000200:28.0:1616513090.214849:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513090.214850:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.200@tcp1 of length 224 into portal 26 MB=0x605a073014a40 00000400:00000200:18.0:1616513090.214857:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.200@tcp1 of length 224/224 into md 0xf97cd5c9 [1] + 0 00000800:00000200:28.0:1616513090.214858:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:18.0:1616513090.214862:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba3b8 00000400:00000200:18.0:1616513090.214864:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513090.214867:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.200@tcp1 : PUT - for me 00000100:00000200:18.0:1616513090.214867:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513090.214871:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba3b8 00000800:00000200:18.0:1616513090.214875:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513090.214876:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.200@tcp1 of length 224 into portal 12 MB=0x605a073014ac0 00000800:00000200:2.0:1616513090.214877:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.214878:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.214881:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513090.214883:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.200@tcp1 of length 224/224 into md 0x865be291 [1] + 18024 00000400:00000200:28.0:1616513090.214888:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.214891:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513090.214898:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.214901:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.214902:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.214905:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513090.214936:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036542634688 00010000:00000200:8.0:1616513090.214961:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5772fa6300 x1695036542634688/t0(0) o400->97cba773-fbd1-7902-a455-1fcc0c268c9f@10.31.129.200@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513090.214970:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036542634560 00000100:00000200:8.0:1616513090.214975:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036542634688, offset 192 00000400:00000200:8.0:1616513090.214982:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.200@tcp1 00010000:00000200:0.0:1616513090.214982:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1695036542634560/t0(0) o400->ce57fde2-8b49-f62d-1f45-3404658c704b@10.31.129.200@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513090.214989:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.200@tcp1 routed destination 00000100:00000200:0.0:1616513090.214998:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036542634560, offset 192 00000400:00000200:8.0:1616513090.215003:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.200@tcp1(10.31.129.200@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513090.215004:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.200@tcp1 00000800:00000200:8.0:1616513090.215009:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513090.215012:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.200@tcp1 routed destination 00000800:00000200:8.0:1616513090.215019:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513090.215022:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513090.215024:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513090.215027:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.200@tcp1(10.31.129.200@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513090.215028:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513090.215033:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513090.215034:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.215040:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.215043:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.215045:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513090.215046:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513090.215050:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513090.215053:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.215056:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1616513090.215057:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513090.215063:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513090.215066:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513090.215067:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2198 00000800:00000200:20.0:1616513090.215071:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.215074:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513090.215076:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.215080:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513090.215083:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513090.215089:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513090.215094:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43650 00000800:00000200:30.0:1616513090.215097:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513090.215099:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.225203:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513090.225261:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.225270:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513090.225279:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.13@tcp1 : PUT - for me 00000400:00000200:28.0:1616513090.225288:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.13@tcp1 of length 224 into portal 26 MB=0x5ffdcc9fd8b80 00000400:00000200:28.0:1616513090.225295:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.13@tcp1 of length 224/224 into md 0xf97cd609 [1] + 0 00000400:00000200:28.0:1616513090.225300:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6badd0 00000400:00000200:28.0:1616513090.225303:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.225305:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513090.225310:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6badd0 00000800:00000200:28.0:1616513090.225313:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.225315:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.225318:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.225323:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.225326:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.225327:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.225336:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.13@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.225345:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.13@tcp1 of length 224 into portal 12 MB=0x5ffdcc9fd8d00 00000400:00000200:16.0:1616513090.225351:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.13@tcp1 of length 224/224 into md 0x865be291 [1] + 18248 00000400:00000200:16.0:1616513090.225356:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.225359:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513090.225367:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.225371:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.225373:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.225375:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698630278016 00010000:00000200:0.0:1616513090.225387:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688698630278016/t0(0) o400->9c026075-68ac-3dff-f253-4f69996f8f96@10.31.133.13@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513090.225390:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698630278400 00000100:00000200:0.0:1616513090.225401:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698630278016, offset 192 00010000:00000200:12.0:1616513090.225403:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd1f80 x1688698630278400/t0(0) o400->e01f68ea-21f0-a9fa-3823-ebf6dcae93ae@10.31.133.13@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513090.225409:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.13@tcp1 00000400:00000200:0.0:1616513090.225416:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.13@tcp1 routed destination 00000100:00000200:12.0:1616513090.225417:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698630278400, offset 192 00000400:00000200:12.0:1616513090.225423:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.13@tcp1 00000400:00000200:0.0:1616513090.225432:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.13@tcp1(10.31.133.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.225456:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513090.225458:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.13@tcp1 routed destination 00000800:00000200:0.0:1616513090.225466:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.225469:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.225471:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513090.225473:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.13@tcp1(10.31.133.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.225476:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513090.225480:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513090.225481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.225484:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513090.225486:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.225488:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.225490:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513090.225494:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513090.225495:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513090.225501:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513090.225503:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513090.225504:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513090.225506:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513090.225511:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513090.225516:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42660 00000800:00000200:24.0:1616513090.225519:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513090.225563:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.225571:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.225574:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.225580:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.225584:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e990 00000800:00000200:2.0:1616513090.225589:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.225591:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.227556:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.227625:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.227634:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.227642:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.104@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.227651:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.104@tcp1 of length 224 into portal 26 MB=0x601872e262900 00000400:00000200:16.0:1616513090.227658:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.104@tcp1 of length 224/224 into md 0xf97cd629 [1] + 0 00000400:00000200:16.0:1616513090.227662:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bbba0 00000400:00000200:16.0:1616513090.227665:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.227667:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.227672:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbba0 00000800:00000200:16.0:1616513090.227674:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.227677:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513090.227679:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.227686:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513090.227689:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.227690:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513090.227698:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.104@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.227707:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.104@tcp1 of length 224 into portal 12 MB=0x601872e262a00 00000400:00000200:20.0:1616513090.227714:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.104@tcp1 of length 224/224 into md 0x865be291 [1] + 18472 00000400:00000200:20.0:1616513090.227719:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.227722:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.227729:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.227732:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.227733:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690529966729472 00000800:00000200:20.0:1616513090.227734:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.227741:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690529966729728 00010000:00000200:0.0:1616513090.227744:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1690529966729472/t0(0) o400->176e1be7-5954-b366-4b8d-00198ddf3218@10.31.138.104@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:12.0:1616513090.227747:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad4c80 x1690529966729728/t0(0) o400->4225e9e9-6a12-a0b3-6420-0d5229921a50@10.31.138.104@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.227757:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690529966729728, offset 192 00000100:00000200:0.0:1616513090.227757:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690529966729472, offset 192 00000400:00000200:12.0:1616513090.227762:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.104@tcp1 00000400:00000200:0.0:1616513090.227768:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.104@tcp1 00000400:00000200:12.0:1616513090.227769:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.104@tcp1 routed destination 00000400:00000200:12.0:1616513090.227782:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.104@tcp1(10.31.138.104@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.227787:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513090.227790:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.104@tcp1 routed destination 00000800:00000200:12.0:1616513090.227796:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.227798:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513090.227800:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513090.227803:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513090.227804:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.104@tcp1(10.31.138.104@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513090.227809:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.227810:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513090.227815:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.227818:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513090.227820:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513090.227820:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513090.227824:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513090.227827:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513090.227829:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513090.227830:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513090.227831:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.227831:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513090.227837:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:30.0:1616513090.227839:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513090.227841:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fc28 00000800:00000200:24.0:1616513090.227845:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:30.0:1616513090.227846:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513090.227849:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513090.227856:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513090.227861:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42f68 00000800:00000200:30.0:1616513090.227865:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513090.227868:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513090.234564:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928160, portal 4 00000100:00000200:14.0:1616513090.234569:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928160, offset 0 00000400:00000200:14.0:1616513090.234575:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:14.0:1616513090.234581:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:14.0:1616513090.234593:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513090.234600:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:14.0:1616513090.234605:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.234608:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:14.0:1616513090.234612:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:14.0:1616513090.234617:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513090.234665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.234684:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513090.234690:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513090.234693:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513090.234700:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513090.234707:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bdc380 x1693067725928160/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513090.234720:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756c38 00000800:00000200:1.0:1616513090.234725:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513090.234728:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513090.234863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513090.234873:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.234880:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513090.234883:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513090.234888:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513090.234896:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6276e0 00000400:00000200:1.0:1616513090.234903:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436fff35 [1] + 192 00000400:00000200:1.0:1616513090.234908:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513090.234913:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bdc380 x1693067725928160/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513090.234927:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.234930:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513090.234931:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:14.0:1616513090.234936:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bdc380 x1693067725928160/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513090.234949:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bdc380 x1693067725928160/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513090.234971:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757cb0 00000800:00000200:7.2:1616513090.267096:0:50569:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513090.267154:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.267164:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.267174:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.137@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.267185:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.137@tcp1 of length 224 into portal 26 MB=0x5ffd3d7e41d00 00000400:00000200:2.0:1616513090.267192:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.137@tcp1 of length 224/224 into md 0xf97cd649 [1] + 0 00000400:00000200:2.0:1616513090.267197:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba088 00000400:00000200:2.0:1616513090.267202:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.267205:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513090.267209:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba088 00000800:00000200:18.0:1616513090.267210:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.267213:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.267216:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.267218:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.267279:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660208786688 00010000:00000200:0.0:1616513090.267291:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688660208786688/t0(0) o400->f1056264-3ef2-2a8a-3df5-91b1ca6b839c@10.31.136.137@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.267307:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660208786688, offset 192 00000400:00000200:0.0:1616513090.267313:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.137@tcp1 00000400:00000200:0.0:1616513090.267321:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.137@tcp1 routed destination 00000400:00000200:0.0:1616513090.267336:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.137@tcp1(10.31.136.137@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.267347:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.267353:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.267356:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.267358:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.267364:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.267370:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.267371:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513090.267425:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.267431:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.267449:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.267456:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.267462:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42908 00000800:00000200:20.0:1616513090.267465:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.267468:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513090.267908:0:50569:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513090.267921:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513090.267930:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513090.267938:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.137@tcp1 : PUT - for me 00000400:00000200:30.0:1616513090.267948:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.137@tcp1 of length 224 into portal 12 MB=0x5ffd3d7e41ec0 00000400:00000200:30.0:1616513090.267954:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.137@tcp1 of length 224/224 into md 0xf964d519 [1] + 87000 00000400:00000200:30.0:1616513090.267959:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513090.267963:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513090.267971:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.267974:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513090.267976:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.267979:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513090.268038:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660208787136 00010000:00000200:10.0:1616513090.268050:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdaf87500 x1688660208787136/t0(0) o400->c45ccdd4-fd81-0f82-5353-895427140ade@10.31.136.137@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513090.268066:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660208787136, offset 192 00000400:00000200:10.0:1616513090.268072:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.137@tcp1 00000400:00000200:10.0:1616513090.268080:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.137@tcp1 routed destination 00000400:00000200:10.0:1616513090.268095:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.137@tcp1(10.31.136.137@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513090.268108:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513090.268114:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.268117:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.268119:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.268124:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.268129:0:50569:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.268140:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.268146:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.268148:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513090.268150:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513090.268155:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513090.268159:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e835c8 00000800:00000200:18.0:1616513090.268163:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513090.282528:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928176, portal 4 00000100:00000200:10.0:1616513090.282533:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928176, offset 0 00000400:00000200:10.0:1616513090.282540:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:10.0:1616513090.282546:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:10.0:1616513090.282558:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513090.282565:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:10.0:1616513090.282571:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.282574:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000100:00000200:23.0:1616513090.282575:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928192, portal 4 00000800:00000200:10.0:1616513090.282577:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:23.0:1616513090.282581:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928192, offset 0 00000800:00000200:10.0:1616513090.282582:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:23.0:1616513090.282588:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:23.0:1616513090.282596:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:23.0:1616513090.282610:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513090.282617:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:23.0:1616513090.282623:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513090.282626:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:23.0:1616513090.282629:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:1.2:1616513090.282629:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:23.0:1616513090.282634:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:1.0:1616513090.282647:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513090.282655:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513090.282657:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513090.282665:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928208, portal 4 00000400:00000200:1.0:1616513090.282665:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:8.0:1616513090.282670:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928208, offset 0 00000100:00000200:1.0:1616513090.282672:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf84380 x1693067725928176/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513090.282676:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:5.2:1616513090.282680:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:8.0:1616513090.282682:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:1.0:1616513090.282685:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82440 00000400:00000200:8.0:1616513090.282694:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:5.0:1616513090.282696:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:8.0:1616513090.282700:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:1.0:1616513090.282700:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:8.0:1616513090.282707:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513090.282709:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:8.0:1616513090.282711:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:8.0:1616513090.282716:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513090.282722:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.282731:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:16.0:1616513090.282735:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:1.0:1616513090.282736:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513090.282738:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513090.282740:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:16.0:1616513090.282742:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:1.0:1616513090.282744:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.282745:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513090.282747:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:1.0:1616513090.282751:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb23e780 x1693067725928208/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513090.282752:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:1.2:1616513090.282754:0:6468:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:16.0:1616513090.282758:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be47980 x1693067725928192/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513090.282762:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:1.0:1616513090.282768:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000400:00000200:16.0:1616513090.282770:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f298 00000800:00000200:5.0:1616513090.282771:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:1.0:1616513090.282771:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513090.282774:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:16.0:1616513090.282775:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000400:00000200:5.0:1616513090.282780:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:5.0:1616513090.282789:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6276f0 00000400:00000200:5.0:1616513090.282796:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436fff45 [1] + 192 00000400:00000200:5.0:1616513090.282801:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:5.0:1616513090.282806:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf84380 x1693067725928176/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.2:1616513090.282819:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513090.282822:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:5.0:1616513090.282825:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:5.0:1616513090.282826:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513090.282828:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:10.0:1616513090.282831:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf84380 x1693067725928176/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513090.282834:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513090.282835:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513090.282842:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000100:00000200:10.0:1616513090.282845:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf84380 x1693067725928176/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513090.282851:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627710 00000400:00000200:10.0:1616513090.282855:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83a08 00000400:00000200:1.0:1616513090.282860:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436fff55 [1] + 192 00000400:00000200:1.0:1616513090.282865:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:5.2:1616513090.282868:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000100:00000200:1.0:1616513090.282869:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb23e780 x1693067725928208/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513090.282881:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:1.0:1616513090.282883:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.282885:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513090.282887:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:20.0:1616513090.282890:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000100:00000200:8.0:1616513090.282890:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb23e780 x1693067725928208/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513090.282899:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000100:00000200:8.0:1616513090.282902:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb23e780 x1693067725928208/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513090.282907:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627700 00000400:00000200:8.0:1616513090.282911:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000400:00000200:20.0:1616513090.282916:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866da661 [1] + 192 00000400:00000200:20.0:1616513090.282921:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.282926:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be47980 x1693067725928192/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:28.0:1616513090.282934:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513090.282941:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513090.282944:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513090.282946:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:23.0:1616513090.282951:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be47980 x1693067725928192/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513090.282965:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be47980 x1693067725928192/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513090.282975:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f4b8 00000800:00000200:7.2:1616513090.283472:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513090.283483:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513090.283490:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:30.0:1616513090.283498:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.104@tcp1 : PUT - for me 00000400:00000200:30.0:1616513090.283507:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.104@tcp1 of length 224 into portal 26 MB=0x5ffdbbb704b40 00000400:00000200:30.0:1616513090.283513:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.104@tcp1 of length 224/224 into md 0xf97cd669 [1] + 0 00000400:00000200:30.0:1616513090.283517:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba220 00000400:00000200:30.0:1616513090.283520:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:30.0:1616513090.283522:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:30.0:1616513090.283526:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba220 00000800:00000200:30.0:1616513090.283529:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.283531:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:30.0:1616513090.283532:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513090.283535:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.283593:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694091172672 00010000:00000200:0.0:1616513090.283604:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688694091172672/t0(0) o400->922d6b1b-8aae-2b02-5717-230eb23c9048@10.31.136.104@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.283618:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694091172672, offset 192 00000400:00000200:0.0:1616513090.283625:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.104@tcp1 00000400:00000200:0.0:1616513090.283632:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.104@tcp1 routed destination 00000400:00000200:0.0:1616513090.283647:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.104@tcp1(10.31.136.104@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513090.283648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.283654:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513090.283660:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.283660:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.283662:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513090.283665:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513090.283669:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.283669:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513090.283671:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:11.2:1616513090.283674:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513090.283676:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513090.283678:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.104@tcp1 : PUT - for me 00000800:00000200:28.0:1616513090.283685:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:24.0:1616513090.283686:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.104@tcp1 of length 224 into portal 12 MB=0x5ffdbbb704d40 00000800:00000200:28.0:1616513090.283690:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513090.283692:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513090.283693:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.104@tcp1 of length 224/224 into md 0x865be291 [1] + 18696 00000800:00000200:20.0:1616513090.283694:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513090.283698:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513090.283699:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:24.0:1616513090.283701:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513090.283708:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513090.283709:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a423b8 00000800:00000200:24.0:1616513090.283711:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.283713:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.283715:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.283771:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694091173184 00010000:00000200:12.0:1616513090.283783:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e6780 x1688694091173184/t0(0) o400->2497b25c-cb88-0791-92da-b06b73851fe9@10.31.136.104@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.283796:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694091173184, offset 192 00000400:00000200:12.0:1616513090.283802:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.104@tcp1 00000400:00000200:12.0:1616513090.283809:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.104@tcp1 routed destination 00000400:00000200:12.0:1616513090.283823:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.104@tcp1(10.31.136.104@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.283829:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.283834:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.283837:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.283838:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.283843:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.283847:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.283903:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.283910:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.283912:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.283919:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.283923:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e220 00000800:00000200:2.0:1616513090.283927:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.283930:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.291877:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.291934:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.291946:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.291955:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.34@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.291964:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.34@tcp1 of length 224 into portal 26 MB=0x5ffe158faccc0 00000400:00000200:16.0:1616513090.291970:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.34@tcp1 of length 224/224 into md 0xf97cd699 [1] + 0 00000400:00000200:16.0:1616513090.291975:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb760 00000400:00000200:16.0:1616513090.291978:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.291981:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.291985:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb760 00000800:00000200:16.0:1616513090.291988:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.291989:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.291991:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513090.291996:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513090.291999:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.292000:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513090.292008:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.34@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.292017:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.34@tcp1 of length 224 into portal 12 MB=0x5ffe158facd40 00000400:00000200:20.0:1616513090.292023:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.34@tcp1 of length 224/224 into md 0x865be291 [1] + 18920 00000400:00000200:20.0:1616513090.292029:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.292031:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.292038:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.292041:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.292043:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.292049:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688718209109184 00010000:00000200:0.0:1616513090.292061:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688718209109184/t0(0) o400->ba8f5b2b-bc81-4a45-f396-1e3a5277f245@10.31.142.34@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.292075:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688718209109184, offset 192 00000400:00000200:0.0:1616513090.292081:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.34@tcp1 00000400:00000200:0.0:1616513090.292088:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.34@tcp1 routed destination 00000100:00000200:12.0:1616513090.292101:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688718209109312 00000400:00000200:0.0:1616513090.292103:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.34@tcp1(10.31.142.34@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.292109:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513090.292113:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0480 x1688718209109312/t0(0) o400->aced667b-9593-564c-3913-b108be3ecdd7@10.31.142.34@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513090.292115:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.292118:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.292122:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513090.292127:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688718209109312, offset 192 00000800:00000200:0.0:1616513090.292127:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513090.292134:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.34@tcp1 00000100:00000200:0.0:1616513090.292134:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513090.292141:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.34@tcp1 routed destination 00000400:00000200:12.0:1616513090.292156:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.34@tcp1(10.31.142.34@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.292163:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.292169:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:11.2:1616513090.292169:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513090.292171:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.292175:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.292180:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513090.292181:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.292188:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.292191:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513090.292192:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513090.292198:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.292203:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42bb0 00000800:00000200:28.0:1616513090.292207:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513090.292218:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.292274:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.292281:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.292284:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:30.0:1616513090.292285:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513090.292291:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.292295:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ef68 00000800:00000200:22.0:1616513090.292299:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.325420:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513090.325440:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513090.325477:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.325487:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513090.325494:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513090.325497:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.200@tcp1 : PUT - for me 00000800:00000200:20.0:1616513090.325498:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.325506:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513090.325508:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.200@tcp1 of length 224 into portal 12 MB=0x5ffdcb21f32c0 00000400:00000200:2.0:1616513090.325515:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.200@tcp1 of length 224/224 into md 0xf964d519 [1] + 87224 00000400:00000200:20.0:1616513090.325516:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.200@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.325520:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.325523:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513090.325525:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.200@tcp1 of length 224 into portal 26 MB=0x5ffdcb21f3080 00000800:00000200:2.0:1616513090.325531:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513090.325532:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.200@tcp1 of length 224/224 into md 0xf97cd6b9 [1] + 0 00000800:00000200:2.0:1616513090.325534:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.325536:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513090.325537:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb430 00000400:00000200:20.0:1616513090.325539:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.325542:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513090.325547:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb430 00000800:00000200:20.0:1616513090.325550:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.325551:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.325552:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:10.0:1616513090.325553:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698229830336 00000800:00000200:20.0:1616513090.325554:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:10.0:1616513090.325565:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d2880 x1688698229830336/t0(0) o400->e740122f-8696-1119-cc38-6e6b5c44e06c@10.31.141.200@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513090.325579:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698229830336, offset 192 00000400:00000200:10.0:1616513090.325585:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.200@tcp1 00000400:00000200:10.0:1616513090.325593:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.200@tcp1 routed destination 00000400:00000200:10.0:1616513090.325607:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.200@tcp1(10.31.141.200@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513090.325611:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698229829760 00000800:00000200:10.0:1616513090.325613:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513090.325619:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.325623:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:0.0:1616513090.325623:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688698229829760/t0(0) o400->0d6636c9-2194-e258-f410-2bdde9edda25@10.31.141.200@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:10.0:1616513090.325625:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513090.325630:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.325636:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.325638:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698229829760, offset 192 00000400:00000200:0.0:1616513090.325644:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.200@tcp1 00000400:00000200:0.0:1616513090.325652:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.200@tcp1 routed destination 00000400:00000200:0.0:1616513090.325667:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.200@tcp1(10.31.141.200@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.325673:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513090.325680:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.325682:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.325684:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513090.325689:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513090.325689:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.325695:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513090.325697:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513090.325697:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513090.325699:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513090.325702:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513090.325706:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513090.325706:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.325710:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83a08 00000800:00000200:30.0:1616513090.325711:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513090.325713:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513090.325715:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:30.0:1616513090.325719:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513090.325723:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ba0 00000800:00000200:30.0:1616513090.325727:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513090.325729:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.326872:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513090.326925:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.326934:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.326945:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.113@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.326954:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.113@tcp1 of length 224 into portal 26 MB=0x5fff22ff51280 00000400:00000200:24.0:1616513090.326961:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.113@tcp1 of length 224/224 into md 0xf97cd6c9 [1] + 0 00000400:00000200:24.0:1616513090.326965:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bba90 00000400:00000200:24.0:1616513090.326968:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.326971:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513090.326976:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bba90 00000800:00000200:20.0:1616513090.326978:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.326979:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.326982:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.326983:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.327038:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688790535312000 00010000:00000200:0.0:1616513090.327049:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688790535312000/t0(0) o400->c66d0437-e51b-33ea-9a01-feef6918f5a9@10.31.136.113@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.327063:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688790535312000, offset 192 00000400:00000200:0.0:1616513090.327070:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.113@tcp1 00000400:00000200:0.0:1616513090.327077:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.113@tcp1 routed destination 00000400:00000200:0.0:1616513090.327092:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.113@tcp1(10.31.136.113@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.327099:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.327104:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.327107:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.327109:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.327114:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.327119:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.327121:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513090.327131:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.327138:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.327141:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513090.327147:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.327153:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42c38 00000800:00000200:28.0:1616513090.327157:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.327160:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513090.327334:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513090.327389:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.327398:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.327407:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.113@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.327416:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.113@tcp1 of length 224 into portal 12 MB=0x5fff22ff51300 00000400:00000200:18.0:1616513090.327422:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.113@tcp1 of length 224/224 into md 0xf964d519 [1] + 87448 00000400:00000200:18.0:1616513090.327429:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.327431:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513090.327447:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.327452:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.327455:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.327456:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513090.327515:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688790535312128 00010000:00000200:10.0:1616513090.327527:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120ad00 x1688790535312128/t0(0) o400->e3be43a8-e93a-f774-9d7c-af3128f945d3@10.31.136.113@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513090.327542:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688790535312128, offset 192 00000400:00000200:10.0:1616513090.327548:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.113@tcp1 00000400:00000200:10.0:1616513090.327555:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.113@tcp1 routed destination 00000400:00000200:10.0:1616513090.327570:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.113@tcp1(10.31.136.113@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513090.327577:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513090.327582:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.327585:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.327587:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.327592:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.327596:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513090.327603:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513090.327606:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:30.0:1616513090.327608:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513090.327613:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:30.0:1616513090.327616:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83e48 00000800:00000200:30.0:1616513090.327618:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:30.0:1616513090.327621:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513090.330615:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928224, portal 4 00000100:00000200:8.0:1616513090.330620:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928224, offset 0 00000400:00000200:8.0:1616513090.330626:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:8.0:1616513090.330631:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:8.0:1616513090.330643:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513090.330649:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:8.0:1616513090.330655:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513090.330658:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:8.0:1616513090.330660:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:8.0:1616513090.330665:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513090.330710:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.330732:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513090.330743:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513090.330746:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513090.330753:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513090.330760:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb23f980 x1693067725928224/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513090.330784:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:5.0:1616513090.330786:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513090.330790:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513090.330877:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513090.330887:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.330893:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513090.330895:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513090.330901:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513090.330909:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627720 00000400:00000200:1.0:1616513090.330916:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436fff65 [1] + 192 00000400:00000200:1.0:1616513090.330921:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513090.330925:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb23f980 x1693067725928224/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513090.330940:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.330942:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513090.330944:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:8.0:1616513090.330947:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb23f980 x1693067725928224/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:8.0:1616513090.330958:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb23f980 x1693067725928224/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513090.330968:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:7.2:1616513090.352638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513090.352694:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.352704:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.352713:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.220@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.352723:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.220@tcp1 of length 224 into portal 26 MB=0x6018508292000 00000400:00000200:2.0:1616513090.352730:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.220@tcp1 of length 224/224 into md 0xf97cd6d9 [1] + 0 00000400:00000200:2.0:1616513090.352735:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bacc0 00000400:00000200:2.0:1616513090.352737:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.352740:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513090.352746:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bacc0 00000800:00000200:18.0:1616513090.352749:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.352750:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.352753:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513090.352756:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.352759:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.352760:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.352768:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.220@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.352777:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.220@tcp1 of length 224 into portal 12 MB=0x6018508292080 00000400:00000200:18.0:1616513090.352784:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.220@tcp1 of length 224/224 into md 0xf964d519 [1] + 87672 00000400:00000200:18.0:1616513090.352790:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.352792:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513090.352799:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.352803:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.352804:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.352812:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690520739454976 00000100:00000200:10.0:1616513090.352823:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690520739455104 00010000:00000200:0.0:1616513090.352824:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1690520739454976/t0(0) o400->1839110d-7913-b35d-781d-66f118a3d807@10.31.140.220@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:10.0:1616513090.352835:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120f980 x1690520739455104/t0(0) o400->f0798153-6a1e-22cc-e1a5-26a736b0e54f@10.31.140.220@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513090.352840:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690520739454976, offset 192 00000400:00000200:0.0:1616513090.352847:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.220@tcp1 00000100:00000200:10.0:1616513090.352849:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690520739455104, offset 192 00000400:00000200:10.0:1616513090.352855:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.220@tcp1 00000400:00000200:0.0:1616513090.352855:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.220@tcp1 routed destination 00000400:00000200:10.0:1616513090.352862:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.220@tcp1 routed destination 00000400:00000200:0.0:1616513090.352870:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.220@tcp1(10.31.140.220@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513090.352876:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.220@tcp1(10.31.140.220@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.352877:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513090.352882:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513090.352882:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.352885:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.352887:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513090.352888:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.352890:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.352892:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513090.352892:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513090.352897:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513090.352899:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.352899:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513090.352901:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513090.352957:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513090.352958:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513090.352966:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513090.352967:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513090.352969:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.352970:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513090.352970:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513090.352976:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.352976:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.352980:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83540 00000400:00000200:20.0:1616513090.352981:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43320 00000800:00000200:22.0:1616513090.352984:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513090.352985:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.352988:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513090.373921:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513090.373977:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.373987:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.373996:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.175@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.374006:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.175@tcp1 of length 224 into portal 26 MB=0x603d865b88a40 00000400:00000200:2.0:1616513090.374012:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.175@tcp1 of length 224/224 into md 0xf97cd6e9 [1] + 0 00000400:00000200:2.0:1616513090.374017:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bbb18 00000400:00000200:2.0:1616513090.374020:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.374023:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513090.374027:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbb18 00000800:00000200:2.0:1616513090.374030:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.374031:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.374033:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.374035:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.374093:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693077814676032 00010000:00000200:0.0:1616513090.374104:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1693077814676032/t0(0) o400->8af92fa2-0eca-dc86-053c-ace91de5ed4e@10.31.131.175@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.374119:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693077814676032, offset 192 00000400:00000200:0.0:1616513090.374125:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.175@tcp1 00000400:00000200:0.0:1616513090.374143:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.175@tcp1 routed destination 00000400:00000200:0.0:1616513090.374156:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.175@tcp1(10.31.131.175@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.374162:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.374168:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.374170:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.374172:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.374177:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.374184:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513090.374190:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.374204:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.374211:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.374214:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513090.374220:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.374226:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42330 00000800:00000200:28.0:1616513090.374229:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.374233:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.374496:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513090.374550:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.374560:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513090.374569:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.175@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.374577:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.175@tcp1 of length 224 into portal 12 MB=0x603d865b88c80 00000400:00000200:20.0:1616513090.374584:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.175@tcp1 of length 224/224 into md 0x865be291 [1] + 19144 00000400:00000200:20.0:1616513090.374589:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.374592:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.374599:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.374602:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.374604:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.374608:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.374663:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693077814676608 00010000:00000200:12.0:1616513090.374676:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad5e80 x1693077814676608/t0(0) o400->6bb71619-361c-1a3a-b4a6-45e8f5e7a1c5@10.31.131.175@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.374690:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693077814676608, offset 192 00000400:00000200:12.0:1616513090.374696:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.175@tcp1 00000400:00000200:12.0:1616513090.374703:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.175@tcp1 routed destination 00000400:00000200:12.0:1616513090.374719:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.175@tcp1(10.31.131.175@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.374725:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.374732:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.374735:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.374737:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.374742:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.374746:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.374756:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.374760:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.374762:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.374768:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.374771:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e908 00000800:00000200:2.0:1616513090.374774:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.374777:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.417047:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513090.417105:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.417125:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513090.417133:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.155@tcp1 : PUT - for me 00000400:00000200:28.0:1616513090.417142:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.155@tcp1 of length 224 into portal 26 MB=0x5fffd300260c0 00000400:00000200:28.0:1616513090.417150:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.155@tcp1 of length 224/224 into md 0xf97cd6f9 [1] + 0 00000400:00000200:28.0:1616513090.417154:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba2a8 00000400:00000200:28.0:1616513090.417157:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.417160:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513090.417164:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba2a8 00000800:00000200:28.0:1616513090.417167:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.417170:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.417172:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.417174:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.417230:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688837780824256 00010000:00000200:0.0:1616513090.417243:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688837780824256/t0(0) o400->3557b71b-023b-4e39-0215-a0270f184346@10.31.131.155@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513090.417251:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513090.417257:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688837780824256, offset 192 00000400:00000200:0.0:1616513090.417263:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.155@tcp1 00000400:00000200:0.0:1616513090.417271:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.155@tcp1 routed destination 00000400:00000200:0.0:1616513090.417288:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.155@tcp1(10.31.131.155@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.417295:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.417301:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.417303:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.417304:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.417305:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513090.417311:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513090.417313:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513090.417318:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513090.417322:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.155@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.417330:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.155@tcp1 of length 224 into portal 12 MB=0x5fffd30026600 00000400:00000200:24.0:1616513090.417337:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.155@tcp1 of length 224/224 into md 0x865be291 [1] + 19368 00000400:00000200:24.0:1616513090.417342:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.417345:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513090.417352:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513090.417355:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513090.417357:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513090.417364:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513090.417366:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513090.417367:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.417368:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.417375:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.417381:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a434b8 00000800:00000200:20.0:1616513090.417384:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.417416:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688837780825600 00010000:00000200:12.0:1616513090.417429:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e0000 x1688837780825600/t0(0) o400->41c06f59-0fbb-f72a-d360-32880b6f02d0@10.31.131.155@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.417456:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688837780825600, offset 192 00000400:00000200:12.0:1616513090.417462:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.155@tcp1 00000400:00000200:12.0:1616513090.417469:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.155@tcp1 routed destination 00000400:00000200:12.0:1616513090.417484:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.155@tcp1(10.31.131.155@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.417491:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.417497:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.417500:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.417502:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.417507:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.417514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.417569:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.417577:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.417580:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513090.417586:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513090.417590:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f980 00000800:00000200:18.0:1616513090.417595:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.417598:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.428927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513090.428991:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.429000:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513090.429009:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.172@tcp1 : PUT - for me 00000400:00000200:28.0:1616513090.429020:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.172@tcp1 of length 224 into portal 26 MB=0x5ffd3d592b700 00000400:00000200:28.0:1616513090.429027:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.172@tcp1 of length 224/224 into md 0xf97cd709 [1] + 0 00000400:00000200:28.0:1616513090.429032:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bad48 00000400:00000200:28.0:1616513090.429035:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.429037:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513090.429042:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bad48 00000800:00000200:28.0:1616513090.429045:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.429045:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.429048:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.429050:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.429062:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660169897728 00010000:00000200:0.0:1616513090.429073:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688660169897728/t0(0) o400->b176838a-6406-0a7d-21f4-76cee78aed85@10.31.128.172@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.429086:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660169897728, offset 192 00000400:00000200:0.0:1616513090.429091:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.172@tcp1 00000400:00000200:0.0:1616513090.429099:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.172@tcp1 routed destination 00000400:00000200:0.0:1616513090.429113:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.172@tcp1(10.31.128.172@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.429119:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.429125:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.429127:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.429129:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.429134:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.429139:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.429140:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513090.429155:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.429161:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513090.429164:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513090.429171:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.429177:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43430 00000800:00000200:16.0:1616513090.429180:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.429183:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.429208:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513090.429219:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.429226:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513090.429228:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513090.429235:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.172@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.429244:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.172@tcp1 of length 224 into portal 12 MB=0x5ffd3d592ba80 00000400:00000200:24.0:1616513090.429251:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.172@tcp1 of length 224/224 into md 0x865be291 [1] + 19592 00000400:00000200:24.0:1616513090.429256:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.429258:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513090.429265:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.429268:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.429271:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.429284:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660169898624 00010000:00000200:12.0:1616513090.429297:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e7980 x1688660169898624/t0(0) o400->78840e73-e108-0534-a943-0d1394e564c1@10.31.128.172@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.429310:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660169898624, offset 192 00000400:00000200:12.0:1616513090.429317:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.172@tcp1 00000400:00000200:12.0:1616513090.429324:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.172@tcp1 routed destination 00000400:00000200:12.0:1616513090.429338:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.172@tcp1(10.31.128.172@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.429344:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.429350:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.429353:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.429355:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.429360:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.429366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.429420:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.429428:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.429431:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.429455:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.429459:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f6d8 00000800:00000200:2.0:1616513090.429463:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.429466:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513090.458596:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928240, portal 4 00000100:00000200:8.0:1616513090.458601:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928240, offset 0 00000400:00000200:8.0:1616513090.458608:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:8.0:1616513090.458614:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:8.0:1616513090.458627:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513090.458633:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:8.0:1616513090.458639:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513090.458642:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:8.0:1616513090.458645:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:8.0:1616513090.458650:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513090.458697:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.458720:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513090.458727:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513090.458730:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513090.458737:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513090.458746:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb23a400 x1693067725928240/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513090.458759:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3c28 00000800:00000200:1.0:1616513090.458764:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513090.458767:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513090.458882:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513090.458892:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.458898:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513090.458901:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513090.458907:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513090.458916:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627730 00000400:00000200:1.0:1616513090.458923:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436fff75 [1] + 192 00000400:00000200:1.0:1616513090.458929:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513090.458934:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb23a400 x1693067725928240/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513090.458949:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.458953:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:8.0:1616513090.458955:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb23a400 x1693067725928240/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513090.458955:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:8.0:1616513090.458978:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb23a400 x1693067725928240/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513090.458988:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:11.2:1616513090.479845:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.479901:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.479910:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.479919:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.188@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.479929:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.188@tcp1 of length 224 into portal 26 MB=0x5ffd55ef040c0 00000400:00000200:16.0:1616513090.479936:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.188@tcp1 of length 224/224 into md 0xf97cd739 [1] + 0 00000400:00000200:16.0:1616513090.479941:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba198 00000400:00000200:16.0:1616513090.479944:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.479949:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.479954:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba198 00000800:00000200:20.0:1616513090.479955:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.479957:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.479961:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.479962:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.480020:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666769473728 00010000:00000200:0.0:1616513090.480031:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688666769473728/t0(0) o400->4552b64b-bb30-0c07-9224-dd0c7be145a2@10.31.132.188@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.480045:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666769473728, offset 192 00000400:00000200:0.0:1616513090.480051:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.188@tcp1 00000400:00000200:0.0:1616513090.480058:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.188@tcp1 routed destination 00000400:00000200:0.0:1616513090.480072:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.188@tcp1(10.31.132.188@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.480078:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.480084:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.480088:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.480090:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.480095:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.480100:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.480102:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513090.480113:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.480128:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.480131:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513090.480137:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.480143:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ff0 00000800:00000200:28.0:1616513090.480146:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.480149:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513090.480171:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513090.480228:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.480238:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.480247:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.188@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.480257:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.188@tcp1 of length 224 into portal 12 MB=0x5ffd55ef041c0 00000400:00000200:18.0:1616513090.480264:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.188@tcp1 of length 224/224 into md 0xf964d519 [1] + 87896 00000400:00000200:18.0:1616513090.480269:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.480272:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513090.480279:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.480282:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.480283:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.480288:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513090.480352:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666769473984 00010000:00000200:10.0:1616513090.480364:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120a400 x1688666769473984/t0(0) o400->b470e150-a62d-e768-64dc-4e71b95d0434@10.31.132.188@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513090.480388:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666769473984, offset 192 00000400:00000200:10.0:1616513090.480393:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.188@tcp1 00000400:00000200:10.0:1616513090.480400:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.188@tcp1 routed destination 00000400:00000200:10.0:1616513090.480420:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.188@tcp1(10.31.132.188@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513090.480426:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513090.480432:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.480434:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.480445:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.480450:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.480455:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.480465:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.480471:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.480481:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.480488:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.480492:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82110 00000800:00000200:2.0:1616513090.480496:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.480499:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.481153:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513090.481162:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.481168:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513090.481171:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513090.481176:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.182@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.481185:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.182@tcp1 of length 224 into portal 26 MB=0x5ffd34b39d240 00000400:00000200:2.0:1616513090.481192:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.182@tcp1 of length 224/224 into md 0xf97cd749 [1] + 0 00000400:00000200:2.0:1616513090.481196:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bbcb0 00000400:00000200:2.0:1616513090.481199:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.481201:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513090.481205:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbcb0 00000800:00000200:2.0:1616513090.481208:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.481211:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.481212:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.481219:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657848816192 00010000:00000200:0.0:1616513090.481224:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688657848816192/t0(0) o400->7c68ed34-461a-07ff-5c09-052bc33be8d6@10.31.128.182@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.481236:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657848816192, offset 192 00000800:00000200:11.2:1616513090.481238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:0.0:1616513090.481241:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.182@tcp1 00000400:00000200:0.0:1616513090.481247:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.182@tcp1 routed destination 00000400:00000200:0.0:1616513090.481260:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.182@tcp1(10.31.128.182@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.481266:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.481270:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.481273:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.481275:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513090.481278:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513090.481282:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513090.481291:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.481300:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513090.481309:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.182@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.481318:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.182@tcp1 of length 224 into portal 12 MB=0x5ffd34b39d6c0 00000400:00000200:20.0:1616513090.481324:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.182@tcp1 of length 224/224 into md 0x865be291 [1] + 19816 00000400:00000200:20.0:1616513090.481330:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.481332:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.481339:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513090.481342:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513090.481344:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513090.481351:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513090.481352:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513090.481355:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513090.481356:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513090.481362:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.481367:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:16.0:1616513090.481371:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.481402:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657848817344 00010000:00000200:12.0:1616513090.481414:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad0900 x1688657848817344/t0(0) o400->9ef55f64-377b-2811-fae7-284fafe7646b@10.31.128.182@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.481429:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657848817344, offset 192 00000400:00000200:12.0:1616513090.481446:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.182@tcp1 00000400:00000200:12.0:1616513090.481454:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.182@tcp1 routed destination 00000400:00000200:12.0:1616513090.481469:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.182@tcp1(10.31.128.182@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.481475:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.481481:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.481484:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.481486:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.481491:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.481496:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.481506:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.481510:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.481512:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513090.481516:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513090.481517:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.481519:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e2a8 00000800:00000200:2.0:1616513090.481522:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513090.491531:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928256, portal 4 00000100:00000200:26.0:1616513090.491538:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928256, offset 0 00000400:00000200:26.0:1616513090.491545:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:26.0:1616513090.491551:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:26.0:1616513090.491564:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513090.491570:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:26.0:1616513090.491576:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513090.491579:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:26.0:1616513090.491581:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:26.0:1616513090.491586:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513090.491632:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513090.491688:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513090.491694:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:24.0:1616513090.491697:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513090.491704:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:24.0:1616513090.491710:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9640d80 x1693067725928256/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513090.491722:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeff0 00000800:00000200:24.0:1616513090.491726:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513090.491729:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513090.491811:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513090.491882:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513090.491891:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513090.491900:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513090.491908:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627740 00000400:00000200:20.0:1616513090.491914:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866da7a1 [1] + 192 00000400:00000200:20.0:1616513090.491919:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.491925:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9640d80 x1693067725928256/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513090.491936:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513090.491939:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513090.491942:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513090.491944:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:26.0:1616513090.491948:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9640d80 x1693067725928256/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513090.491961:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9640d80 x1693067725928256/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513090.491971:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af100 00000800:00000200:11.2:1616513090.497481:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513090.497539:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.497548:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.497557:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.229@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.497567:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.229@tcp1 of length 224 into portal 26 MB=0x603e0e57040c0 00000400:00000200:24.0:1616513090.497573:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.229@tcp1 of length 224/224 into md 0xf97cd789 [1] + 0 00000400:00000200:24.0:1616513090.497578:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb540 00000400:00000200:24.0:1616513090.497581:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.497584:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513090.497589:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb540 00000800:00000200:24.0:1616513090.497592:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.497593:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.497595:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.497600:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.497603:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.497604:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513090.497612:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.229@tcp1 : PUT - for me 00000100:00000200:0.0:1616513090.497615:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114317160640 00000400:00000200:28.0:1616513090.497622:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.229@tcp1 of length 224 into portal 12 MB=0x603e0e5704140 00010000:00000200:0.0:1616513090.497626:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1693114317160640/t0(0) o400->07c4568c-ec73-922e-98cb-f42196ed81ea@10.31.136.229@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513090.497630:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.229@tcp1 of length 224/224 into md 0x865be291 [1] + 20040 00000400:00000200:28.0:1616513090.497635:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.497638:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513090.497645:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.497645:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114317160640, offset 192 00000800:00000200:28.0:1616513090.497648:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.497650:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513090.497651:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.229@tcp1 00000400:00000200:0.0:1616513090.497659:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.229@tcp1 routed destination 00000400:00000200:0.0:1616513090.497674:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.229@tcp1(10.31.136.229@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.497681:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.497687:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.497690:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.497692:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.497697:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.497702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.497703:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513090.497715:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114317160768 00010000:00000200:12.0:1616513090.497727:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedde80 x1693114317160768/t0(0) o400->dc85f6e2-de61-386a-ab66-9ebe904a5a6c@10.31.136.229@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.497741:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114317160768, offset 192 00000400:00000200:12.0:1616513090.497748:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.229@tcp1 00000400:00000200:12.0:1616513090.497755:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.229@tcp1 routed destination 00000800:00000200:20.0:1616513090.497756:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.497763:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.497766:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513090.497767:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513090.497770:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.229@tcp1(10.31.136.229@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513090.497772:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:12.0:1616513090.497777:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513090.497778:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42198 00000800:00000200:20.0:1616513090.497782:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513090.497783:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.497787:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.497789:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.497794:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.497800:0:50592:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.497856:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.497862:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.497865:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513090.497872:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513090.497876:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f870 00000800:00000200:18.0:1616513090.497880:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.497883:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513090.506561:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928272, portal 4 00000100:00000200:30.0:1616513090.506566:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928272, offset 0 00000400:00000200:30.0:1616513090.506573:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:30.0:1616513090.506579:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:30.0:1616513090.506592:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513090.506598:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513090.506605:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513090.506608:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513090.506610:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513090.506615:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513090.506662:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513090.506730:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513090.506737:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513090.506740:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.506746:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513090.506752:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d747c697080 x1693067725928272/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513090.506766:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efed0 00000800:00000200:22.0:1616513090.506770:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513090.506773:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513090.506829:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513090.506883:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513090.506892:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:2.0:1616513090.506895:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513090.506901:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513090.506909:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627750 00000400:00000200:18.0:1616513090.506915:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97ce009 [1] + 192 00000400:00000200:18.0:1616513090.506924:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.506929:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d747c697080 x1693067725928272/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513090.506944:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513090.506947:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:30.0:1616513090.506949:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d747c697080 x1693067725928272/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513090.506949:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:30.0:1616513090.506961:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d747c697080 x1693067725928272/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513090.506970:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eebb0 00000800:00000200:7.2:1616513090.520542:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513090.520597:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.520607:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513090.520616:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.85@tcp1 : PUT - for me 00000400:00000200:22.0:1616513090.520625:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.85@tcp1 of length 224 into portal 26 MB=0x5ffdbec7a5cc0 00000400:00000200:22.0:1616513090.520631:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.85@tcp1 of length 224/224 into md 0xf97cd799 [1] + 0 00000400:00000200:22.0:1616513090.520636:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf34b8 00000400:00000200:22.0:1616513090.520638:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513090.520641:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513090.520646:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf34b8 00000800:00000200:22.0:1616513090.520649:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.520651:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513090.520652:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513090.520659:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513090.520662:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.520663:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.520671:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.85@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.520680:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.85@tcp1 of length 224 into portal 12 MB=0x5ffdbec7a5d40 00000400:00000200:2.0:1616513090.520686:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.85@tcp1 of length 224/224 into md 0xf964d519 [1] + 88120 00000400:00000200:2.0:1616513090.520692:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.520695:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513090.520702:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.520705:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.520709:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.520712:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694913916096 00010000:00000200:0.0:1616513090.520723:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688694913916096/t0(0) o400->a55e50f3-30b2-e9f4-a54d-adad064fc19e@10.31.141.85@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.520737:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694913916096, offset 192 00000400:00000200:0.0:1616513090.520744:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.85@tcp1 00000400:00000200:0.0:1616513090.520751:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.85@tcp1 routed destination 00000400:00000200:0.0:1616513090.520765:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.85@tcp1(10.31.141.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513090.520766:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694913916224 00000800:00000200:0.0:1616513090.520772:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.520778:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:10.0:1616513090.520779:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6780 x1688694913916224/t0(0) o400->aa9c6e11-bc8b-f32c-8c08-b20b67c64ed0@10.31.141.85@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513090.520780:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.520783:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.520787:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:10.0:1616513090.520793:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694913916224, offset 192 00000800:00000200:11.2:1616513090.520794:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.520794:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:10.0:1616513090.520799:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.85@tcp1 00000400:00000200:10.0:1616513090.520807:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.85@tcp1 routed destination 00000400:00000200:10.0:1616513090.520821:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.85@tcp1(10.31.141.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513090.520828:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513090.520834:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.520837:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.520839:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.520844:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.520848:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513090.520850:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.520857:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513090.520860:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513090.520866:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.520872:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42cc0 00000800:00000200:16.0:1616513090.520875:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.520878:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513090.520913:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.520920:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.520923:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513090.520929:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513090.520933:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513090.520933:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83100 00000800:00000200:18.0:1616513090.520937:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.546244:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513090.546301:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.546310:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.546319:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.132@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.546329:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.132@tcp1 of length 224 into portal 26 MB=0x601b999d953c0 00000400:00000200:24.0:1616513090.546335:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.132@tcp1 of length 224/224 into md 0xf97cd7b9 [1] + 0 00000400:00000200:24.0:1616513090.546340:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2d48 00000400:00000200:24.0:1616513090.546343:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.546346:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513090.546350:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2d48 00000800:00000200:20.0:1616513090.546353:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.546355:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.546358:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513090.546360:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.546363:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.546363:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513090.546371:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.132@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.546379:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.132@tcp1 of length 224 into portal 12 MB=0x601b999d95440 00000400:00000200:20.0:1616513090.546385:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.132@tcp1 of length 224/224 into md 0x865be291 [1] + 20264 00000400:00000200:20.0:1616513090.546390:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.546393:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.546400:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.546403:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.546405:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.546417:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690746521998272 00010000:00000200:0.0:1616513090.546428:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1690746521998272/t0(0) o400->e0876726-a7c0-9f88-3fb2-8f056d693830@10.31.143.132@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.546457:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690746521998272, offset 192 00000400:00000200:0.0:1616513090.546464:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.132@tcp1 00000400:00000200:0.0:1616513090.546471:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.132@tcp1 routed destination 00000100:00000200:12.0:1616513090.546472:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690746521998400 00010000:00000200:12.0:1616513090.546484:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad2400 x1690746521998400/t0(0) o400->7443ffc5-6100-91ca-43c6-08ce5fc033e0@10.31.143.132@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513090.546486:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.132@tcp1(10.31.143.132@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.546493:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:12.0:1616513090.546497:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690746521998400, offset 192 00000800:00000200:0.0:1616513090.546499:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.546501:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:12.0:1616513090.546503:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.132@tcp1 00000800:00000200:0.0:1616513090.546503:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.546508:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513090.546510:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.132@tcp1 routed destination 00000800:00000200:11.2:1616513090.546513:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.546515:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513090.546524:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.132@tcp1(10.31.143.132@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.546530:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.546536:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.546538:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.546540:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.546545:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.546553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513090.546564:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513090.546568:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:6.0:1616513090.546570:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513090.546572:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513090.546575:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:6.0:1616513090.546577:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.546578:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513090.546579:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:6.0:1616513090.546580:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fdc0 00000800:00000200:6.0:1616513090.546583:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513090.546585:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:6.0:1616513090.546585:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513090.546590:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42cc0 00000800:00000200:28.0:1616513090.546594:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.548513:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513090.548567:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.548577:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513090.548586:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.135@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.548595:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.135@tcp1 of length 224 into portal 26 MB=0x601b7caab45c0 00000400:00000200:20.0:1616513090.548603:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.135@tcp1 of length 224/224 into md 0xf97cd7c9 [1] + 0 00000400:00000200:20.0:1616513090.548608:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2880 00000400:00000200:20.0:1616513090.548611:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.548614:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513090.548618:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2880 00000800:00000200:16.0:1616513090.548620:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.548621:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.548624:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513090.548626:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.548657:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513090.548681:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690738751129024 00010000:00000200:0.0:1616513090.548692:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1690738751129024/t0(0) o400->76ee9c56-d706-39d5-2c30-f720ce4da962@10.31.143.135@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.548707:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690738751129024, offset 192 00000800:00000200:24.0:1616513090.548710:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513090.548715:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.135@tcp1 00000800:00000200:24.0:1616513090.548720:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513090.548722:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.135@tcp1 routed destination 00000400:00000200:0.0:1616513090.548737:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.135@tcp1(10.31.143.135@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.548744:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.548747:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.135@tcp1 : PUT - for me 00000800:00000200:0.0:1616513090.548750:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.548753:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513090.548755:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:24.0:1616513090.548756:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.135@tcp1 of length 224 into portal 12 MB=0x601b7caab46c0 00000800:00000200:0.0:1616513090.548760:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:24.0:1616513090.548762:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.135@tcp1 of length 224/224 into md 0x865be291 [1] + 20488 00000800:00000200:28.0:1616513090.548764:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:0.0:1616513090.548766:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513090.548767:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.548770:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513090.548772:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513090.548775:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513090.548776:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513090.548776:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.548780:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.548782:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513090.548782:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513090.548787:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:28.0:1616513090.548791:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.548839:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690738751129280 00010000:00000200:12.0:1616513090.548852:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e4c80 x1690738751129280/t0(0) o400->9ebdd335-4729-20a2-ee3f-8f1347f00d47@10.31.143.135@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.548866:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690738751129280, offset 192 00000400:00000200:12.0:1616513090.548872:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.135@tcp1 00000400:00000200:12.0:1616513090.548880:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.135@tcp1 routed destination 00000400:00000200:12.0:1616513090.548895:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.135@tcp1(10.31.143.135@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.548901:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.548907:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.548910:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.548912:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.548916:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.548921:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.548975:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.548983:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.548986:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.548993:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.548997:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eaa0 00000800:00000200:22.0:1616513090.549001:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.549004:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.564596:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513090.564610:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513090.564618:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:6.0:1616513090.564627:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.242@tcp1 : PUT - for me 00000400:00000200:6.0:1616513090.564636:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.242@tcp1 of length 224 into portal 12 MB=0x604b819a33d00 00000400:00000200:6.0:1616513090.564643:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.242@tcp1 of length 224/224 into md 0xf964d519 [1] + 88344 00000400:00000200:6.0:1616513090.564649:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513090.564651:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:6.0:1616513090.564659:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513090.564662:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.564662:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513090.564667:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513090.564724:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694038610885888 00010000:00000200:10.0:1616513090.564736:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74d9647500 x1694038610885888/t0(0) o400->69c4923f-ce3a-6e98-c7d2-0d104aacb495@10.31.140.242@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513090.564750:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694038610885888, offset 192 00000400:00000200:10.0:1616513090.564757:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.242@tcp1 00000400:00000200:10.0:1616513090.564764:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.242@tcp1 routed destination 00000400:00000200:10.0:1616513090.564779:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.242@tcp1(10.31.140.242@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513090.564799:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513090.564805:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.564807:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513090.564809:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513090.564814:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.564820:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.564888:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.564895:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.564898:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.564904:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.564908:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82b28 00000800:00000200:20.0:1616513090.564912:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.564915:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.565217:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513090.565271:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.565281:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.565290:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.242@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.565298:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.242@tcp1 of length 224 into portal 26 MB=0x604b819a33c80 00000400:00000200:24.0:1616513090.565305:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.242@tcp1 of length 224/224 into md 0xf97cd7d9 [1] + 0 00000400:00000200:24.0:1616513090.565309:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f36d8 00000400:00000200:24.0:1616513090.565312:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.565315:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513090.565320:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f36d8 00000800:00000200:24.0:1616513090.565323:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.565325:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.565326:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.565329:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.565386:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694038610885760 00010000:00000200:0.0:1616513090.565397:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1694038610885760/t0(0) o400->585a3540-dcd8-7748-db03-7b476c5b1ad7@10.31.140.242@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.565411:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694038610885760, offset 192 00000400:00000200:0.0:1616513090.565418:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.242@tcp1 00000400:00000200:0.0:1616513090.565425:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.242@tcp1 routed destination 00000400:00000200:0.0:1616513090.565454:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.242@tcp1(10.31.140.242@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.565460:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513090.565467:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.565470:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.565472:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513090.565477:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.565481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513090.565484:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513090.565536:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.565544:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.565547:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.565553:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.565559:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43430 00000800:00000200:22.0:1616513090.565562:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.565565:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.569180:0:50600:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513090.569235:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.569244:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:6.0:1616513090.569246:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513090.569253:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.62@tcp1 : PUT - for me 00000800:00000200:6.0:1616513090.569254:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:6.0:1616513090.569262:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.62@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.569263:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.62@tcp1 of length 224 into portal 26 MB=0x5ffe1f89a2d00 00000400:00000200:6.0:1616513090.569270:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.62@tcp1 of length 224 into portal 12 MB=0x5ffe1f89a3000 00000400:00000200:2.0:1616513090.569270:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.62@tcp1 of length 224/224 into md 0xf97cd7f9 [1] + 0 00000400:00000200:2.0:1616513090.569274:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2088 00000400:00000200:2.0:1616513090.569277:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:6.0:1616513090.569279:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.62@tcp1 of length 224/224 into md 0xf964d519 [1] + 88568 00000400:00000200:6.0:1616513090.569284:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.569284:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513090.569288:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2088 00000100:00000200:6.0:1616513090.569290:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513090.569291:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513090.569294:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513090.569295:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513090.569296:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:6.0:1616513090.569299:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.569299:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:6.0:1616513090.569300:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.569351:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688720887131392 00010000:00000200:0.0:1616513090.569362:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688720887131392/t0(0) o400->a421036b-c6d1-750e-e7a9-644623e52b0e@10.31.140.62@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513090.569373:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688720887132160 00000100:00000200:0.0:1616513090.569377:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688720887131392, offset 192 00000400:00000200:0.0:1616513090.569383:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.62@tcp1 00010000:00000200:10.0:1616513090.569385:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74d9640000 x1688720887132160/t0(0) o400->6ae388ef-e636-1bf2-37df-f667db6c56b9@10.31.140.62@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513090.569390:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.62@tcp1 routed destination 00000100:00000200:10.0:1616513090.569399:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688720887132160, offset 192 00000400:00000200:0.0:1616513090.569405:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.62@tcp1(10.31.140.62@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513090.569406:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.62@tcp1 00000400:00000200:10.0:1616513090.569413:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.62@tcp1 routed destination 00000800:00000200:0.0:1616513090.569416:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.569422:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.569425:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.569427:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513090.569428:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.62@tcp1(10.31.140.62@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.569431:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.569438:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513090.569447:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:0.0:1616513090.569451:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513090.569453:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513090.569456:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.569458:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.569463:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.569466:0:50600:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513090.569494:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.569500:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.569503:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.569509:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.569515:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42330 00000800:00000200:20.0:1616513090.569518:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.569521:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513090.569530:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.569538:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.569540:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513090.569543:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513090.569547:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513090.569551:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e833a8 00000800:00000200:18.0:1616513090.569555:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:29.0:1616513090.570885:0:6492:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928288, portal 4 00000100:00000200:29.0:1616513090.570891:0:6492:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928288, offset 0 00000400:00000200:29.0:1616513090.570899:0:6492:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:29.0:1616513090.570905:0:6492:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:29.0:1616513090.570918:0:6492:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513090.570928:0:6492:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:29.0:1616513090.570935:0:6492:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513090.570938:0:6492:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:29.0:1616513090.570940:0:6492:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:29.0:1616513090.570946:0:6492:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513090.570954:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.570969:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513090.570975:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513090.570977:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513090.570985:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513090.570992:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8302322d00 x1693067725928288/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513090.571004:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2110 00000800:00000200:1.0:1616513090.571009:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513090.571011:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513090.571158:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513090.571166:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.571172:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513090.571175:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513090.571180:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513090.571188:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627760 00000400:00000200:1.0:1616513090.571195:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436fff85 [1] + 192 00000400:00000200:1.0:1616513090.571200:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513090.571204:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8302322d00 x1693067725928288/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513090.571219:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513090.571222:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513090.571223:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:29.0:1616513090.571226:0:6492:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8302322d00 x1693067725928288/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:29.0:1616513090.571239:0:6492:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8302322d00 x1693067725928288/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:29.0:1616513090.571248:0:6492:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b23b8 00000800:00000200:11.2:1616513090.575953:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513090.576008:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.576017:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513090.576026:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.164@tcp1 : PUT - for me 00000400:00000200:28.0:1616513090.576036:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.164@tcp1 of length 224 into portal 26 MB=0x601aa4ab859c0 00000400:00000200:28.0:1616513090.576042:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.164@tcp1 of length 224/224 into md 0xf97cd809 [1] + 0 00000400:00000200:28.0:1616513090.576047:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2440 00000400:00000200:28.0:1616513090.576050:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.576052:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513090.576057:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2440 00000800:00000200:28.0:1616513090.576060:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.576061:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.576062:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.576067:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.576070:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.576070:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.576078:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.164@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.576087:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.164@tcp1 of length 224 into portal 12 MB=0x601aa4ab85a80 00000400:00000200:24.0:1616513090.576093:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.164@tcp1 of length 224/224 into md 0x865be291 [1] + 20712 00000400:00000200:24.0:1616513090.576099:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.576101:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513090.576107:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.576110:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.576112:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.576118:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690680769927616 00010000:00000200:0.0:1616513090.576129:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1690680769927616/t0(0) o400->fdd92199-8400-0991-0b8b-de5cdd03077b@10.31.140.164@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.576142:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690680769927616, offset 192 00000400:00000200:0.0:1616513090.576148:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.164@tcp1 00000400:00000200:0.0:1616513090.576155:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.164@tcp1 routed destination 00000400:00000200:0.0:1616513090.576169:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.164@tcp1(10.31.140.164@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:12.0:1616513090.576173:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690680769927808 00000800:00000200:0.0:1616513090.576175:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.576181:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.576183:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:12.0:1616513090.576185:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e1200 x1690680769927808/t0(0) o400->a82c5e68-51a9-5f6f-78b2-bec0eeac2af5@10.31.140.164@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513090.576185:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.576190:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.576196:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513090.576198:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513090.576210:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690680769927808, offset 192 00000400:00000200:12.0:1616513090.576216:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.164@tcp1 00000400:00000200:12.0:1616513090.576224:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.164@tcp1 routed destination 00000400:00000200:12.0:1616513090.576238:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.164@tcp1(10.31.140.164@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.576245:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.576250:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.576253:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.576255:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513090.576257:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513090.576260:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513090.576264:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513090.576267:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513090.576267:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513090.576268:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513090.576274:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.576279:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42c38 00000800:00000200:16.0:1616513090.576283:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513090.576320:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.576328:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.576331:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.576338:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.576342:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e330 00000800:00000200:2.0:1616513090.576345:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.576348:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.645876:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513090.645934:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.645944:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.645953:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.100@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.645964:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.100@tcp1 of length 224 into portal 26 MB=0x6019534a20100 00000400:00000200:24.0:1616513090.645971:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.100@tcp1 of length 224/224 into md 0xf97cd839 [1] + 0 00000400:00000200:24.0:1616513090.645976:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3cb0 00000400:00000200:24.0:1616513090.645979:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.645982:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513090.645988:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3cb0 00000800:00000200:20.0:1616513090.645988:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.645991:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.645994:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.645996:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.646021:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513090.646056:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590205051136 00010000:00000200:0.0:1616513090.646069:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1690590205051136/t0(0) o400->6869dd1d-7e02-003f-28f1-33908da68a95@10.31.140.100@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513090.646076:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.646085:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690590205051136, offset 192 00000800:00000200:28.0:1616513090.646086:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513090.646091:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.100@tcp1 00000400:00000200:28.0:1616513090.646095:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.100@tcp1 : PUT - for me 00000400:00000200:0.0:1616513090.646100:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.100@tcp1 routed destination 00000400:00000200:0.0:1616513090.646115:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.100@tcp1(10.31.140.100@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.646123:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513090.646125:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.100@tcp1 of length 224 into portal 12 MB=0x6019534a20480 00000800:00000200:0.0:1616513090.646129:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513090.646132:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.100@tcp1 of length 224/224 into md 0x865be291 [1] + 20936 00000800:00000200:0.0:1616513090.646132:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.646136:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513090.646136:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:28.0:1616513090.646137:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.646140:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513090.646142:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:11.2:1616513090.646146:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.646147:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513090.646149:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513090.646151:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513090.646153:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.646156:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.646160:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513090.646161:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.646162:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513090.646168:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.646172:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42bb0 00000800:00000200:28.0:1616513090.646175:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.646212:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590205052032 00010000:00000200:12.0:1616513090.646226:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedad00 x1690590205052032/t0(0) o400->1660668d-28d6-de04-393d-db896f040df0@10.31.140.100@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.646240:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690590205052032, offset 192 00000400:00000200:12.0:1616513090.646247:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.100@tcp1 00000400:00000200:12.0:1616513090.646254:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.100@tcp1 routed destination 00000400:00000200:12.0:1616513090.646270:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.100@tcp1(10.31.140.100@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.646276:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.646283:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.646286:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.646288:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.646293:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.646300:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.646357:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.646373:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.646376:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.646383:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.646387:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ecc0 00000800:00000200:22.0:1616513090.646391:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.646394:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.657558:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.657614:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.657623:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.657633:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.174@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.657642:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.174@tcp1 of length 224 into portal 26 MB=0x5ffde538a67c0 00000400:00000200:16.0:1616513090.657649:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.174@tcp1 of length 224/224 into md 0xf97cd869 [1] + 0 00000400:00000200:16.0:1616513090.657654:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2ff0 00000400:00000200:16.0:1616513090.657658:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.657661:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.657666:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2ff0 00000800:00000200:20.0:1616513090.657668:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.657670:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.657672:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513090.657675:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513090.657678:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.657679:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513090.657688:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.174@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.657697:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.174@tcp1 of length 224 into portal 12 MB=0x5ffde538a6880 00000400:00000200:20.0:1616513090.657703:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.174@tcp1 of length 224/224 into md 0x865be291 [1] + 21160 00000400:00000200:20.0:1616513090.657708:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.657711:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.657718:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.657721:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.657725:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.657731:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705232955328 00010000:00000200:0.0:1616513090.657743:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688705232955328/t0(0) o400->c9fe4ec2-0809-fd2a-0552-4dcf26515895@10.31.141.174@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.657758:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705232955328, offset 192 00000400:00000200:0.0:1616513090.657764:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.174@tcp1 00000400:00000200:0.0:1616513090.657771:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.174@tcp1 routed destination 00000100:00000200:12.0:1616513090.657781:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705232955520 00000400:00000200:0.0:1616513090.657787:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.174@tcp1(10.31.141.174@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:12.0:1616513090.657792:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56cbad6780 x1688705232955520/t0(0) o400->0592d5f9-cbca-3713-97b3-9c37c2d04c5a@10.31.141.174@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513090.657793:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.657799:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.657802:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.657804:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513090.657806:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705232955520, offset 192 00000800:00000200:0.0:1616513090.657809:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513090.657812:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.174@tcp1 00000800:00000200:11.2:1616513090.657814:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.657816:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513090.657819:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.174@tcp1 routed destination 00000400:00000200:12.0:1616513090.657833:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.174@tcp1(10.31.141.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.657841:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.657846:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.657849:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.657851:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.657856:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.657864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513090.657868:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.657876:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.657878:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513090.657880:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513090.657885:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.657891:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42908 00000800:00000200:28.0:1616513090.657894:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513090.657919:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.657926:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.657929:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.657935:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.657939:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f210 00000800:00000200:2.0:1616513090.657943:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.657945:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.657951:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513090.657953:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.657955:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.657956:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.657959:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.664165:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513090.664219:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.664228:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513090.664237:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.190@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.664246:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.190@tcp1 of length 224 into portal 26 MB=0x60542aee79500 00000400:00000200:20.0:1616513090.664252:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.190@tcp1 of length 224/224 into md 0xf97cd899 [1] + 0 00000400:00000200:20.0:1616513090.664257:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3d38 00000400:00000200:20.0:1616513090.664260:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.664263:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513090.664267:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3d38 00000800:00000200:20.0:1616513090.664270:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.664272:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.664273:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513090.664279:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513090.664281:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.664282:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.664291:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.190@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.664299:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.190@tcp1 of length 224 into portal 12 MB=0x60542aee79580 00000400:00000200:24.0:1616513090.664306:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.190@tcp1 of length 224/224 into md 0x865be291 [1] + 21384 00000400:00000200:24.0:1616513090.664311:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.664314:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513090.664321:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.664324:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.664325:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.664342:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694633820656896 00010000:00000200:0.0:1616513090.664353:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1694633820656896/t0(0) o400->80df1aef-d172-6539-3dee-d96dd0222091@10.31.140.190@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.664368:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694633820656896, offset 192 00000400:00000200:0.0:1616513090.664374:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.190@tcp1 00000400:00000200:0.0:1616513090.664382:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.190@tcp1 routed destination 00000100:00000200:12.0:1616513090.664385:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694633820657024 00010000:00000200:12.0:1616513090.664397:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e3a80 x1694633820657024/t0(0) o400->bbfe1661-2600-d94c-3cd8-bf37299ecacf@10.31.140.190@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513090.664397:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.190@tcp1(10.31.140.190@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.664404:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.664409:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:12.0:1616513090.664411:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694633820657024, offset 192 00000800:00000200:0.0:1616513090.664412:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.664414:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513090.664417:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.190@tcp1 00000800:00000200:0.0:1616513090.664419:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.664424:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513090.664425:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.190@tcp1 routed destination 00000100:00000200:0.0:1616513090.664426:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513090.664463:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.190@tcp1(10.31.140.190@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.664469:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.664475:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.664477:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.664479:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.664483:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513090.664488:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513090.664490:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513090.664495:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.664498:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513090.664499:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513090.664504:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.664510:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42660 00000800:00000200:28.0:1616513090.664513:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513090.664545:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.664552:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.664557:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513090.664564:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513090.664568:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e4c8 00000800:00000200:18.0:1616513090.664572:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.664575:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.668862:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513090.668916:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.668926:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.668935:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.72@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.668944:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.72@tcp1 of length 224 into portal 26 MB=0x5ffd460afb040 00000400:00000200:24.0:1616513090.668951:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.72@tcp1 of length 224/224 into md 0xf97cd8a9 [1] + 0 00000400:00000200:24.0:1616513090.668957:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3dc0 00000400:00000200:24.0:1616513090.668960:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.668963:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513090.668967:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3dc0 00000800:00000200:24.0:1616513090.668970:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.668970:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.668973:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.668977:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.668980:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.668981:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.668989:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.72@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.668999:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.72@tcp1 of length 224 into portal 12 MB=0x5ffd460afb0c0 00000400:00000200:16.0:1616513090.669005:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.72@tcp1 of length 224/224 into md 0x865be291 [1] + 21608 00000400:00000200:16.0:1616513090.669010:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.669013:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513090.669019:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.669023:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.669024:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.669032:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662503829568 00010000:00000200:0.0:1616513090.669043:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688662503829568/t0(0) o400->28e237ec-5a6a-f1a0-44bf-3acd741b61e3@10.31.138.72@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.669059:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662503829568, offset 192 00000400:00000200:0.0:1616513090.669065:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.72@tcp1 00000400:00000200:0.0:1616513090.669073:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.72@tcp1 routed destination 00000100:00000200:12.0:1616513090.669085:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662503829696 00000400:00000200:0.0:1616513090.669088:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.72@tcp1(10.31.138.72@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.669094:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513090.669098:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd7500 x1688662503829696/t0(0) o400->84efd5ed-7de5-f30e-15e4-0d6362ef6d97@10.31.138.72@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513090.669100:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.669102:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.669105:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.669109:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513090.669112:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662503829696, offset 192 00000800:00000200:11.2:1616513090.669114:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.669116:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513090.669118:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.72@tcp1 00000400:00000200:12.0:1616513090.669126:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.72@tcp1 routed destination 00000400:00000200:12.0:1616513090.669141:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.72@tcp1(10.31.138.72@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.669147:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.669153:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.669156:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.669158:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.669162:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513090.669167:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513090.669169:0:50611:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513090.669174:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.669177:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513090.669178:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513090.669184:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.669189:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42990 00000800:00000200:28.0:1616513090.669193:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513090.669224:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.669230:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513090.669233:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513090.669240:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513090.669244:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fb18 00000800:00000200:10.0:1616513090.669248:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513090.669251:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.671671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513090.671697:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.671706:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.671715:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.30@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.671725:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.30@tcp1 of length 224 into portal 26 MB=0x5ffde5e2f49c0 00000400:00000200:18.0:1616513090.671731:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.30@tcp1 of length 224/224 into md 0xf97cd8b9 [1] + 0 00000400:00000200:18.0:1616513090.671736:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f34b8 00000400:00000200:18.0:1616513090.671739:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.671741:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513090.671745:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f34b8 00000800:00000200:18.0:1616513090.671749:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.671750:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.671751:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.671757:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.671760:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.671761:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.671769:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.30@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.671778:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.30@tcp1 of length 224 into portal 12 MB=0x5ffde5e2f4a40 00000400:00000200:2.0:1616513090.671785:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.30@tcp1 of length 224/224 into md 0xf964d519 [1] + 88792 00000400:00000200:2.0:1616513090.671790:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.671793:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513090.671801:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.671804:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.671806:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.671809:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705411533248 00010000:00000200:0.0:1616513090.671820:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688705411533248/t0(0) o400->041ab194-ce0d-5fc6-1ab9-7ee7a0e31371@10.31.141.30@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.671835:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705411533248, offset 192 00000400:00000200:0.0:1616513090.671841:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.30@tcp1 00000400:00000200:0.0:1616513090.671848:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.30@tcp1 routed destination 00000400:00000200:0.0:1616513090.671863:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.30@tcp1(10.31.141.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:14.0:1616513090.671867:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705411533376 00000800:00000200:0.0:1616513090.671870:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.671875:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.671878:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:14.0:1616513090.671879:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0d80 x1688705411533376/t0(0) o400->868a8399-b0a9-70b9-352c-5124457053a2@10.31.141.30@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513090.671880:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.671885:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.671890:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.671892:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:14.0:1616513090.671893:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705411533376, offset 192 00000400:00000200:14.0:1616513090.671899:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.30@tcp1 00000400:00000200:14.0:1616513090.671907:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.30@tcp1 routed destination 00000400:00000200:14.0:1616513090.671921:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.30@tcp1(10.31.141.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513090.671928:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513090.671934:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.671936:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513090.671939:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513090.671942:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513090.671943:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513090.671948:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513090.671948:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513090.671951:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513090.671957:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.671962:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42000 00000800:00000200:16.0:1616513090.671965:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.671968:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513090.672019:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.672026:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.672029:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513090.672030:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513090.672036:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.672040:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757cb0 00000800:00000200:22.0:1616513090.672044:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.686843:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513090.686901:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.686911:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.686920:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.225@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.686929:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.225@tcp1 of length 224 into portal 26 MB=0x5ffd3d517d780 00000400:00000200:2.0:1616513090.686936:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.225@tcp1 of length 224/224 into md 0xf97cd8e9 [1] + 0 00000400:00000200:2.0:1616513090.686941:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3ed0 00000400:00000200:2.0:1616513090.686944:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.686947:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513090.686951:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3ed0 00000800:00000200:2.0:1616513090.686954:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.686956:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.686957:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.686960:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.687017:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660161845120 00010000:00000200:0.0:1616513090.687028:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688660161845120/t0(0) o400->0ac88585-02db-5433-cae3-f485f0845852@10.31.128.225@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.687043:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660161845120, offset 192 00000400:00000200:0.0:1616513090.687049:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.225@tcp1 00000400:00000200:0.0:1616513090.687057:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.225@tcp1 routed destination 00000400:00000200:0.0:1616513090.687073:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.225@tcp1(10.31.128.225@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513090.687078:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.687080:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.687086:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.687088:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.687091:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.687096:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.687102:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513090.687103:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513090.687131:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.687141:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.687150:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.225@tcp1 : PUT - for me 00000800:00000200:24.0:1616513090.687158:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513090.687158:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.225@tcp1 of length 224 into portal 12 MB=0x5ffd3d517da40 00000800:00000200:24.0:1616513090.687165:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513090.687165:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.225@tcp1 of length 224/224 into md 0xf964d519 [1] + 89016 00000400:00000200:24.0:1616513090.687167:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513090.687170:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.687172:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513090.687173:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513090.687178:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a427f8 00000800:00000200:18.0:1616513090.687179:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513090.687182:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513090.687182:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513090.687184:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513090.687184:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.687187:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513090.687248:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660161845824 00010000:00000200:14.0:1616513090.687260:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120c800 x1688660161845824/t0(0) o400->0294cb44-78e6-b36c-0721-3cf93dc5af83@10.31.128.225@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513090.687275:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660161845824, offset 192 00000400:00000200:14.0:1616513090.687281:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.225@tcp1 00000400:00000200:14.0:1616513090.687288:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.225@tcp1 routed destination 00000400:00000200:14.0:1616513090.687303:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.225@tcp1(10.31.128.225@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513090.687310:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513090.687316:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.687319:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513090.687321:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513090.687328:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.687333:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.687344:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.687349:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513090.687352:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513090.687353:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513090.687358:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513090.687362:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756330 00000800:00000200:10.0:1616513090.687367:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.688386:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.688452:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.688461:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.688469:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.32@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.688478:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.32@tcp1 of length 224 into portal 26 MB=0x5ffd46839ea80 00000400:00000200:16.0:1616513090.688485:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.32@tcp1 of length 224/224 into md 0xf97cd909 [1] + 0 00000400:00000200:16.0:1616513090.688489:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3980 00000400:00000200:16.0:1616513090.688491:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.688494:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.688500:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3980 00000800:00000200:16.0:1616513090.688503:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.688505:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513090.688507:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.688509:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.688514:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.688520:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.32@tcp1 : PUT - for me 00000800:00000200:20.0:1616513090.688523:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513090.688525:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.32@tcp1 of length 224 into portal 12 MB=0x5ffd46839eb00 00000400:00000200:16.0:1616513090.688530:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.32@tcp1 of length 224/224 into md 0x865be291 [1] + 21832 00000100:00000200:0.0:1616513090.688531:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662630328960 00000400:00000200:16.0:1616513090.688533:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.688535:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513090.688540:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.688542:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:0.0:1616513090.688542:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688662630328960/t0(0) o400->04c84577-49b4-32c8-eb14-988987f0987d@10.31.138.32@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513090.688543:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.688555:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662630328960, offset 192 00000400:00000200:0.0:1616513090.688561:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.32@tcp1 00000400:00000200:0.0:1616513090.688568:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.32@tcp1 routed destination 00000400:00000200:0.0:1616513090.688582:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.32@tcp1(10.31.138.32@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.688588:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.688593:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.688596:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.688598:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.688602:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513090.688605:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662630329088 00000100:00000200:0.0:1616513090.688608:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:12.0:1616513090.688616:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5100 x1688662630329088/t0(0) o400->6482f57f-82d4-e6fa-edf7-25ce55886672@10.31.138.32@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.688630:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662630329088, offset 192 00000400:00000200:12.0:1616513090.688635:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.32@tcp1 00000400:00000200:12.0:1616513090.688642:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.32@tcp1 routed destination 00000800:00000200:11.2:1616513090.688643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513090.688656:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.32@tcp1(10.31.138.32@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.688662:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.688667:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.688670:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.688672:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.688676:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.688681:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513090.688697:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.688704:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.688707:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513090.688708:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513090.688714:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.688719:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43f58 00000800:00000200:28.0:1616513090.688723:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513090.688735:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.688742:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.688745:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.688752:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.688756:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f320 00000800:00000200:22.0:1616513090.688760:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.688763:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.692127:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513090.692180:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.692189:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.692198:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.40@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.692207:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.40@tcp1 of length 224 into portal 26 MB=0x60196b5b519c0 00000400:00000200:2.0:1616513090.692213:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.40@tcp1 of length 224/224 into md 0xf97cd919 [1] + 0 00000400:00000200:2.0:1616513090.692218:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3320 00000400:00000200:2.0:1616513090.692221:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.692223:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513090.692227:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3320 00000800:00000200:2.0:1616513090.692232:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.692234:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.692235:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.692239:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.692289:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690596665530816 00010000:00000200:0.0:1616513090.692300:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1690596665530816/t0(0) o400->c96674d0-4f4f-160e-bcdf-53aec131cacc@10.31.141.40@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.692313:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690596665530816, offset 192 00000400:00000200:0.0:1616513090.692319:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.40@tcp1 00000400:00000200:0.0:1616513090.692325:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.40@tcp1 routed destination 00000800:00000200:7.2:1616513090.692335:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513090.692339:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.40@tcp1(10.31.141.40@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.692345:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.692351:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.692355:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.692357:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.692361:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.692366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.692368:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513090.692389:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.692398:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513090.692407:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.40@tcp1 : PUT - for me 00000400:00000200:22.0:1616513090.692415:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.40@tcp1 of length 224 into portal 12 MB=0x60196b5b51b80 00000800:00000200:16.0:1616513090.692419:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513090.692421:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.40@tcp1 of length 224/224 into md 0xf964d519 [1] + 89240 00000800:00000200:16.0:1616513090.692425:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513090.692427:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513090.692428:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:22.0:1616513090.692430:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513090.692446:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513090.692448:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513090.692452:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513090.692454:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42000 00000800:00000200:22.0:1616513090.692455:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.692457:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513090.692458:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.692461:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513090.692472:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690596665531264 00010000:00000200:14.0:1616513090.692485:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2da00 x1690596665531264/t0(0) o400->3c0ba0cf-e462-163c-e1e9-20681eb107cc@10.31.141.40@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513090.692499:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690596665531264, offset 192 00000400:00000200:14.0:1616513090.692505:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.40@tcp1 00000400:00000200:14.0:1616513090.692512:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.40@tcp1 routed destination 00000400:00000200:14.0:1616513090.692527:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.40@tcp1(10.31.141.40@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513090.692534:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513090.692539:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.692542:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513090.692544:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513090.692549:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.692552:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.692562:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.692570:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513090.692571:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513090.692575:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513090.692577:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513090.692578:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.194@tcp1 : PUT - for me 00000800:00000200:18.0:1616513090.692580:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:2.0:1616513090.692582:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.692585:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47567f8 00000400:00000200:10.0:1616513090.692587:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.194@tcp1 of length 224 into portal 26 MB=0x5ffe126671700 00000800:00000200:18.0:1616513090.692588:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513090.692589:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513090.692591:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513090.692594:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.194@tcp1 of length 224/224 into md 0xf97cd929 [1] + 0 00000400:00000200:18.0:1616513090.692599:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.194@tcp1 : PUT - for me 00000400:00000200:10.0:1616513090.692599:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3a08 00000400:00000200:10.0:1616513090.692601:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513090.692604:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513090.692607:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.194@tcp1 of length 224 into portal 12 MB=0x5ffe126671940 00000400:00000200:10.0:1616513090.692609:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3a08 00000800:00000200:10.0:1616513090.692612:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513090.692614:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.194@tcp1 of length 224/224 into md 0xf964d519 [1] + 89464 00000800:00000200:10.0:1616513090.692615:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513090.692616:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513090.692619:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513090.692621:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717360568064 00000100:00000200:18.0:1616513090.692622:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:0.0:1616513090.692626:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688717360568064/t0(0) o400->466e39ab-1a32-2f5e-0d46-b061377d069e@10.31.142.194@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513090.692628:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.692631:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.692633:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.692637:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688717360568064, offset 192 00000100:00000200:14.0:1616513090.692638:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717360568640 00000400:00000200:0.0:1616513090.692642:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.194@tcp1 00010000:00000200:14.0:1616513090.692644:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746120ba80 x1688717360568640/t0(0) o400->888a910a-f104-c1be-8f9a-255a6768c1ea@10.31.142.194@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513090.692647:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.194@tcp1 routed destination 00000100:00000200:14.0:1616513090.692655:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688717360568640, offset 192 00000400:00000200:14.0:1616513090.692660:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.194@tcp1 00000400:00000200:0.0:1616513090.692660:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.194@tcp1(10.31.142.194@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513090.692666:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.194@tcp1 routed destination 00000800:00000200:0.0:1616513090.692666:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.692670:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.692672:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.692674:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.692677:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513090.692679:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.194@tcp1(10.31.142.194@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513090.692681:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.692681:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513090.692685:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513090.692689:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.692691:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513090.692693:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513090.692696:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.692701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.692723:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.692726:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.692728:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513090.692731:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513090.692733:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.692736:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756a18 00000800:00000200:2.0:1616513090.692738:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513090.692748:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.692754:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.692757:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.692764:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.692769:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42660 00000800:00000200:20.0:1616513090.692773:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.692776:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.697142:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513090.697172:0:50614:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513090.697197:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.697207:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.697216:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.215@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.697225:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.215@tcp1 of length 224 into portal 12 MB=0x5ffd4abf06ec0 00000800:00000200:10.0:1616513090.697228:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513090.697231:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.215@tcp1 of length 224/224 into md 0x865be291 [1] + 22056 00000400:00000200:16.0:1616513090.697236:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:10.0:1616513090.697238:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000100:00000200:16.0:1616513090.697239:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513090.697246:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:10.0:1616513090.697247:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.215@tcp1 : PUT - for me 00000800:00000200:16.0:1616513090.697248:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513090.697250:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.697253:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513090.697255:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.215@tcp1 of length 224 into portal 26 MB=0x5ffd4abf06d80 00000400:00000200:10.0:1616513090.697262:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.215@tcp1 of length 224/224 into md 0xf97cd939 [1] + 0 00000400:00000200:10.0:1616513090.697266:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2cc0 00000400:00000200:10.0:1616513090.697269:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513090.697272:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513090.697276:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2cc0 00000800:00000200:10.0:1616513090.697279:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.697281:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.697283:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513090.697287:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513090.697317:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663766363840 00010000:00000200:12.0:1616513090.697329:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5a00 x1688663766363840/t0(0) o400->821ebb29-33fb-a798-2c70-18bbbf2c2a01@10.31.132.215@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.697344:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663766363840, offset 192 00000100:00000200:0.0:1616513090.697344:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663766363520 00000400:00000200:12.0:1616513090.697350:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.215@tcp1 00010000:00000200:0.0:1616513090.697355:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688663766363520/t0(0) o400->ad863576-d2e3-ecbb-26ca-1719f1138dce@10.31.132.215@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513090.697357:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.215@tcp1 routed destination 00000100:00000200:0.0:1616513090.697370:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663766363520, offset 192 00000400:00000200:12.0:1616513090.697372:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.215@tcp1(10.31.132.215@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513090.697376:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.215@tcp1 00000800:00000200:12.0:1616513090.697378:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513090.697383:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.215@tcp1 routed destination 00000800:00000200:12.0:1616513090.697391:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.697394:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513090.697396:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513090.697399:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.215@tcp1(10.31.132.215@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.697401:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.697405:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.697405:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513090.697411:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.697414:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.697416:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513090.697421:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.697426:0:50614:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513090.697427:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513090.697458:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.697465:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.697468:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.697474:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.697478:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fed0 00000800:00000200:22.0:1616513090.697482:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513090.697482:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.697485:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513090.697489:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.697492:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.697501:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.697506:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42bb0 00000800:00000200:22.0:1616513090.697510:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.697513:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.764158:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.764215:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.764225:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.764237:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.26@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.764247:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.26@tcp1 of length 224 into portal 26 MB=0x5ffdfe97a4140 00000400:00000200:16.0:1616513090.764254:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.26@tcp1 of length 224/224 into md 0xf97cd949 [1] + 0 00000400:00000200:16.0:1616513090.764260:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3760 00000400:00000200:16.0:1616513090.764263:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.764266:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513090.764270:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513090.764272:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3760 00000800:00000200:16.0:1616513090.764275:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513090.764278:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513090.764280:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.764282:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513090.764283:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513090.764291:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.26@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.764300:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.26@tcp1 of length 224 into portal 12 MB=0x5ffdfe97a4280 00000400:00000200:24.0:1616513090.764307:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.26@tcp1 of length 224/224 into md 0x865be291 [1] + 22280 00000400:00000200:24.0:1616513090.764313:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.764316:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513090.764323:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.764327:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.764329:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.764338:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688712043446592 00010000:00000200:0.0:1616513090.764350:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688712043446592/t0(0) o400->c98a3a2c-3995-2297-84c9-861904d41434@10.31.142.26@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.764364:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688712043446592, offset 192 00000400:00000200:0.0:1616513090.764370:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.26@tcp1 00000400:00000200:0.0:1616513090.764377:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.26@tcp1 routed destination 00000100:00000200:12.0:1616513090.764389:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688712043446912 00000400:00000200:0.0:1616513090.764392:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.26@tcp1(10.31.142.26@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.764398:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513090.764403:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e1f80 x1688712043446912/t0(0) o400->d0c2ba49-07b1-ffdb-fa8a-121ba96c816f@10.31.142.26@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513090.764404:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.764406:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.764408:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.764413:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513090.764417:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688712043446912, offset 192 00000800:00000200:11.2:1616513090.764419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.764421:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513090.764424:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.26@tcp1 00000400:00000200:12.0:1616513090.764432:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.26@tcp1 routed destination 00000400:00000200:12.0:1616513090.764468:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.26@tcp1(10.31.142.26@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513090.764479:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513090.764482:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513090.764487:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513090.764488:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513090.764489:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513090.764490:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513090.764491:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.764494:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513090.764496:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:12.0:1616513090.764499:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513090.764502:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42330 00000800:00000200:28.0:1616513090.764505:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513090.764506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.764561:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.764569:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.764572:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513090.764578:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513090.764582:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:18.0:1616513090.764587:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.764590:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.768095:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513090.768149:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.768159:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.768168:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.28@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.768177:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.28@tcp1 of length 224 into portal 26 MB=0x5ffd3f7ee0580 00000400:00000200:24.0:1616513090.768184:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.28@tcp1 of length 224/224 into md 0xf97cd989 [1] + 0 00000400:00000200:24.0:1616513090.768188:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2198 00000400:00000200:24.0:1616513090.768193:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.768196:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513090.768200:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2198 00000800:00000200:24.0:1616513090.768203:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.768203:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.768206:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.768208:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.768264:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660746306944 00010000:00000200:0.0:1616513090.768275:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688660746306944/t0(0) o400->35c4b055-c138-0f63-54f1-6cadb32e647a@10.31.129.28@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.768290:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660746306944, offset 192 00000400:00000200:0.0:1616513090.768296:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.28@tcp1 00000400:00000200:0.0:1616513090.768303:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.28@tcp1 routed destination 00000400:00000200:0.0:1616513090.768318:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.28@tcp1(10.31.129.28@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.768325:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.768330:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.768333:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.768335:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.768340:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.768345:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.768346:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513090.768356:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.768363:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513090.768366:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513090.768372:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.768378:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:16.0:1616513090.768381:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.768384:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.768474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513090.768485:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.768493:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513090.768495:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513090.768502:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.28@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.768512:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.28@tcp1 of length 224 into portal 12 MB=0x5ffd3f7ee09c0 00000400:00000200:20.0:1616513090.768518:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.28@tcp1 of length 224/224 into md 0x865be291 [1] + 22504 00000400:00000200:20.0:1616513090.768524:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.768526:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.768534:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.768537:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.768539:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.768603:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660746308032 00010000:00000200:12.0:1616513090.768615:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d8d80 x1688660746308032/t0(0) o400->96ae860e-4202-27f6-8fed-ecf8fe37b9ac@10.31.129.28@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.768630:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660746308032, offset 192 00000400:00000200:12.0:1616513090.768636:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.28@tcp1 00000400:00000200:12.0:1616513090.768644:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.28@tcp1 routed destination 00000400:00000200:12.0:1616513090.768659:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.28@tcp1(10.31.129.28@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.768665:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.768671:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.768674:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.768676:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.768681:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.768688:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.768743:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.768750:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.768753:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.768759:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.768763:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f4b8 00000800:00000200:22.0:1616513090.768767:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.768769:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.770175:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.770228:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.770237:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.770246:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.14@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.770255:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.14@tcp1 of length 224 into portal 26 MB=0x60182c9d7a3c0 00000400:00000200:16.0:1616513090.770261:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.14@tcp1 of length 224/224 into md 0xf97cd9a9 [1] + 0 00000400:00000200:16.0:1616513090.770266:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3ba0 00000400:00000200:16.0:1616513090.770269:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.770272:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.770276:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3ba0 00000800:00000200:16.0:1616513090.770279:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.770281:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.770282:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513090.770285:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.770339:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511103992768 00010000:00000200:0.0:1616513090.770350:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1690511103992768/t0(0) o400->73f13184-0ce2-1d86-3b78-06a24ce0b2c9@10.31.140.14@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.770365:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511103992768, offset 192 00000400:00000200:0.0:1616513090.770372:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.14@tcp1 00000400:00000200:0.0:1616513090.770380:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.14@tcp1 routed destination 00000400:00000200:0.0:1616513090.770395:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.14@tcp1(10.31.140.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.770401:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.770407:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.770410:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.770412:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.770417:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.770422:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.770423:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513090.770474:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.770481:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513090.770483:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513090.770490:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513090.770495:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:24.0:1616513090.770499:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.770502:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.770552:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513090.770563:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.770571:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513090.770573:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513090.770580:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.14@tcp1 : PUT - for me 00000400:00000200:28.0:1616513090.770589:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.14@tcp1 of length 224 into portal 12 MB=0x60182c9d7a7c0 00000400:00000200:28.0:1616513090.770595:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.14@tcp1 of length 224/224 into md 0x865be291 [1] + 22728 00000400:00000200:28.0:1616513090.770600:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.770603:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513090.770610:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.770612:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513090.770614:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.770673:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511103993792 00010000:00000200:12.0:1616513090.770685:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedba80 x1690511103993792/t0(0) o400->091d01ee-73a9-c2e7-ece2-4ea7ff330247@10.31.140.14@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.770700:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511103993792, offset 192 00000400:00000200:12.0:1616513090.770706:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.14@tcp1 00000400:00000200:12.0:1616513090.770713:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.14@tcp1 routed destination 00000400:00000200:12.0:1616513090.770728:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.14@tcp1(10.31.140.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.770734:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.770740:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.770743:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.770747:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.770751:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.770757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.770812:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.770818:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.770821:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513090.770828:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513090.770832:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fd38 00000800:00000200:18.0:1616513090.770836:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.770838:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.770844:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513090.770846:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.770848:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.770850:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.770852:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.785223:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513090.785279:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.785289:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513090.785294:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513090.785297:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.74@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.785307:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.74@tcp1 of length 224 into portal 12 MB=0x5ffe34e131880 00000400:00000200:2.0:1616513090.785314:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.74@tcp1 of length 224/224 into md 0xf964d519 [1] + 89688 00000400:00000200:2.0:1616513090.785319:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.785322:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513090.785330:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.785333:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.785335:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.785339:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513090.785356:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.785366:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513090.785374:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.74@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.785383:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.74@tcp1 of length 224 into portal 26 MB=0x5ffe34e131800 00000400:00000200:24.0:1616513090.785389:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.74@tcp1 of length 224/224 into md 0xf97cd9c9 [1] + 0 00000400:00000200:24.0:1616513090.785394:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3210 00000400:00000200:24.0:1616513090.785397:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513090.785398:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726616086656 00000100:00000200:24.0:1616513090.785400:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513090.785404:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3210 00000800:00000200:24.0:1616513090.785407:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.785410:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:14.0:1616513090.785410:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7080 x1688726616086656/t0(0) o400->48b26724-c897-a1ad-c895-32a1606a4548@10.31.132.74@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513090.785412:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513090.785416:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513090.785424:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726616086656, offset 192 00000400:00000200:14.0:1616513090.785431:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.74@tcp1 00000400:00000200:14.0:1616513090.785455:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.74@tcp1 routed destination 00000100:00000200:0.0:1616513090.785468:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726616086528 00000400:00000200:14.0:1616513090.785470:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.74@tcp1(10.31.132.74@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513090.785477:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:0.0:1616513090.785479:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688726616086528/t0(0) o400->46d8141d-04cc-5fe8-1f1e-590a2a67c2df@10.31.132.74@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513090.785482:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.785485:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513090.785487:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513090.785492:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.785494:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726616086528, offset 192 00000800:00000200:11.2:1616513090.785496:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513090.785500:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.74@tcp1 00000400:00000200:0.0:1616513090.785507:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.74@tcp1 routed destination 00000800:00000200:16.0:1616513090.785520:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513090.785522:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.74@tcp1(10.31.132.74@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513090.785529:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513090.785529:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513090.785532:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513090.785535:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.785537:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513090.785538:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513090.785540:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513090.785542:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757ba0 00000800:00000200:0.0:1616513090.785544:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513090.785546:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.785549:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513090.785549:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513090.785551:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513090.785620:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.785627:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513090.785630:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513090.785637:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513090.785642:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42bb0 00000800:00000200:10.0:1616513090.785646:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513090.785649:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.811390:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513090.811449:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.811459:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.811468:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.59@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.811477:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.59@tcp1 of length 224 into portal 12 MB=0x5ffd4d4323d40 00000400:00000200:18.0:1616513090.811484:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.59@tcp1 of length 224/224 into md 0xf964d519 [1] + 89912 00000400:00000200:18.0:1616513090.811491:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.811494:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513090.811502:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.811503:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.811506:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.811511:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.811514:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.811514:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.811524:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.59@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.811533:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.59@tcp1 of length 224 into portal 26 MB=0x5ffd4d4323cc0 00000400:00000200:2.0:1616513090.811539:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.59@tcp1 of length 224/224 into md 0xf97cdb69 [1] + 0 00000400:00000200:2.0:1616513090.811544:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2330 00000400:00000200:2.0:1616513090.811547:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.811550:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513090.811556:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2330 00000800:00000200:2.0:1616513090.811559:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.811563:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.811564:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513090.811569:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664441765184 00010000:00000200:14.0:1616513090.811582:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461208d80 x1688664441765184/t0(0) o400->c109f775-d179-e317-19c2-749a65c8960c@10.31.131.59@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513090.811597:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664441765184, offset 192 00000400:00000200:14.0:1616513090.811603:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.59@tcp1 00000400:00000200:14.0:1616513090.811610:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.59@tcp1 routed destination 00000100:00000200:0.0:1616513090.811619:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664441765056 00000400:00000200:14.0:1616513090.811625:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.59@tcp1(10.31.131.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:0.0:1616513090.811630:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688664441765056/t0(0) o400->523b6c69-f7b3-abf2-f066-bf0b0113f8e4@10.31.131.59@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513090.811632:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513090.811638:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.811640:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513090.811642:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.811643:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664441765056, offset 192 00000800:00000200:14.0:1616513090.811648:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513090.811649:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.59@tcp1 00000800:00000200:11.2:1616513090.811655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513090.811656:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.59@tcp1 routed destination 00000400:00000200:0.0:1616513090.811670:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.59@tcp1(10.31.131.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.811676:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513090.811683:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.811686:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.811688:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513090.811693:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.811698:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513090.811699:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513090.811711:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.811718:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.811721:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.811727:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.811731:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757c28 00000800:00000200:20.0:1616513090.811735:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.811738:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513090.811755:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.811763:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.811765:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513090.811767:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513090.811772:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.811778:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42000 00000800:00000200:22.0:1616513090.811781:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.813388:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513090.813398:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513090.813454:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513090.813462:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513090.813463:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:16.0:1616513090.813471:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513090.813472:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.90@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.813480:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.90@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.813481:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.90@tcp1 of length 224 into portal 26 MB=0x602ad3a352cc0 00000400:00000200:2.0:1616513090.813487:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.90@tcp1 of length 224/224 into md 0xf97cdb79 [1] + 0 00000400:00000200:16.0:1616513090.813489:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.90@tcp1 of length 224 into portal 12 MB=0x602ad3a352e80 00000400:00000200:2.0:1616513090.813492:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f3b18 00000400:00000200:16.0:1616513090.813495:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.90@tcp1 of length 224/224 into md 0x865be291 [1] + 22952 00000400:00000200:2.0:1616513090.813495:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.813497:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.813501:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.813501:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3b18 00000100:00000200:16.0:1616513090.813503:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513090.813504:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.813507:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.813509:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513090.813510:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513090.813512:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513090.813513:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513090.813515:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.813518:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.813582:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691792889425088 00000100:00000200:12.0:1616513090.813583:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691792889425536 00010000:00000200:0.0:1616513090.813593:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1691792889425088/t0(0) o400->addee483-e073-96cc-e092-50d42c954091@10.31.141.90@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:12.0:1616513090.813596:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd6300 x1691792889425536/t0(0) o400->bbb97a49-90be-f18e-834b-8ee8baa1130b@10.31.141.90@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513090.813608:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691792889425088, offset 192 00000100:00000200:12.0:1616513090.813610:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691792889425536, offset 192 00000400:00000200:0.0:1616513090.813614:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.90@tcp1 00000400:00000200:12.0:1616513090.813620:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.90@tcp1 00000400:00000200:0.0:1616513090.813621:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.90@tcp1 routed destination 00000400:00000200:0.0:1616513090.813636:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.90@tcp1(10.31.141.90@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.813643:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513090.813645:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.90@tcp1 routed destination 00000800:00000200:0.0:1616513090.813652:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.813655:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.813658:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513090.813660:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.90@tcp1(10.31.141.90@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.813662:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513090.813667:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513090.813667:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.813669:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513090.813672:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.813675:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.813677:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.813682:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.813687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513090.813719:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.813726:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.813729:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.813735:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.813741:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42000 00000800:00000200:18.0:1616513090.813741:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513090.813744:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.813747:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513090.813748:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.813751:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513090.813757:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513090.813761:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eee0 00000800:00000200:18.0:1616513090.813765:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.813768:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:31.0:1616513090.871543:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513090.871551:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513090.890856:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.890934:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.890948:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513090.890953:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513090.890958:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.118@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.890969:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.118@tcp1 of length 224 into portal 26 MB=0x6018350c29d00 00000400:00000200:16.0:1616513090.890976:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.118@tcp1 of length 224/224 into md 0xf97cdb99 [1] + 0 00000400:00000200:16.0:1616513090.890981:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dc32f2e58 00000400:00000200:16.0:1616513090.890985:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.890988:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.890993:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2e58 00000800:00000200:16.0:1616513090.890996:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.890999:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.891001:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.891060:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513367538944 00010000:00000200:0.0:1616513090.891073:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1690513367538944/t0(0) o400->890589bb-9cf3-27b0-7abb-40cd1d752075@10.31.142.118@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.891089:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513367538944, offset 192 00000400:00000200:0.0:1616513090.891095:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.118@tcp1 00000400:00000200:0.0:1616513090.891103:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.118@tcp1 routed destination 00000400:00000200:0.0:1616513090.891119:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.118@tcp1(10.31.142.118@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.891126:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.891132:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.891135:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.891138:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.891143:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.891149:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.891153:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513090.891204:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.891211:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.891214:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.891221:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.891227:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:20.0:1616513090.891230:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.891234:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.891240:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513090.891242:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513090.891247:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.118@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.891256:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.118@tcp1 of length 224 into portal 12 MB=0x6018350c2a140 00000400:00000200:20.0:1616513090.891262:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.118@tcp1 of length 224/224 into md 0x865be291 [1] + 23176 00000400:00000200:20.0:1616513090.891269:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.891272:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.891278:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.891280:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.891282:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.891343:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513367540032 00010000:00000200:12.0:1616513090.891357:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d8000 x1690513367540032/t0(0) o400->8c5e1114-3e6e-633b-2ec8-42974026d6ab@10.31.142.118@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.891372:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513367540032, offset 192 00000400:00000200:12.0:1616513090.891378:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.118@tcp1 00000400:00000200:12.0:1616513090.891386:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.118@tcp1 routed destination 00000400:00000200:12.0:1616513090.891401:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.118@tcp1(10.31.142.118@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.891408:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.891414:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.891417:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.891419:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.891425:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.891433:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.891491:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.891497:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.891500:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.891507:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.891511:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:2.0:1616513090.891515:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.891518:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.892956:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.893011:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.893021:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.893029:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.116@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.893039:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.116@tcp1 of length 224 into portal 12 MB=0x604f82630b780 00000400:00000200:16.0:1616513090.893047:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.116@tcp1 of length 224/224 into md 0x865be291 [1] + 23400 00000400:00000200:16.0:1616513090.893052:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.893055:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513090.893061:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.893064:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.893065:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513090.893069:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513090.893070:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.893124:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694313699391360 00000800:00000200:18.0:1616513090.893129:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:12.0:1616513090.893136:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd3600 x1694313699391360/t0(0) o400->64a9559c-9b0d-e6f3-7733-b8e49d4b2e97@10.31.140.116@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513090.893138:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.893147:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.116@tcp1 : PUT - for me 00000100:00000200:12.0:1616513090.893150:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694313699391360, offset 192 00000400:00000200:18.0:1616513090.893157:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.116@tcp1 of length 224 into portal 26 MB=0x604f82630b540 00000400:00000200:12.0:1616513090.893158:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.116@tcp1 00000400:00000200:18.0:1616513090.893163:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.116@tcp1 of length 224/224 into md 0xf97cdba9 [1] + 0 00000400:00000200:12.0:1616513090.893166:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.116@tcp1 routed destination 00000400:00000200:18.0:1616513090.893168:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc5d8 00000400:00000200:18.0:1616513090.893171:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.893174:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513090.893178:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc5d8 00000400:00000200:12.0:1616513090.893180:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.116@tcp1(10.31.140.116@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513090.893181:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.893183:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513090.893184:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513090.893187:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513090.893188:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513090.893192:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.893195:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513090.893197:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513090.893202:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.893207:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.893243:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694313699390784 00010000:00000200:0.0:1616513090.893254:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1694313699390784/t0(0) o400->0b6d0a50-41f2-6741-bc1f-a28f2f26e8dc@10.31.140.116@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513090.893260:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.893266:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513090.893268:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694313699390784, offset 192 00000400:00000200:24.0:1616513090.893269:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513090.893274:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.116@tcp1 00000400:00000200:24.0:1616513090.893276:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513090.893280:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f650 00000400:00000200:0.0:1616513090.893282:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.116@tcp1 routed destination 00000800:00000200:24.0:1616513090.893288:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.893291:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513090.893297:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.116@tcp1(10.31.140.116@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.893304:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513090.893310:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.893312:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.893315:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513090.893319:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.893324:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513090.893326:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513090.893377:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.893384:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513090.893386:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513090.893393:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513090.893399:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42770 00000800:00000200:2.0:1616513090.893402:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.893405:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513090.906475:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928304, portal 4 00000100:00000200:6.0:1616513090.906482:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 24, xid 1693067725928304, offset 0 00000400:00000200:6.0:1616513090.906489:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:6.0:1616513090.906494:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:6.0:1616513090.906507:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513090.906514:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:6.0:1616513090.906519:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513090.906522:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:6.0:1616513090.906524:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:6.0:1616513090.906530:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513090.906576:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:22.0:1616513090.906634:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:22.0:1616513090.906641:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:22.0:1616513090.906643:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.906650:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000100:00000200:22.0:1616513090.906656:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf82400 x1693067725928304/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513097 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513090.906668:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01b18 00000800:00000200:22.0:1616513090.906672:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:22.0:1616513090.906675:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:31.2:1616513090.906746:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:2.0:1616513090.906758:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:2.0:1616513090.906768:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.234@o2ib1 00000800:00000200:10.0:1616513090.906770:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:2.0:1616513090.906777:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513090.906785:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 336 into portal 4 MB=0x603d60c627770 00000400:00000200:2.0:1616513090.906794:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.234@o2ib1 of length 336/336 into md 0xf97ce239 [1] + 192 00000400:00000200:2.0:1616513090.906799:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.906804:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf82400 x1693067725928304/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513090.906819:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:2.0:1616513090.906822:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)-- 00000100:00000200:6.0:1616513090.906824:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf82400 x1693067725928304/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513090.906824:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:6.0:1616513090.906837:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf82400 x1693067725928304/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513097 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513090.906846:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01650 00000800:00000200:11.2:1616513090.908714:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513090.908770:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513090.908780:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513090.908789:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.79@tcp1 : PUT - for me 00000400:00000200:28.0:1616513090.908798:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.79@tcp1 of length 224 into portal 26 MB=0x5f90f340fa740 00000400:00000200:28.0:1616513090.908805:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.79@tcp1 of length 224/224 into md 0xf97cdbb9 [1] + 0 00000400:00000200:28.0:1616513090.908810:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc4c8 00000400:00000200:28.0:1616513090.908812:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513090.908817:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513090.908822:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc4c8 00000800:00000200:28.0:1616513090.908825:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.908825:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.908827:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.908831:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.908834:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.908835:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.908843:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.79@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.908852:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.79@tcp1 of length 224 into portal 12 MB=0x5f90f340fa7c0 00000400:00000200:16.0:1616513090.908859:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.79@tcp1 of length 224/224 into md 0x865be291 [1] + 23624 00000400:00000200:16.0:1616513090.908864:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.908866:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513090.908873:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.908876:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.908878:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.908885:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681218576820032 00010000:00000200:0.0:1616513090.908896:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1681218576820032/t0(0) o400->68e5867d-8ff7-6b4f-7bc5-104208ab0655@10.31.131.79@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.908911:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681218576820032, offset 192 00000400:00000200:0.0:1616513090.908917:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.79@tcp1 00000400:00000200:0.0:1616513090.908924:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.79@tcp1 routed destination 00000100:00000200:12.0:1616513090.908936:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681218576820160 00000400:00000200:0.0:1616513090.908939:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.79@tcp1(10.31.131.79@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.908946:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513090.908948:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd4c80 x1681218576820160/t0(0) o400->401fb5ca-806b-fea5-ef51-836fea93ba2a@10.31.131.79@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513090.908951:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.908954:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.908956:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.908961:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513090.908963:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681218576820160, offset 192 00000100:00000200:0.0:1616513090.908968:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513090.908969:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.79@tcp1 00000800:00000200:11.2:1616513090.908972:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513090.908977:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.79@tcp1 routed destination 00000800:00000200:20.0:1616513090.908984:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.908991:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513090.908991:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.79@tcp1(10.31.131.79@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513090.908994:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513090.908995:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513090.908998:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513090.909000:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:12.0:1616513090.909004:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513090.909006:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:12.0:1616513090.909006:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513090.909009:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513090.909009:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.909013:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.909021:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.909076:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.909083:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.909086:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.909093:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.909097:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f650 00000800:00000200:22.0:1616513090.909101:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.909104:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.917435:0:50637:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.917449:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.917458:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513090.917467:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.14@tcp1 : PUT - for me 00000400:00000200:10.0:1616513090.917476:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.14@tcp1 of length 224 into portal 12 MB=0x603d58c06b840 00000400:00000200:10.0:1616513090.917483:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.21.14@tcp1 of length 224/224 into md 0xf964d519 [1] + 90136 00000400:00000200:10.0:1616513090.917488:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513090.917491:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513090.917499:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.917502:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513090.917503:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.917506:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.917549:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:14.0:1616513090.917564:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693065572431936 00010000:00000200:14.0:1616513090.917577:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bd8000 x1693065572431936/t0(0) o400->661d41f4-74f2-53da-e9c6-dc82c3077857@10.31.21.14@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513090.917591:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693065572431936, offset 192 00000400:00000200:14.0:1616513090.917597:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.14@tcp1 00000400:00000200:14.0:1616513090.917605:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.14@tcp1 routed destination 00000800:00000200:16.0:1616513090.917617:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513090.917619:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.14@tcp1(10.31.21.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513090.917626:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513090.917627:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513090.917631:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.917634:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513090.917636:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.14@tcp1 : PUT - for me 00000800:00000200:14.0:1616513090.917639:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:14.0:1616513090.917643:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513090.917644:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.14@tcp1 of length 224 into portal 26 MB=0x603d58c06b6c0 00000400:00000200:16.0:1616513090.917651:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.21.14@tcp1 of length 224/224 into md 0xf97cdbc9 [1] + 0 00000400:00000200:16.0:1616513090.917656:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc2a8 00000400:00000200:16.0:1616513090.917659:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.917661:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.917675:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc2a8 00000800:00000200:24.0:1616513090.917678:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513090.917678:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513090.917681:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513090.917691:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513090.917692:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513090.917694:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513090.917695:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513090.917702:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513090.917706:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756000 00000800:00000200:24.0:1616513090.917710:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.917738:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693065572431552 00010000:00000200:0.0:1616513090.917749:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1693065572431552/t0(0) o400->fdcc94b8-282e-a355-abce-91fc7520d5e2@10.31.21.14@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.917762:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693065572431552, offset 192 00000400:00000200:0.0:1616513090.917775:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.14@tcp1 00000400:00000200:0.0:1616513090.917781:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.14@tcp1 routed destination 00000400:00000200:0.0:1616513090.917795:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.14@tcp1(10.31.21.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.917801:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513090.917807:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.917809:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513090.917811:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513090.917816:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.917821:0:50637:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513090.917822:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513090.917875:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.917889:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.917892:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.917898:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.917904:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:22.0:1616513090.917907:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.917910:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.948614:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.948626:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.948635:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513090.948642:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.189@tcp1 : PUT - for me 00000400:00000200:10.0:1616513090.948652:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.189@tcp1 of length 224 into portal 26 MB=0x6021577c8e4c0 00000400:00000200:10.0:1616513090.948657:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.189@tcp1 of length 224/224 into md 0xf97cdbd9 [1] + 0 00000400:00000200:10.0:1616513090.948661:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc088 00000400:00000200:10.0:1616513090.948664:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513090.948666:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513090.948669:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc088 00000800:00000200:10.0:1616513090.948672:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.948674:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513090.948676:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.948678:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.948732:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691141087487168 00010000:00000200:0.0:1616513090.948743:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1691141087487168/t0(0) o400->bdc05494-bb1b-1091-9424-856ac5e9d5a6@10.31.132.189@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.948756:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691141087487168, offset 192 00000400:00000200:0.0:1616513090.948762:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.189@tcp1 00000400:00000200:0.0:1616513090.948770:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.189@tcp1 routed destination 00000800:00000200:11.2:1616513090.948784:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:0.0:1616513090.948784:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.189@tcp1(10.31.132.189@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.948790:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.948795:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.948797:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.948799:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.948799:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513090.948805:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513090.948810:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513090.948811:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513090.948819:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.189@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.948828:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.189@tcp1 of length 224 into portal 12 MB=0x6021577c8e5c0 00000400:00000200:20.0:1616513090.948835:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.189@tcp1 of length 224/224 into md 0x865be291 [1] + 23848 00000400:00000200:20.0:1616513090.948840:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.948843:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.948850:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513090.948852:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513090.948853:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513090.948859:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513090.948860:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513090.948862:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513090.948862:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513090.948870:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513090.948876:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:16.0:1616513090.948879:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.948911:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691141087487424 00010000:00000200:12.0:1616513090.948922:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dec00 x1691141087487424/t0(0) o400->92b2cd36-4c14-bce4-4389-2b8706951466@10.31.132.189@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.948936:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691141087487424, offset 192 00000400:00000200:12.0:1616513090.948941:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.189@tcp1 00000400:00000200:12.0:1616513090.948949:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.189@tcp1 routed destination 00000400:00000200:12.0:1616513090.948962:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.189@tcp1(10.31.132.189@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.948968:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.948974:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.948978:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.948980:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.948985:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.948990:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.949042:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.949049:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.949052:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.949058:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.949062:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:22.0:1616513090.949065:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.949068:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.955192:0:50641:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.955204:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.955212:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513090.955221:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.143@tcp1 : PUT - for me 00000400:00000200:10.0:1616513090.955229:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.143@tcp1 of length 224 into portal 26 MB=0x5ffe4cc0e4700 00000400:00000200:10.0:1616513090.955235:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.143@tcp1 of length 224/224 into md 0xf97cdc09 [1] + 0 00000400:00000200:10.0:1616513090.955240:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc3b8 00000400:00000200:10.0:1616513090.955242:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513090.955245:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513090.955249:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc3b8 00000800:00000200:10.0:1616513090.955251:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.955255:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513090.955257:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513090.955259:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513090.955270:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513090.955278:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.143@tcp1 : PUT - for me 00000400:00000200:2.0:1616513090.955287:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.143@tcp1 of length 224 into portal 12 MB=0x5ffe4cc0e49c0 00000400:00000200:2.0:1616513090.955293:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.143@tcp1 of length 224/224 into md 0xf964d519 [1] + 90360 00000400:00000200:2.0:1616513090.955298:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513090.955301:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513090.955308:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513090.955311:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.955313:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513090.955314:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688733024667392 00000800:00000200:2.0:1616513090.955316:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513090.955325:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688733024667392/t0(0) o400->afc3bf3a-792d-b03c-4849-9f29400d957f@10.31.141.143@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.955339:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688733024667392, offset 192 00000400:00000200:0.0:1616513090.955345:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.143@tcp1 00000400:00000200:0.0:1616513090.955352:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.143@tcp1 routed destination 00000400:00000200:0.0:1616513090.955367:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.143@tcp1(10.31.141.143@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:14.0:1616513090.955373:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688733024668096 00000800:00000200:0.0:1616513090.955374:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.955381:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.955384:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:14.0:1616513090.955385:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5e80 x1688733024668096/t0(0) o400->89d85de0-4207-daa7-a00a-7dfffcd1f757@10.31.141.143@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513090.955386:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.955391:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.955398:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:14.0:1616513090.955399:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688733024668096, offset 192 00000400:00000200:14.0:1616513090.955405:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.143@tcp1 00000400:00000200:14.0:1616513090.955412:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.143@tcp1 routed destination 00000400:00000200:14.0:1616513090.955427:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.143@tcp1(10.31.141.143@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513090.955436:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513090.955448:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513090.955454:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.955457:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513090.955459:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513090.955464:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.955468:0:50641:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513090.955493:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513090.955500:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513090.955503:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513090.955505:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513090.955509:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513090.955515:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:28.0:1616513090.955518:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513090.955524:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513090.955531:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513090.955534:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513090.955535:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513090.955540:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513090.955544:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47562a8 00000800:00000200:18.0:1616513090.955548:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513090.979902:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513090.979959:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513090.979968:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513090.979978:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.241@tcp1 : PUT - for me 00000400:00000200:16.0:1616513090.979988:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.241@tcp1 of length 224 into portal 26 MB=0x5ffd54831df40 00000400:00000200:16.0:1616513090.979995:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.241@tcp1 of length 224/224 into md 0xf97cdc29 [1] + 0 00000400:00000200:16.0:1616513090.980000:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc7f8 00000400:00000200:16.0:1616513090.980003:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513090.980006:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513090.980011:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc7f8 00000800:00000200:24.0:1616513090.980013:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.980014:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513090.980017:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513090.980019:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513090.980077:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666387898176 00010000:00000200:0.0:1616513090.980088:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688666387898176/t0(0) o400->1adb190c-da90-ac01-1cd6-6686c7f0ed1e@10.31.131.241@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.980103:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666387898176, offset 192 00000400:00000200:0.0:1616513090.980109:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.241@tcp1 00000400:00000200:0.0:1616513090.980118:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.241@tcp1 routed destination 00000400:00000200:0.0:1616513090.980133:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.241@tcp1(10.31.131.241@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.980140:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.980146:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.980148:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.980151:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.980156:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513090.980161:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513090.980163:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513090.980173:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.980181:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.980183:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.980190:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.980196:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:20.0:1616513090.980199:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513090.980202:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513090.980626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513090.980636:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.980644:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513090.980646:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513090.980653:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.241@tcp1 : PUT - for me 00000400:00000200:24.0:1616513090.980662:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.241@tcp1 of length 224 into portal 12 MB=0x5ffd54831e040 00000400:00000200:24.0:1616513090.980669:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.241@tcp1 of length 224/224 into md 0x865be291 [1] + 24072 00000400:00000200:24.0:1616513090.980674:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513090.980677:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513090.980684:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.980686:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.980688:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.980748:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666387898432 00010000:00000200:12.0:1616513090.980761:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e2880 x1688666387898432/t0(0) o400->094908f2-3e85-f913-06e4-3aedf1911abb@10.31.131.241@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.980777:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666387898432, offset 192 00000400:00000200:12.0:1616513090.980783:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.241@tcp1 00000400:00000200:12.0:1616513090.980791:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.241@tcp1 routed destination 00000400:00000200:12.0:1616513090.980806:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.241@tcp1(10.31.131.241@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.980813:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.980818:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.980821:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.980824:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.980828:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.980837:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.980894:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.980901:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.980904:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.980911:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.980915:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eee0 00000800:00000200:22.0:1616513090.980919:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.980922:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.984447:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513090.984510:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.984528:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513090.984536:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.75@tcp1 : PUT - for me 00000400:00000200:10.0:1616513090.984545:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.75@tcp1 of length 224 into portal 26 MB=0x5ffd4a7f50240 00000400:00000200:10.0:1616513090.984550:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.75@tcp1 of length 224/224 into md 0xf97cdc39 [1] + 0 00000400:00000200:10.0:1616513090.984554:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5cd4fcc908 00000400:00000200:10.0:1616513090.984557:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513090.984559:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513090.984563:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc908 00000800:00000200:10.0:1616513090.984565:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.984568:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513090.984569:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513090.984571:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.984627:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663699554880 00010000:00000200:0.0:1616513090.984638:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688663699554880/t0(0) o400->1a74bc9d-a53b-0636-fd69-6839df8c58ea@10.31.136.75@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.984653:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663699554880, offset 192 00000400:00000200:0.0:1616513090.984659:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.75@tcp1 00000400:00000200:0.0:1616513090.984667:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.75@tcp1 routed destination 00000400:00000200:0.0:1616513090.984681:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.75@tcp1(10.31.136.75@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.984688:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.984694:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.984696:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.984698:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.984703:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.984711:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513090.984715:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513090.984730:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.984739:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513090.984748:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.75@tcp1 : PUT - for me 00000400:00000200:20.0:1616513090.984757:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.75@tcp1 of length 224 into portal 12 MB=0x5ffd4a7f50340 00000400:00000200:20.0:1616513090.984765:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.75@tcp1 of length 224/224 into md 0x865be291 [1] + 24296 00000400:00000200:20.0:1616513090.984770:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513090.984773:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513090.984779:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513090.984782:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513090.984784:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513090.984786:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513090.984790:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513090.984792:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513090.984792:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513090.984798:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513090.984802:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a08 00000800:00000200:20.0:1616513090.984805:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513090.984843:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663699555136 00010000:00000200:12.0:1616513090.984855:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dc380 x1688663699555136/t0(0) o400->35e4a0fa-7fa8-2b2f-885a-cd60877d82bf@10.31.136.75@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513090.984869:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663699555136, offset 192 00000400:00000200:12.0:1616513090.984875:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.75@tcp1 00000400:00000200:12.0:1616513090.984882:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.75@tcp1 routed destination 00000400:00000200:12.0:1616513090.984897:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.75@tcp1(10.31.136.75@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513090.984904:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513090.984909:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513090.984912:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513090.984914:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513090.984919:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.984923:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.984976:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.984983:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513090.984985:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513090.984991:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513090.984995:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fd38 00000800:00000200:22.0:1616513090.984999:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.985002:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513090.987363:0:50644:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513090.987418:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.987428:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513090.987456:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.113@tcp1 : PUT - for me 00000400:00000200:18.0:1616513090.987466:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.113@tcp1 of length 224 into portal 26 MB=0x602be1cfa7d00 00000400:00000200:18.0:1616513090.987473:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.113@tcp1 of length 224/224 into md 0xf97cdc49 [1] + 0 00000800:00000200:10.0:1616513090.987473:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513090.987477:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42b28 00000400:00000200:18.0:1616513090.987480:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513090.987483:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513090.987487:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42b28 00000800:00000200:18.0:1616513090.987490:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513090.987493:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513090.987495:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513090.987550:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691865413483776 00010000:00000200:0.0:1616513090.987562:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1691865413483776/t0(0) o400->b55b8af2-7b69-fb98-ab08-76308299d745@10.31.140.113@tcp1:47/0 lens 224/224 e 0 to 0 dl 1616513182 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513090.987576:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691865413483776, offset 192 00000400:00000200:0.0:1616513090.987582:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.113@tcp1 00000400:00000200:0.0:1616513090.987590:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.113@tcp1 routed destination 00000400:00000200:0.0:1616513090.987605:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.113@tcp1(10.31.140.113@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513090.987612:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513090.987617:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513090.987620:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513090.987622:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513090.987627:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513090.987634:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513090.987671:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513090.987726:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513090.987738:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513090.987741:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513090.987748:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513090.987753:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:24.0:1616513090.987756:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513090.987760:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513090.987801:0:50644:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513090.987852:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513090.987861:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513090.987869:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.113@tcp1 : PUT - for me 00000400:00000200:22.0:1616513090.987877:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.113@tcp1 of length 224 into portal 12 MB=0x602be1cfa83c0 00000400:00000200:22.0:1616513090.987883:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.113@tcp1 of length 224/224 into md 0xf964d519 [1] + 90584 00000400:00000200:22.0:1616513090.987888:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513090.987891:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513090.987900:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513090.987903:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513090.987905:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513090.987909:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513090.987966:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691865413485504 00010000:00000200:14.0:1616513090.987978:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2de80 x1691865413485504/t0(0) o400->3052d8ae-c5bc-26af-c28e-f0b89beffddd@10.31.140.113@tcp1:716/0 lens 224/224 e 0 to 0 dl 1616513096 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513090.987992:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691865413485504, offset 192 00000400:00000200:14.0:1616513090.987998:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.113@tcp1 00000400:00000200:14.0:1616513090.988006:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.113@tcp1 routed destination 00000400:00000200:14.0:1616513090.988021:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.113@tcp1(10.31.140.113@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513090.988027:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513090.988033:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513090.988036:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513090.988038:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513090.988043:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513090.988047:0:50644:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513090.988057:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513090.988061:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513090.988063:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513090.988066:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513090.988069:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513090.988073:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756088 00000800:00000200:10.0:1616513090.988076:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.001480:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513091.001503:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513091.001535:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.001544:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.001553:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.127@tcp1 : PUT - for me 00000800:00000200:20.0:1616513091.001562:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513091.001562:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.127@tcp1 of length 224 into portal 12 MB=0x603fc19405e40 00000400:00000200:2.0:1616513091.001569:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.127@tcp1 of length 224/224 into md 0xf964d519 [1] + 90808 00000800:00000200:20.0:1616513091.001571:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513091.001574:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.001577:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513091.001579:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.127@tcp1 : PUT - for me 00000800:00000200:2.0:1616513091.001584:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513091.001587:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.127@tcp1 of length 224 into portal 26 MB=0x603fc19405dc0 00000800:00000200:2.0:1616513091.001587:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.001589:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.001591:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513091.001593:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.127@tcp1 of length 224/224 into md 0xf97cdc59 [1] + 0 00000400:00000200:20.0:1616513091.001598:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a433a8 00000400:00000200:20.0:1616513091.001601:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.001603:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.001607:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a433a8 00000800:00000200:20.0:1616513091.001610:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.001613:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.001615:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.001617:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513091.001651:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693231150554688 00010000:00000200:14.0:1616513091.001664:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4c80 x1693231150554688/t0(0) o400->eb557423-c425-8ec8-9857-c669296a0865@10.31.138.127@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513091.001674:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693231150554560 00000100:00000200:14.0:1616513091.001678:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693231150554688, offset 192 00010000:00000200:0.0:1616513091.001685:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1693231150554560/t0(0) o400->93773d01-b106-7bf5-e7ab-3e5d24877fcd@10.31.138.127@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513091.001687:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.127@tcp1 00000400:00000200:14.0:1616513091.001694:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.127@tcp1 routed destination 00000100:00000200:0.0:1616513091.001698:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693231150554560, offset 192 00000400:00000200:0.0:1616513091.001704:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.127@tcp1 00000400:00000200:14.0:1616513091.001709:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.127@tcp1(10.31.138.127@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513091.001715:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513091.001718:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.127@tcp1 routed destination 00000800:00000200:14.0:1616513091.001721:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513091.001724:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513091.001726:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513091.001731:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513091.001732:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.127@tcp1(10.31.138.127@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513091.001737:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.001738:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513091.001743:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.001746:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.001748:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.001752:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.001758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513091.001759:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513091.001791:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.001798:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.001801:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.001807:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.001811:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756110 00000800:00000200:24.0:1616513091.001815:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.001818:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513091.001827:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.001834:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.001837:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.001843:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.001849:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05320 00000800:00000200:18.0:1616513091.001853:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.001856:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:1.0:1616513091.004769:0:6493:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928320, portal 4 00000100:00000200:1.0:1616513091.004776:0:6493:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928320, offset 0 00000400:00000200:1.0:1616513091.004783:0:6493:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:1.0:1616513091.004790:0:6493:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:1.0:1616513091.004805:0:6493:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:1.0:1616513091.004812:0:6493:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:1.0:1616513091.004819:0:6493:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513091.004823:0:6493:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:1.0:1616513091.004825:0:6493:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:1.0:1616513091.004832:0:6493:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513091.004879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513091.004895:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513091.004904:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513091.004907:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513091.004914:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513091.004920:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1395a00 x1693067725928320/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513091.004933:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85540 00000800:00000200:2.0:1616513091.004938:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513091.004941:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513091.005065:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513091.005082:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513091.005091:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513091.005100:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513091.005108:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627780 00000400:00000200:18.0:1616513091.005115:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97ce319 [1] + 192 00000400:00000200:18.0:1616513091.005120:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.005125:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1395a00 x1693067725928320/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:10.0:1616513091.005135:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513091.005140:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513091.005143:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513091.005145:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:1.0:1616513091.005152:0:6493:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1395a00 x1693067725928320/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513091.005166:0:6493:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1395a00 x1693067725928320/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513091.005176:0:6493:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84000 00000800:00000200:7.2:1616513091.019896:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513091.019948:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513091.019968:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.019977:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.019987:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.75@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.019996:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.75@tcp1 of length 224 into portal 12 MB=0x60037fdeb7bc0 00000400:00000200:2.0:1616513091.020003:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.75@tcp1 of length 224/224 into md 0xf964d519 [1] + 91032 00000800:00000200:20.0:1616513091.020006:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513091.020008:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.020011:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.020015:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513091.020017:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.020020:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.020022:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513091.020025:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.75@tcp1 : PUT - for me 00000800:00000200:2.0:1616513091.020025:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513091.020033:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.75@tcp1 of length 224 into portal 26 MB=0x60037fdeb7b40 00000400:00000200:20.0:1616513091.020040:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.75@tcp1 of length 224/224 into md 0xf97cdc79 [1] + 0 00000400:00000200:20.0:1616513091.020045:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a425d8 00000400:00000200:20.0:1616513091.020047:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.020052:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.020057:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a425d8 00000800:00000200:20.0:1616513091.020060:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.020060:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.020063:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.020065:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:14.0:1616513091.020094:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689090343533504 00010000:00000200:14.0:1616513091.020105:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0900 x1689090343533504/t0(0) o400->6d000886-ac0a-a9ef-17b3-0aeeb28f3f8c@10.31.141.75@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513091.020119:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689090343533504, offset 192 00000100:00000200:0.0:1616513091.020119:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689090343533376 00000400:00000200:14.0:1616513091.020125:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.75@tcp1 00010000:00000200:0.0:1616513091.020128:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1689090343533376/t0(0) o400->d87d56a1-5090-d1ec-f142-a009bba485d1@10.31.141.75@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513091.020132:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.75@tcp1 routed destination 00000100:00000200:0.0:1616513091.020141:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689090343533376, offset 192 00000400:00000200:14.0:1616513091.020146:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.75@tcp1(10.31.141.75@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513091.020148:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.75@tcp1 00000800:00000200:14.0:1616513091.020152:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513091.020154:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.75@tcp1 routed destination 00000800:00000200:14.0:1616513091.020158:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513091.020160:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513091.020162:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513091.020167:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513091.020167:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.75@tcp1(10.31.141.75@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513091.020172:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.020173:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513091.020178:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.020181:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.020183:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.020187:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.020193:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513091.020193:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513091.020227:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.020243:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.020245:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513091.020252:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513091.020252:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.020258:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47577e8 00000800:00000200:24.0:1616513091.020262:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513091.020263:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.020271:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.020273:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513091.020280:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513091.020286:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:10.0:1616513091.020289:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.020292:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.038383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.038462:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.038477:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.038486:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.195@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.038495:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.195@tcp1 of length 224 into portal 26 MB=0x5ffdff24b5ac0 00000400:00000200:16.0:1616513091.038501:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.195@tcp1 of length 224/224 into md 0xf97cdcb9 [1] + 0 00000400:00000200:16.0:1616513091.038506:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43c28 00000400:00000200:16.0:1616513091.038509:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.038512:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513091.038517:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43c28 00000800:00000200:16.0:1616513091.038520:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.038523:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513091.038523:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513091.038529:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.038532:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513091.038532:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.038541:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.195@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.038549:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.195@tcp1 of length 224 into portal 12 MB=0x5ffdff24b5d00 00000400:00000200:20.0:1616513091.038555:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.195@tcp1 of length 224/224 into md 0x865be291 [1] + 24520 00000400:00000200:20.0:1616513091.038560:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.038562:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.038569:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.038572:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.038574:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.038579:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688712191367872 00010000:00000200:0.0:1616513091.038589:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688712191367872/t0(0) o400->1e9c6b9b-8e6d-37eb-b0a9-7af7715e515e@10.31.142.195@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.038602:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688712191367872, offset 192 00000400:00000200:0.0:1616513091.038607:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.195@tcp1 00000400:00000200:0.0:1616513091.038614:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.195@tcp1 routed destination 00000400:00000200:0.0:1616513091.038627:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.195@tcp1(10.31.142.195@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.038632:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.038637:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.038640:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.038642:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513091.038646:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688712191368448 00000800:00000200:0.0:1616513091.038646:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.038651:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.038652:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:12.0:1616513091.038659:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dcc80 x1688712191368448/t0(0) o400->c725bf08-5602-c465-c5a3-719d887de8d1@10.31.142.195@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513091.038663:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.038670:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.038673:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.038674:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513091.038680:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:12.0:1616513091.038683:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688712191368448, offset 192 00000400:00000200:24.0:1616513091.038686:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05dc0 00000800:00000200:24.0:1616513091.038690:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:12.0:1616513091.038692:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.195@tcp1 00000400:00000200:12.0:1616513091.038699:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.195@tcp1 routed destination 00000400:00000200:12.0:1616513091.038714:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.195@tcp1(10.31.142.195@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.038720:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.038726:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.038729:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.038731:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.038736:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.038743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.038799:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.038807:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.038809:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.038816:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.038820:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f4b8 00000800:00000200:22.0:1616513091.038824:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.038827:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.044905:0:50650:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513091.044961:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.044971:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513091.044980:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.147@tcp1 : PUT - for me 00000400:00000200:10.0:1616513091.044989:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.147@tcp1 of length 224 into portal 26 MB=0x6039495b00b80 00000400:00000200:10.0:1616513091.044997:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.147@tcp1 of length 224/224 into md 0xf97cdcc9 [1] + 0 00000400:00000200:10.0:1616513091.045002:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42dd0 00000400:00000200:10.0:1616513091.045005:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513091.045008:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513091.045012:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42dd0 00000800:00000200:18.0:1616513091.045015:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.045015:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.045018:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513091.045020:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.045077:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692786561649536 00010000:00000200:0.0:1616513091.045088:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1692786561649536/t0(0) o400->f8da4790-330a-291f-ccf8-e98b976d4645@10.31.140.147@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.045103:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692786561649536, offset 192 00000400:00000200:0.0:1616513091.045109:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.147@tcp1 00000400:00000200:0.0:1616513091.045116:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.147@tcp1 routed destination 00000400:00000200:0.0:1616513091.045131:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.147@tcp1(10.31.140.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.045142:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.045147:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.045150:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.045152:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.045157:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.045164:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513091.045204:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.045260:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.045267:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.045270:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.045277:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.045282:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04110 00000800:00000200:28.0:1616513091.045286:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.045289:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.047196:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.047252:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.047262:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.047273:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.147@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.047281:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.147@tcp1 of length 224 into portal 12 MB=0x6039495b00cc0 00000400:00000200:16.0:1616513091.047288:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.147@tcp1 of length 224/224 into md 0x865be291 [1] + 24744 00000400:00000200:16.0:1616513091.047293:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.047296:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.047307:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.047307:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.047309:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513091.047312:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.047372:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692786561649856 00010000:00000200:12.0:1616513091.047384:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd0480 x1692786561649856/t0(0) o400->9657c4e3-c195-ac96-5e32-cd50408403db@10.31.140.147@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.047399:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692786561649856, offset 192 00000400:00000200:12.0:1616513091.047405:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.147@tcp1 00000400:00000200:12.0:1616513091.047412:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.147@tcp1 routed destination 00000400:00000200:12.0:1616513091.047427:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.147@tcp1(10.31.140.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.047446:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.047452:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.047455:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.047457:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.047462:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.047470:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.047528:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.047534:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.047537:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.047544:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.047548:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:22.0:1616513091.047552:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.047555:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.057981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.057999:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.058008:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.058017:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.169@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.058026:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.169@tcp1 of length 224 into portal 26 MB=0x5ffde1d4c89c0 00000400:00000200:28.0:1616513091.058033:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.169@tcp1 of length 224/224 into md 0xf97cdcd9 [1] + 0 00000400:00000200:28.0:1616513091.058044:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43188 00000400:00000200:28.0:1616513091.058047:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.058049:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513091.058053:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513091.058054:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43188 00000800:00000200:28.0:1616513091.058057:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.058060:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.058062:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.058117:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704322931136 00010000:00000200:0.0:1616513091.058129:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688704322931136/t0(0) o400->d42d8f14-c389-2d8b-fcc5-7d9fb5b5143f@10.31.132.169@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.058143:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704322931136, offset 192 00000400:00000200:0.0:1616513091.058150:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.169@tcp1 00000400:00000200:0.0:1616513091.058157:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.169@tcp1 routed destination 00000400:00000200:0.0:1616513091.058172:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.169@tcp1(10.31.132.169@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.058178:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.058184:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.058187:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.058189:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.058194:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.058199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.058200:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513091.058211:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.058218:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.058221:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.058227:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.058233:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05f58 00000800:00000200:24.0:1616513091.058236:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.058240:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.058828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.058839:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.058847:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513091.058849:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.058856:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.169@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.058867:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.169@tcp1 of length 224 into portal 12 MB=0x5ffde1d4c8ac0 00000400:00000200:20.0:1616513091.058873:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.169@tcp1 of length 224/224 into md 0x865be291 [1] + 24968 00000400:00000200:20.0:1616513091.058879:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.058881:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.058887:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.058890:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.058892:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.058954:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704322931392 00010000:00000200:12.0:1616513091.058967:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d8480 x1688704322931392/t0(0) o400->209da191-8d71-66b6-f773-abfb736d1878@10.31.132.169@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.058981:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704322931392, offset 192 00000400:00000200:12.0:1616513091.058987:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.169@tcp1 00000400:00000200:12.0:1616513091.058994:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.169@tcp1 routed destination 00000400:00000200:12.0:1616513091.059009:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.169@tcp1(10.31.132.169@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.059017:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.059023:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.059026:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.059028:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.059033:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.059040:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.059095:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.059102:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.059104:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.059113:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.059117:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fed0 00000800:00000200:18.0:1616513091.059121:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.059124:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.061355:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513091.061411:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.061420:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.061429:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.243@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.061458:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.243@tcp1 of length 224 into portal 26 MB=0x5fc531de65680 00000800:00000200:16.0:1616513091.061464:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513091.061465:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.243@tcp1 of length 224/224 into md 0xf97cdcf9 [1] + 0 00000400:00000200:24.0:1616513091.061470:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43dc0 00000400:00000200:24.0:1616513091.061473:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.061475:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:16.0:1616513091.061475:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.061480:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43dc0 00000800:00000200:24.0:1616513091.061483:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:16.0:1616513091.061484:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.243@tcp1 : PUT - for me 00000800:00000200:24.0:1616513091.061486:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513091.061488:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.061491:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513091.061492:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.243@tcp1 of length 224 into portal 12 MB=0x5fc531de658c0 00000400:00000200:16.0:1616513091.061499:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.243@tcp1 of length 224/224 into md 0x865be291 [1] + 25192 00000400:00000200:16.0:1616513091.061504:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.061506:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513091.061513:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.061516:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.061518:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.061553:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1684808797673088 00010000:00000200:0.0:1616513091.061563:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1684808797673088/t0(0) o400->c34c510d-232c-e050-e255-e48654b204c8@10.31.130.243@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.061577:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1684808797673088, offset 192 00000100:00000200:12.0:1616513091.061578:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1684808797673664 00000400:00000200:0.0:1616513091.061582:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.243@tcp1 00010000:00000200:12.0:1616513091.061589:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd7080 x1684808797673664/t0(0) o400->9472cd7e-3f5e-bb7d-220f-1a844926508c@10.31.130.243@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513091.061589:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.243@tcp1 routed destination 00000100:00000200:12.0:1616513091.061602:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1684808797673664, offset 192 00000400:00000200:0.0:1616513091.061603:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.243@tcp1(10.31.130.243@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513091.061608:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.243@tcp1 00000800:00000200:0.0:1616513091.061610:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513091.061615:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.243@tcp1 routed destination 00000800:00000200:0.0:1616513091.061622:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.061625:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.061628:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513091.061629:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.243@tcp1(10.31.130.243@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.061633:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513091.061635:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513091.061638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.061639:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513091.061641:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.061643:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.061645:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.061649:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.061665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513091.061694:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.061701:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.061704:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.061704:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.061710:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.061716:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:20.0:1616513091.061719:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513091.061722:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.061730:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.061732:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.061739:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.061743:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f320 00000800:00000200:22.0:1616513091.061747:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.061750:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:9.0:1616513091.066723:0:6494:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928336, portal 4 00000100:00000200:9.0:1616513091.066729:0:6494:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928336, offset 0 00000400:00000200:9.0:1616513091.066736:0:6494:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:9.0:1616513091.066742:0:6494:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:9.0:1616513091.066755:0:6494:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:9.0:1616513091.066762:0:6494:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:9.0:1616513091.066769:0:6494:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:9.0:1616513091.066772:0:6494:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:9.0:1616513091.066774:0:6494:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:9.0:1616513091.066779:0:6494:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513091.066825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513091.066844:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513091.066851:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513091.066854:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513091.066859:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513091.066861:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513091.066868:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cab101b00 x1693067725928336/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513091.066880:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35210 00000800:00000200:18.0:1616513091.066884:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513091.066996:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513091.067049:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513091.067059:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513091.067068:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513091.067075:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627790 00000400:00000200:22.0:1616513091.067082:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97ce389 [1] + 192 00000400:00000200:22.0:1616513091.067088:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.067093:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cab101b00 x1693067725928336/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513091.067102:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513091.067108:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513091.067115:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513091.067116:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:9.0:1616513091.067118:0:6494:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cab101b00 x1693067725928336/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:9.0:1616513091.067131:0:6494:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cab101b00 x1693067725928336/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:9.0:1616513091.067144:0:6494:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35430 00000800:00000200:7.2:1616513091.075091:0:50653:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513091.075146:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.075156:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513091.075165:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.44@tcp1 : PUT - for me 00000400:00000200:10.0:1616513091.075175:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.44@tcp1 of length 224 into portal 26 MB=0x60008636c4c00 00000400:00000200:10.0:1616513091.075181:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.44@tcp1 of length 224/224 into md 0xf97cdd09 [1] + 0 00000400:00000200:10.0:1616513091.075186:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42088 00000400:00000200:10.0:1616513091.075189:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513091.075191:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513091.075195:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42088 00000800:00000200:10.0:1616513091.075198:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.075201:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513091.075201:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513091.075205:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.075241:0:50653:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513091.075261:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688885888044032 00010000:00000200:0.0:1616513091.075272:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688885888044032/t0(0) o400->bb19e626-5a8d-7a7a-78f5-95d2072a8621@10.31.131.44@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.075286:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688885888044032, offset 192 00000400:00000200:0.0:1616513091.075293:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.44@tcp1 00000800:00000200:2.0:1616513091.075296:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513091.075300:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.44@tcp1 routed destination 00000800:00000200:2.0:1616513091.075305:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.075314:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.44@tcp1 : PUT - for me 00000400:00000200:0.0:1616513091.075316:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.44@tcp1(10.31.131.44@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513091.075323:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.44@tcp1 of length 224 into portal 12 MB=0x60008636c4c80 00000800:00000200:0.0:1616513091.075323:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.075329:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513091.075330:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.44@tcp1 of length 224/224 into md 0xf964d519 [1] + 91256 00000800:00000200:0.0:1616513091.075331:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.075333:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513091.075335:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.075338:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513091.075339:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.075345:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513091.075346:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.075349:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.075350:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.075355:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.075383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:14.0:1616513091.075419:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688885888044160 00010000:00000200:14.0:1616513091.075432:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5580 x1688885888044160/t0(0) o400->9da59bf9-d4db-1e16-4f4c-c7a4341ddb21@10.31.131.44@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513091.075457:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:14.0:1616513091.075461:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688885888044160, offset 192 00000800:00000200:28.0:1616513091.075463:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.075466:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513091.075467:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.44@tcp1 00000400:00000200:28.0:1616513091.075473:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.075474:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.44@tcp1 routed destination 00000400:00000200:28.0:1616513091.075478:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb056d8 00000800:00000200:28.0:1616513091.075482:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.075485:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513091.075489:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.44@tcp1(10.31.131.44@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513091.075496:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513091.075502:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513091.075506:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.075508:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.075513:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.075518:0:50653:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.075529:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.075535:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.075537:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513091.075539:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513091.075544:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.075548:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757100 00000800:00000200:18.0:1616513091.075552:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.079653:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513091.079684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513091.079707:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.079717:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513091.079725:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.142@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.079734:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.142@tcp1 of length 224 into portal 26 MB=0x5ffd7fe3a6540 00000800:00000200:24.0:1616513091.079740:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513091.079741:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.142@tcp1 of length 224/224 into md 0xf97cdd19 [1] + 0 00000400:00000200:22.0:1616513091.079745:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43980 00000800:00000200:24.0:1616513091.079748:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513091.079748:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.079751:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513091.079755:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43980 00000400:00000200:24.0:1616513091.079757:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.142@tcp1 : PUT - for me 00000800:00000200:22.0:1616513091.079758:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.079760:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.079762:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513091.079765:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.142@tcp1 of length 224 into portal 12 MB=0x5ffd7fe3a65c0 00000800:00000200:2.0:1616513091.079765:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513091.079773:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.142@tcp1 of length 224/224 into md 0x865be291 [1] + 25416 00000400:00000200:24.0:1616513091.079778:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.079780:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.079785:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.079788:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.079790:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.079793:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.079833:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678031844672 00010000:00000200:0.0:1616513091.079843:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688678031844672/t0(0) o400->1ca31d25-482f-e2d4-da0e-ea7eea4b664e@10.31.143.142@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513091.079855:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688678031844800 00000100:00000200:0.0:1616513091.079856:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688678031844672, offset 192 00000400:00000200:0.0:1616513091.079862:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.142@tcp1 00010000:00000200:12.0:1616513091.079866:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e0480 x1688678031844800/t0(0) o400->37aa717d-85fe-5726-a4e0-c5361141b752@10.31.143.142@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513091.079868:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.142@tcp1 routed destination 00000100:00000200:12.0:1616513091.079880:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688678031844800, offset 192 00000400:00000200:0.0:1616513091.079882:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.142@tcp1(10.31.143.142@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513091.079885:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.142@tcp1 00000800:00000200:0.0:1616513091.079889:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513091.079892:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.142@tcp1 routed destination 00000800:00000200:0.0:1616513091.079899:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.079901:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.079903:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513091.079906:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.142@tcp1(10.31.143.142@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.079908:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513091.079912:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513091.079913:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.079914:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513091.079917:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.079919:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.079921:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.079926:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.079930:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513091.079968:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.079975:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.079978:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.079980:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513091.079984:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:10.0:1616513091.079986:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513091.079990:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb044c8 00000800:00000200:28.0:1616513091.079993:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513091.079993:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.079996:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513091.080002:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513091.080006:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fb18 00000800:00000200:10.0:1616513091.080010:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.080013:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.088062:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.088118:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.088129:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.088138:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.45@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.088147:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.45@tcp1 of length 224 into portal 26 MB=0x5ffe3c97282c0 00000400:00000200:20.0:1616513091.088153:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.45@tcp1 of length 224/224 into md 0xf97cdd39 [1] + 0 00000400:00000200:20.0:1616513091.088158:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43298 00000400:00000200:20.0:1616513091.088161:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.088164:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.088168:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43298 00000800:00000200:20.0:1616513091.088171:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.088173:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.088174:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.088180:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.088183:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.088184:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.088192:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.45@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.088201:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.45@tcp1 of length 224 into portal 12 MB=0x5ffe3c9728600 00000400:00000200:24.0:1616513091.088207:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.45@tcp1 of length 224/224 into md 0x865be291 [1] + 25640 00000400:00000200:24.0:1616513091.088212:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.088215:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.088221:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.088224:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.088226:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.088233:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728685937344 00010000:00000200:0.0:1616513091.088244:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688728685937344/t0(0) o400->df860f56-e62c-e2e4-019c-3cd173cbf221@10.31.142.45@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.088260:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688728685937344, offset 192 00000400:00000200:0.0:1616513091.088266:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.45@tcp1 00000400:00000200:0.0:1616513091.088274:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.45@tcp1 routed destination 00000100:00000200:12.0:1616513091.088287:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728685938176 00000400:00000200:0.0:1616513091.088288:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.45@tcp1(10.31.142.45@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.088295:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513091.088299:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e4800 x1688728685938176/t0(0) o400->26d0b545-18d8-ac44-251e-2f53c6ff0ef8@10.31.142.45@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.088313:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.088316:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.088318:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.088322:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513091.088324:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688728685938176, offset 192 00000800:00000200:11.2:1616513091.088327:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513091.088329:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.45@tcp1 00000100:00000200:0.0:1616513091.088329:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513091.088335:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.45@tcp1 routed destination 00000800:00000200:28.0:1616513091.088340:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.088347:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513091.088348:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.45@tcp1(10.31.142.45@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513091.088349:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513091.088351:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513091.088354:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513091.088356:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:12.0:1616513091.088359:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.088361:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513091.088362:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04908 00000800:00000200:12.0:1616513091.088363:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513091.088365:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513091.088367:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.088374:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.088428:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.088451:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.088454:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.088460:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.088464:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e4c8 00000800:00000200:22.0:1616513091.088468:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.088471:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.091690:0:50655:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513091.091747:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.091756:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513091.091765:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.165@tcp1 : PUT - for me 00000400:00000200:10.0:1616513091.091774:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.165@tcp1 of length 224 into portal 26 MB=0x5ffdd31993f00 00000400:00000200:10.0:1616513091.091782:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.165@tcp1 of length 224/224 into md 0xf97cdd49 [1] + 0 00000400:00000200:10.0:1616513091.091787:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43760 00000400:00000200:10.0:1616513091.091790:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513091.091792:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513091.091797:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43760 00000800:00000200:10.0:1616513091.091800:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.091802:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.091802:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513091.091806:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.091862:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700368535296 00010000:00000200:0.0:1616513091.091873:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688700368535296/t0(0) o400->b052b36c-c1dc-de40-b970-f96dba500d9e@10.31.141.165@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.091888:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700368535296, offset 192 00000400:00000200:0.0:1616513091.091894:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.165@tcp1 00000400:00000200:0.0:1616513091.091914:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.165@tcp1 routed destination 00000400:00000200:0.0:1616513091.091928:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.165@tcp1(10.31.141.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.091934:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.091939:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.091942:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.091944:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:11.2:1616513091.091945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.091948:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.091954:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513091.092000:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.092009:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.092017:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.165@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.092025:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.165@tcp1 of length 224 into portal 12 MB=0x5ffdd31994180 00000400:00000200:24.0:1616513091.092031:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.165@tcp1 of length 224/224 into md 0x865be291 [1] + 25864 00000400:00000200:24.0:1616513091.092036:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.092038:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.092044:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513091.092047:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513091.092048:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.092051:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.092054:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.092056:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.092061:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.092065:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:24.0:1616513091.092069:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.092072:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.092102:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700368535936 00010000:00000200:12.0:1616513091.092113:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e5e80 x1688700368535936/t0(0) o400->516074db-d6f0-0110-266c-d3cf7ac18575@10.31.141.165@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.092126:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700368535936, offset 192 00000400:00000200:12.0:1616513091.092131:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.165@tcp1 00000400:00000200:12.0:1616513091.092137:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.165@tcp1 routed destination 00000400:00000200:12.0:1616513091.092150:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.165@tcp1(10.31.141.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.092156:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.092161:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.092163:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.092165:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.092170:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.092174:0:50655:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.092241:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.092247:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.092250:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.092257:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.092261:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f210 00000800:00000200:22.0:1616513091.092265:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.092268:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.098030:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.098086:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513091.098086:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513091.098095:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.098104:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.49@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.098112:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.49@tcp1 of length 224 into portal 26 MB=0x5ffdb8fefa580 00000400:00000200:28.0:1616513091.098119:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.49@tcp1 of length 224/224 into md 0xf97cdd69 [1] + 0 00000400:00000200:28.0:1616513091.098124:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43e48 00000400:00000200:28.0:1616513091.098126:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.098129:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513091.098134:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43e48 00000800:00000200:28.0:1616513091.098137:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.098139:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.098141:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513091.098141:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513091.098144:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513091.098150:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.098159:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.49@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.098168:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.49@tcp1 of length 224 into portal 12 MB=0x5ffdb8fefa800 00000400:00000200:18.0:1616513091.098175:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.49@tcp1 of length 224/224 into md 0xf964d519 [1] + 91480 00000400:00000200:18.0:1616513091.098180:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.098182:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513091.098189:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.098192:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.098194:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.098197:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.098214:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693361321344 00010000:00000200:0.0:1616513091.098225:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688693361321344/t0(0) o400->87f17ae4-7ec6-1f97-ecaa-2dae2f3b6175@10.31.141.49@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.098238:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693361321344, offset 192 00000400:00000200:0.0:1616513091.098244:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.49@tcp1 00000400:00000200:0.0:1616513091.098251:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.49@tcp1 routed destination 00000100:00000200:14.0:1616513091.098253:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693361321984 00010000:00000200:14.0:1616513091.098264:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7461209680 x1688693361321984/t0(0) o400->5c7da2bf-6da9-c4dc-bfec-1fa4596de832@10.31.141.49@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513091.098266:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.49@tcp1(10.31.141.49@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.098272:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:14.0:1616513091.098277:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693361321984, offset 192 00000800:00000200:0.0:1616513091.098278:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.098280:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.098282:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513091.098283:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.49@tcp1 00000800:00000200:0.0:1616513091.098287:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513091.098290:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.49@tcp1 routed destination 00000800:00000200:11.2:1616513091.098292:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.098293:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513091.098303:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.49@tcp1(10.31.141.49@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513091.098310:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513091.098315:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513091.098317:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.098319:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.098324:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.098328:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513091.098344:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.098350:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.098353:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.098359:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.098364:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05210 00000800:00000200:24.0:1616513091.098368:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.098370:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513091.098381:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.098388:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.098390:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.098397:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.098401:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47575c8 00000800:00000200:22.0:1616513091.098407:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.098410:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:29.0:1616513091.114701:0:6488:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928368, portal 4 00000100:00000200:25.0:1616513091.114703:0:6494:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928352, portal 4 00000100:00000200:1.0:1616513091.114704:0:6489:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928384, portal 4 00000100:00000200:29.0:1616513091.114707:0:6488:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928368, offset 0 00000100:00000200:25.0:1616513091.114708:0:6494:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928352, offset 0 00000100:00000200:1.0:1616513091.114710:0:6489:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928384, offset 0 00000400:00000200:29.0:1616513091.114714:0:6488:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:29.0:1616513091.114720:0:6488:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:25.0:1616513091.114720:0:6494:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:1.0:1616513091.114726:0:6489:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:25.0:1616513091.114727:0:6494:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:29.0:1616513091.114734:0:6488:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000400:00000200:1.0:1616513091.114735:0:6489:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:29.0:1616513091.114741:0:6488:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000400:00000200:25.0:1616513091.114741:0:6494:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:25.0:1616513091.114747:0:6494:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:1.0:1616513091.114748:0:6489:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:1.0:1616513091.114755:0:6489:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:29.0:1616513091.114756:0:6488:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:25.0:1616513091.114756:0:6494:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513091.114759:0:6488:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:25.0:1616513091.114759:0:6494:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:29.0:1616513091.114761:0:6488:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:1.0:1616513091.114761:0:6489:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:25.0:1616513091.114762:0:6494:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:1.0:1616513091.114764:0:6489:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:29.0:1616513091.114766:0:6488:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:1.0:1616513091.114766:0:6489:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:25.0:1616513091.114767:0:6494:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:1.0:1616513091.114771:0:6489:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:25.2:1616513091.114773:0:6494:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:1.2:1616513091.114778:0:6489:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.2:1616513091.114813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:10.0:1616513091.114829:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:5.0:1616513091.114831:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:10.0:1616513091.114837:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000800:00000200:5.0:1616513091.114839:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:10.0:1616513091.114840:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:5.0:1616513091.114842:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:9.0:1616513091.114845:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:10.0:1616513091.114847:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:5.0:1616513091.114849:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:10.0:1616513091.114855:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1395a00 x1693067725928352/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513091.114867:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492672330 00000800:00000200:28.0:1616513091.114869:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:10.0:1616513091.114872:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000100:00000200:5.0:1616513091.114873:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdd689f80 x1693067725928384/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:10.0:1616513091.114875:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:28.0:1616513091.114878:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513091.114881:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:5.0:1616513091.114885:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84e58 00000400:00000200:28.0:1616513091.114888:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:5.0:1616513091.114889:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:28.0:1616513091.114895:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4558ad1680 x1693067725928368/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513091.114908:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b3100 00000800:00000200:28.0:1616513091.114912:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513091.114915:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:25.2:1616513091.114986:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:5.2:1616513091.114986:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:1.2:1616513091.115001:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513091.115010:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513091.115017:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513091.115019:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513091.115025:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513091.115033:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6277c0 00000400:00000200:1.0:1616513091.115039:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x436fffc5 [1] + 192 00000800:00000200:22.0:1616513091.115040:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000400:00000200:1.0:1616513091.115044:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513091.115050:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000100:00000200:1.0:1616513091.115050:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdd689f80 x1693067725928384/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:24.0:1616513091.115058:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:22.0:1616513091.115059:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:1.0:1616513091.115064:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:24.0:1616513091.115067:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:22.0:1616513091.115067:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6277a0 00000800:00000200:1.0:1616513091.115067:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513091.115069:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:5.0:1616513091.115072:0:6489:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdd689f80 x1693067725928384/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513091.115075:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97ce409 [1] + 192 00000400:00000200:24.0:1616513091.115076:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513091.115080:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:5.0:1616513091.115083:0:6489:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdd689f80 x1693067725928384/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513091.115085:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6277b0 00000100:00000200:22.0:1616513091.115085:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1395a00 x1693067725928352/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513091.115092:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dac11 [1] + 192 00000800:00000200:2.0:1616513091.115094:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:5.0:1616513091.115095:0:6489:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84000 00000400:00000200:24.0:1616513091.115097:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513091.115101:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000100:00000200:24.0:1616513091.115102:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4558ad1680 x1693067725928368/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513091.115104:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513091.115105:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:25.0:1616513091.115110:0:6494:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1395a00 x1693067725928352/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513091.115116:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513091.115118:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513091.115120:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513091.115122:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:25.0:1616513091.115126:0:6494:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1395a00 x1693067725928352/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:29.0:1616513091.115127:0:6488:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4558ad1680 x1693067725928368/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:25.0:1616513091.115136:0:6494:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492672e58 00000100:00000200:29.0:1616513091.115140:0:6488:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4558ad1680 x1693067725928368/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:29.0:1616513091.115151:0:6488:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b23b8 00000100:00000200:21.0:1616513091.115500:0:6490:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928400, portal 4 00000100:00000200:21.0:1616513091.115506:0:6490:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928400, offset 0 00000400:00000200:21.0:1616513091.115513:0:6490:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:21.0:1616513091.115519:0:6490:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:21.0:1616513091.115532:0:6490:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:21.0:1616513091.115538:0:6490:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:21.0:1616513091.115544:0:6490:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513091.115549:0:6490:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:21.0:1616513091.115551:0:6490:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:21.0:1616513091.115557:0:6490:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513091.115603:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513091.115624:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513091.115630:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513091.115632:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513091.115639:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513091.115645:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdd68e780 x1693067725928400/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513091.115657:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf507f8 00000800:00000200:27.0:1616513091.115661:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513091.115664:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513091.115779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513091.115789:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513091.115795:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513091.115799:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513091.115804:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513091.115811:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6277d0 00000400:00000200:27.0:1616513091.115817:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371670bd [1] + 192 00000400:00000200:27.0:1616513091.115822:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513091.115827:0:6475:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdd68e780 x1693067725928400/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513091.115840:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513091.115843:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513091.115845:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:21.0:1616513091.115848:0:6490:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdd68e780 x1693067725928400/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:21.0:1616513091.115861:0:6490:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdd68e780 x1693067725928400/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:21.0:1616513091.115871:0:6490:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51298 00000800:00000200:7.2:1616513091.140318:0:50660:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513091.140374:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.140392:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513091.140402:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.76@tcp1 : PUT - for me 00000400:00000200:10.0:1616513091.140412:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.76@tcp1 of length 224 into portal 26 MB=0x5ffd4fd741000 00000400:00000200:10.0:1616513091.140419:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.76@tcp1 of length 224/224 into md 0xf97cdd89 [1] + 0 00000400:00000200:10.0:1616513091.140423:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a424c8 00000400:00000200:10.0:1616513091.140426:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513091.140429:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513091.140446:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a424c8 00000800:00000200:10.0:1616513091.140450:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.140453:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513091.140455:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.140458:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.140467:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.140476:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.76@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.140494:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.76@tcp1 of length 224 into portal 12 MB=0x5ffd4fd741080 00000400:00000200:18.0:1616513091.140501:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.76@tcp1 of length 224/224 into md 0xf964d519 [1] + 91704 00000100:00000200:0.0:1616513091.140502:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665133944832 00000400:00000200:18.0:1616513091.140506:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.140509:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:0.0:1616513091.140514:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688665133944832/t0(0) o400->d4d18cfc-8d52-4a91-625f-ea96e8ad1383@10.31.131.76@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513091.140517:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.140518:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.140521:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.140524:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.140528:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665133944832, offset 192 00000400:00000200:0.0:1616513091.140534:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.76@tcp1 00000400:00000200:0.0:1616513091.140541:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.76@tcp1 routed destination 00000400:00000200:0.0:1616513091.140556:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.76@tcp1(10.31.131.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.140562:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.140568:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.140570:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.140573:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.140578:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.140584:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:14.0:1616513091.140586:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665133944960 00000800:00000200:11.2:1616513091.140586:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:14.0:1616513091.140598:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd705e80 x1688665133944960/t0(0) o400->decf93a1-02c4-f999-0c33-41347478a7e2@10.31.131.76@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513091.140601:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.140607:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.140610:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.140612:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665133944960, offset 192 00000400:00000200:28.0:1616513091.140616:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.140618:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.76@tcp1 00000400:00000200:28.0:1616513091.140621:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb055c8 00000800:00000200:28.0:1616513091.140625:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513091.140625:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.76@tcp1 routed destination 00000800:00000200:28.0:1616513091.140627:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513091.140639:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.76@tcp1(10.31.131.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513091.140645:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513091.140650:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513091.140653:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.140655:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.140659:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.140664:0:50660:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.140677:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.140684:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.140687:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513091.140689:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513091.140694:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.140698:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757ed0 00000800:00000200:22.0:1616513091.140702:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.143592:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.143648:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.143658:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.143667:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.253@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.143675:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.253@tcp1 of length 224 into portal 26 MB=0x60194e25a4700 00000400:00000200:20.0:1616513091.143682:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.253@tcp1 of length 224/224 into md 0xf97cdd99 [1] + 0 00000400:00000200:20.0:1616513091.143687:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42220 00000400:00000200:20.0:1616513091.143690:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.143692:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.143697:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42220 00000800:00000200:20.0:1616513091.143699:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.143701:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.143702:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.143705:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.143760:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588824618752 00010000:00000200:0.0:1616513091.143771:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1690588824618752/t0(0) o400->7f842893-0f1b-1bfa-ac76-8ab51ebaed17@10.31.140.253@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.143785:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588824618752, offset 192 00000400:00000200:0.0:1616513091.143791:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.253@tcp1 00000400:00000200:0.0:1616513091.143798:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.253@tcp1 routed destination 00000400:00000200:0.0:1616513091.143814:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.253@tcp1(10.31.140.253@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.143820:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.143826:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.143830:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.143832:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.143837:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.143842:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.143844:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513091.143854:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.143861:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.143864:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.143870:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.143876:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04440 00000800:00000200:28.0:1616513091.143879:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.143882:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.143888:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513091.143890:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513091.143896:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.253@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.143904:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.253@tcp1 of length 224 into portal 12 MB=0x60194e25a49c0 00000400:00000200:28.0:1616513091.143910:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.253@tcp1 of length 224/224 into md 0x865be291 [1] + 26088 00000400:00000200:28.0:1616513091.143915:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.143917:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513091.143923:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.143926:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.143927:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.143984:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588824619456 00010000:00000200:12.0:1616513091.143995:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedd100 x1690588824619456/t0(0) o400->1921ac92-6cdf-b9e2-e175-71a168caa04d@10.31.140.253@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.144007:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588824619456, offset 192 00000400:00000200:12.0:1616513091.144012:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.253@tcp1 00000400:00000200:12.0:1616513091.144019:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.253@tcp1 routed destination 00000400:00000200:12.0:1616513091.144033:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.253@tcp1(10.31.140.253@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.144039:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.144043:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.144046:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.144048:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.144052:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.144058:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.144111:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.144118:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513091.144120:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513091.144127:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513091.144131:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ecc0 00000800:00000200:2.0:1616513091.144135:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.144138:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.156401:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.156465:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.156474:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.156484:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.26@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.156494:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.26@tcp1 of length 224 into portal 12 MB=0x5ffdcbda6e540 00000400:00000200:20.0:1616513091.156500:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.26@tcp1 of length 224/224 into md 0x865be291 [1] + 26312 00000400:00000200:20.0:1616513091.156506:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.156510:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.156517:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.156520:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513091.156520:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.156526:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.156529:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.156530:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.156539:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.26@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.156548:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.26@tcp1 of length 224 into portal 26 MB=0x5ffdcbda6e480 00000400:00000200:24.0:1616513091.156555:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.26@tcp1 of length 224/224 into md 0xf97cddc9 [1] + 0 00000400:00000200:24.0:1616513091.156560:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a438f8 00000400:00000200:24.0:1616513091.156562:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.156565:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513091.156570:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a438f8 00000800:00000200:24.0:1616513091.156573:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.156576:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.156580:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.156593:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698423272768 00010000:00000200:12.0:1616513091.156606:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0df500 x1688698423272768/t0(0) o400->f2084a60-bf5e-b4c4-a02d-3de619c60865@10.31.138.26@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.156620:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698423272768, offset 192 00000400:00000200:12.0:1616513091.156626:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.26@tcp1 00000400:00000200:12.0:1616513091.156634:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.26@tcp1 routed destination 00000100:00000200:0.0:1616513091.156634:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698423272576 00010000:00000200:0.0:1616513091.156645:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688698423272576/t0(0) o400->976ab216-d101-8745-8e36-6fdfb3b75c62@10.31.138.26@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513091.156649:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.26@tcp1(10.31.138.26@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.156656:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:0.0:1616513091.156659:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698423272576, offset 192 00000800:00000200:12.0:1616513091.156661:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.156664:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:0.0:1616513091.156665:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.26@tcp1 00000800:00000200:12.0:1616513091.156666:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513091.156671:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513091.156673:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.26@tcp1 routed destination 00000800:00000200:11.2:1616513091.156687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513091.156689:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.26@tcp1(10.31.138.26@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.156695:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513091.156699:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513091.156701:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.156704:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513091.156706:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513091.156706:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513091.156709:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513091.156710:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513091.156711:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513091.156718:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:0.0:1616513091.156718:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513091.156721:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e330 00000800:00000200:7.2:1616513091.156723:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513091.156725:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513091.156779:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.156786:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.156788:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513091.156796:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513091.156802:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:10.0:1616513091.156805:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.156808:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.163120:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513091.163175:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.163185:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.163194:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.114@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.163202:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.114@tcp1 of length 224 into portal 26 MB=0x5fdda427d5f40 00000400:00000200:2.0:1616513091.163209:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.114@tcp1 of length 224/224 into md 0xf97cde09 [1] + 0 00000400:00000200:2.0:1616513091.163213:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42ee0 00000400:00000200:2.0:1616513091.163216:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.163219:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513091.163223:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ee0 00000800:00000200:2.0:1616513091.163226:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.163229:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.163229:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.163233:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.163287:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1686488743763776 00010000:00000200:0.0:1616513091.163298:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1686488743763776/t0(0) o400->7da6d211-56ab-4d46-a5e6-f75eccfb131d@10.31.143.114@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.163312:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1686488743763776, offset 192 00000400:00000200:0.0:1616513091.163318:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.114@tcp1 00000400:00000200:0.0:1616513091.163326:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.114@tcp1 routed destination 00000800:00000200:11.2:1616513091.163340:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:0.0:1616513091.163341:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.114@tcp1(10.31.143.114@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.163347:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.163353:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.163356:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.163358:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513091.163363:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513091.163369:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513091.163396:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.163405:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.163414:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.114@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.163423:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.114@tcp1 of length 224 into portal 12 MB=0x5fdda427d6000 00000400:00000200:16.0:1616513091.163429:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.114@tcp1 of length 224/224 into md 0x865be291 [1] + 26536 00000400:00000200:16.0:1616513091.163450:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513091.163451:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:16.0:1616513091.163453:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.163459:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513091.163459:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000400:00000200:24.0:1616513091.163462:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.163463:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513091.163465:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513091.163467:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513091.163468:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.163474:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb058f8 00000800:00000200:24.0:1616513091.163478:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.163539:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1686488743763968 00010000:00000200:12.0:1616513091.163552:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd7980 x1686488743763968/t0(0) o400->c8779211-709d-8e31-b6ff-0171f33c28b5@10.31.143.114@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.163566:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1686488743763968, offset 192 00000400:00000200:12.0:1616513091.163572:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.114@tcp1 00000400:00000200:12.0:1616513091.163579:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.114@tcp1 routed destination 00000400:00000200:12.0:1616513091.163594:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.114@tcp1(10.31.143.114@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.163601:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.163607:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.163610:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.163614:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.163618:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.163623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.163676:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.163683:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.163694:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513091.163700:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513091.163704:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eaa0 00000800:00000200:10.0:1616513091.163708:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.163711:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.170534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513091.170586:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.170595:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.170603:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.8@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.170611:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.8@tcp1 of length 224 into portal 26 MB=0x5fffc7af1b8c0 00000400:00000200:18.0:1616513091.170617:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.8@tcp1 of length 224/224 into md 0xf97cde39 [1] + 0 00000400:00000200:18.0:1616513091.170622:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43870 00000400:00000200:18.0:1616513091.170625:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.170627:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513091.170631:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43870 00000800:00000200:18.0:1616513091.170634:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.170636:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.170638:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.170640:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.170694:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688834743056576 00010000:00000200:0.0:1616513091.170704:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688834743056576/t0(0) o400->4fb0f8f5-7530-9369-387a-758fb17aec51@10.31.142.8@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.170717:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688834743056576, offset 192 00000400:00000200:0.0:1616513091.170723:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.8@tcp1 00000400:00000200:0.0:1616513091.170729:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.8@tcp1 routed destination 00000400:00000200:0.0:1616513091.170743:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.8@tcp1(10.31.142.8@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.170749:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.170755:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.170757:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.170759:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.170764:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.170770:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513091.170776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.170791:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.170800:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.170809:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.8@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.170816:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.8@tcp1 of length 224 into portal 12 MB=0x5fffc7af1bb00 00000400:00000200:28.0:1616513091.170823:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.8@tcp1 of length 224/224 into md 0x865be291 [1] + 26760 00000400:00000200:28.0:1616513091.170828:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.170830:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513091.170836:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513091.170839:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513091.170840:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.170842:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.170847:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.170849:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.170849:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513091.170854:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.170858:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a90 00000800:00000200:28.0:1616513091.170860:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.170897:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688834743057152 00010000:00000200:12.0:1616513091.170908:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedda00 x1688834743057152/t0(0) o400->b9d0f8b2-da7d-ff03-0c97-6db799ef4ab6@10.31.142.8@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.170921:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688834743057152, offset 192 00000400:00000200:12.0:1616513091.170927:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.8@tcp1 00000400:00000200:12.0:1616513091.170934:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.8@tcp1 routed destination 00000400:00000200:12.0:1616513091.170947:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.8@tcp1(10.31.142.8@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.170953:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.170958:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.170961:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.170963:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.170967:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.170973:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.170982:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.170988:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.170991:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513091.170997:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513091.171001:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fdc0 00000800:00000200:10.0:1616513091.171004:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.171007:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.180312:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513091.180365:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.180374:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.180383:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.94@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.180391:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.94@tcp1 of length 224 into portal 26 MB=0x5ffdd2508ce00 00000400:00000200:18.0:1616513091.180397:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.94@tcp1 of length 224/224 into md 0xf97cde49 [1] + 0 00000400:00000200:18.0:1616513091.180402:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a422a8 00000400:00000200:18.0:1616513091.180404:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.180407:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513091.180411:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a422a8 00000800:00000200:18.0:1616513091.180413:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.180416:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.180417:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.180422:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.180472:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700157742592 00010000:00000200:0.0:1616513091.180482:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688700157742592/t0(0) o400->00fbe7ef-8e9a-c257-4dca-7050379923ed@10.31.128.94@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.180496:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700157742592, offset 192 00000400:00000200:0.0:1616513091.180501:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.94@tcp1 00000400:00000200:0.0:1616513091.180508:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.94@tcp1 routed destination 00000400:00000200:0.0:1616513091.180522:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.94@tcp1(10.31.128.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.180528:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.180533:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.180536:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.180538:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.180542:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.180548:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513091.180550:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.180609:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.180616:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.180619:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.180625:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.180631:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05188 00000800:00000200:20.0:1616513091.180634:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.180637:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.180722:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.180735:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.180754:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.180763:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.94@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.180771:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.94@tcp1 of length 224 into portal 12 MB=0x5ffdd2508d200 00000400:00000200:28.0:1616513091.180777:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.94@tcp1 of length 224/224 into md 0x865be291 [1] + 26984 00000400:00000200:28.0:1616513091.180783:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.180785:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513091.180792:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.180795:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.180797:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.180799:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.180853:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700157743616 00010000:00000200:12.0:1616513091.180864:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed9b00 x1688700157743616/t0(0) o400->8d0b429b-47e0-54a8-3ba9-4c8789183dce@10.31.128.94@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.180877:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700157743616, offset 192 00000400:00000200:12.0:1616513091.180883:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.94@tcp1 00000400:00000200:12.0:1616513091.180890:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.94@tcp1 routed destination 00000400:00000200:12.0:1616513091.180904:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.94@tcp1(10.31.128.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.180910:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.180915:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.180918:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.180920:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.180924:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.180929:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.180938:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.180943:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.180946:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513091.180952:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513091.180955:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f870 00000800:00000200:10.0:1616513091.180958:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.180961:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.184916:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.184973:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.184982:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.184991:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.62@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.184999:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.62@tcp1 of length 224 into portal 26 MB=0x5f924360fa500 00000400:00000200:20.0:1616513091.185006:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.62@tcp1 of length 224/224 into md 0xf97cde59 [1] + 0 00000400:00000200:20.0:1616513091.185011:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43a90 00000400:00000200:20.0:1616513091.185014:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.185016:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.185021:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a90 00000800:00000200:20.0:1616513091.185024:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.185025:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.185027:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.185033:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.185036:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.185036:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.185044:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.62@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.185052:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.62@tcp1 of length 224 into portal 12 MB=0x5f924360fa580 00000400:00000200:16.0:1616513091.185058:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.62@tcp1 of length 224/224 into md 0x865be291 [1] + 27208 00000400:00000200:16.0:1616513091.185063:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.185065:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513091.185071:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.185074:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.185076:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.185083:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681308804687104 00010000:00000200:0.0:1616513091.185093:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1681308804687104/t0(0) o400->dd7c48ee-b9e9-80a9-a6aa-6a6d59f106ac@10.31.131.62@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.185107:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681308804687104, offset 192 00000400:00000200:0.0:1616513091.185114:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.62@tcp1 00000400:00000200:0.0:1616513091.185121:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.62@tcp1 routed destination 00000100:00000200:12.0:1616513091.185145:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681308804687232 00000400:00000200:0.0:1616513091.185147:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.62@tcp1(10.31.131.62@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.185153:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513091.185156:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd3180 x1681308804687232/t0(0) o400->7a8adf45-d540-215f-69ac-b23a5f8a3536@10.31.131.62@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.185159:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.185161:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.185163:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.185168:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513091.185170:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681308804687232, offset 192 00000800:00000200:11.2:1616513091.185173:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.185174:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513091.185178:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.62@tcp1 00000400:00000200:12.0:1616513091.185185:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.62@tcp1 routed destination 00000400:00000200:12.0:1616513091.185199:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.62@tcp1(10.31.131.62@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.185204:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.185210:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.185212:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.185214:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.185219:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.185224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513091.185226:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.185233:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.185236:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513091.185238:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513091.185243:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.185248:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb043b8 00000800:00000200:24.0:1616513091.185252:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513091.185274:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.185281:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.185283:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.185289:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.185293:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e2a8 00000800:00000200:18.0:1616513091.185297:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.185300:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.194733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.194789:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.194798:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.194808:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.10@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.194818:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.10@tcp1 of length 224 into portal 26 MB=0x603e394673b00 00000400:00000200:16.0:1616513091.194825:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.10@tcp1 of length 224/224 into md 0xf97cde99 [1] + 0 00000400:00000200:16.0:1616513091.194830:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a437e8 00000400:00000200:16.0:1616513091.194832:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.194835:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513091.194840:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a437e8 00000800:00000200:28.0:1616513091.194843:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.194843:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.194846:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513091.194847:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.194899:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693125842516736 00010000:00000200:0.0:1616513091.194909:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1693125842516736/t0(0) o400->eef560ac-7b3c-d8b6-7ba0-65478c748280@10.31.130.10@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.194921:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693125842516736, offset 192 00000400:00000200:0.0:1616513091.194927:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.10@tcp1 00000400:00000200:0.0:1616513091.194934:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.10@tcp1 routed destination 00000400:00000200:0.0:1616513091.194947:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.10@tcp1(10.31.130.10@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.194952:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.194957:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.194959:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.194961:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.194965:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.194971:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513091.194976:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.195029:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.195036:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.195039:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.195045:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.195051:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04000 00000800:00000200:20.0:1616513091.195054:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.195057:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.195087:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.195098:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.195106:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513091.195108:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513091.195116:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.10@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.195127:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.10@tcp1 of length 224 into portal 12 MB=0x603e394674100 00000400:00000200:28.0:1616513091.195133:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.10@tcp1 of length 224/224 into md 0x865be291 [1] + 27432 00000400:00000200:28.0:1616513091.195138:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.195141:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513091.195147:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.195150:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.195152:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.195206:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693125842518272 00010000:00000200:12.0:1616513091.195216:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedf980 x1693125842518272/t0(0) o400->d427f57d-e60b-7763-86ba-9be39c590533@10.31.130.10@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.195228:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693125842518272, offset 192 00000400:00000200:12.0:1616513091.195234:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.10@tcp1 00000400:00000200:12.0:1616513091.195241:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.10@tcp1 routed destination 00000400:00000200:12.0:1616513091.195254:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.10@tcp1(10.31.130.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.195260:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.195265:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.195267:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.195269:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.195273:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.195320:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.195337:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.195342:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.195345:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513091.195351:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513091.195354:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f6d8 00000800:00000200:10.0:1616513091.195358:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.195361:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.195847:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513091.195904:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.195914:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.195922:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.45@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.195932:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.45@tcp1 of length 224 into portal 26 MB=0x60182f1073ac0 00000400:00000200:18.0:1616513091.195939:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.45@tcp1 of length 224/224 into md 0xf97cdea9 [1] + 0 00000400:00000200:18.0:1616513091.195944:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43b18 00000400:00000200:18.0:1616513091.195947:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.195950:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513091.195954:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43b18 00000800:00000200:18.0:1616513091.195957:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.195957:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.195960:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.195961:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.195966:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511761423040 00010000:00000200:0.0:1616513091.195971:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1690511761423040/t0(0) o400->071419e8-95bf-c81d-13ba-bf985df6fd81@10.31.136.45@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.195981:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511761423040, offset 192 00000400:00000200:0.0:1616513091.195985:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.45@tcp1 00000400:00000200:0.0:1616513091.195991:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.45@tcp1 routed destination 00000400:00000200:0.0:1616513091.196002:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.45@tcp1(10.31.136.45@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.196007:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.196012:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.196014:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.196016:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.196018:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.196022:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513091.196024:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.196033:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.196036:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.196038:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513091.196042:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.196043:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.196046:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05430 00000800:00000200:20.0:1616513091.196049:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.196272:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.196281:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.196285:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.196292:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.45@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.196298:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.45@tcp1 of length 224 into portal 12 MB=0x60182f1073c40 00000400:00000200:16.0:1616513091.196303:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.45@tcp1 of length 224/224 into md 0x865be291 [1] + 27656 00000400:00000200:16.0:1616513091.196307:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.196309:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513091.196314:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.196316:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513091.196317:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.196321:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.196326:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511761423424 00010000:00000200:12.0:1616513091.196331:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd6c00 x1690511761423424/t0(0) o400->23eec0cc-93d7-b28d-ba58-70beb84b1d03@10.31.136.45@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.196341:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511761423424, offset 192 00000400:00000200:12.0:1616513091.196346:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.45@tcp1 00000400:00000200:12.0:1616513091.196351:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.45@tcp1 routed destination 00000400:00000200:12.0:1616513091.196362:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.45@tcp1(10.31.136.45@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.196367:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.196371:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.196375:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.196376:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.196379:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.196385:0:50666:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.196396:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.196400:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.196402:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513091.196406:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513091.196409:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513091.196412:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f980 00000800:00000200:10.0:1616513091.196415:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.217829:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513091.217885:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.217894:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.217903:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.11@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.217911:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.11@tcp1 of length 224 into portal 26 MB=0x5fffe1aa44980 00000400:00000200:24.0:1616513091.217918:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.11@tcp1 of length 224/224 into md 0xf97cdeb9 [1] + 0 00000400:00000200:24.0:1616513091.217923:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42440 00000400:00000200:24.0:1616513091.217926:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.217929:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513091.217933:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42440 00000800:00000200:24.0:1616513091.217937:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.217937:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.217940:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.217941:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.217991:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688841717303680 00010000:00000200:0.0:1616513091.218001:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688841717303680/t0(0) o400->7202d6ca-abba-917c-64bb-3ad83c1b1e76@10.31.142.11@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.218012:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688841717303680, offset 192 00000400:00000200:0.0:1616513091.218017:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.11@tcp1 00000400:00000200:0.0:1616513091.218023:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.11@tcp1 routed destination 00000400:00000200:0.0:1616513091.218035:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.11@tcp1(10.31.142.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.218041:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.218046:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.218048:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.218050:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.218053:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.218059:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.218059:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513091.218067:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513091.218112:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.218119:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513091.218121:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513091.218128:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513091.218133:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05c28 00000800:00000200:2.0:1616513091.218133:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513091.218136:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.218139:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513091.218142:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.218151:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.11@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.218160:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.11@tcp1 of length 224 into portal 12 MB=0x5fffe1aa44d00 00000400:00000200:2.0:1616513091.218167:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.11@tcp1 of length 224/224 into md 0xf964d519 [1] + 91928 00000400:00000200:2.0:1616513091.218172:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.218175:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513091.218182:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.218185:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.218187:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.218190:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513091.218255:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688841717304576 00010000:00000200:14.0:1616513091.218266:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7500 x1688841717304576/t0(0) o400->030f7771-6ed4-0d31-72f6-d60b4b9d97be@10.31.142.11@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513091.218290:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688841717304576, offset 192 00000400:00000200:14.0:1616513091.218296:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.11@tcp1 00000400:00000200:14.0:1616513091.218302:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.11@tcp1 routed destination 00000400:00000200:14.0:1616513091.218316:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.11@tcp1(10.31.142.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513091.218322:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513091.218327:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513091.218330:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.218332:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.218337:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.218348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.218404:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.218411:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.218414:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513091.218415:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513091.218421:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.218425:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756dd0 00000800:00000200:22.0:1616513091.218428:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.246979:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513091.247005:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.247014:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513091.247018:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513091.247024:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.214@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.247034:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.214@tcp1 of length 224 into portal 26 MB=0x5ffde49481780 00000400:00000200:18.0:1616513091.247041:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.214@tcp1 of length 224/224 into md 0xf97cded9 [1] + 0 00000400:00000200:18.0:1616513091.247045:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a426e8 00000400:00000200:18.0:1616513091.247048:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.247051:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513091.247056:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a426e8 00000800:00000200:18.0:1616513091.247059:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.247061:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.247063:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.247123:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705060837248 00010000:00000200:0.0:1616513091.247134:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688705060837248/t0(0) o400->9748b849-5c4b-a05b-a603-76b75a95d0b8@10.31.128.214@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.247149:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705060837248, offset 192 00000400:00000200:0.0:1616513091.247155:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.214@tcp1 00000400:00000200:0.0:1616513091.247163:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.214@tcp1 routed destination 00000400:00000200:0.0:1616513091.247178:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.214@tcp1(10.31.128.214@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.247184:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.247190:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.247193:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.247195:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.247200:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.247208:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.247209:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513091.247262:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.247270:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.247273:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.247279:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.247285:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000800:00000200:20.0:1616513091.247288:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.247291:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.247297:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.247305:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.214@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.247313:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.214@tcp1 of length 224 into portal 12 MB=0x5ffde49481b40 00000800:00000200:24.0:1616513091.247317:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.247320:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.214@tcp1 of length 224/224 into md 0x865be291 [1] + 27880 00000400:00000200:20.0:1616513091.247325:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.247327:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.247333:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.247336:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.247338:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.247395:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705060838208 00010000:00000200:12.0:1616513091.247406:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0db600 x1688705060838208/t0(0) o400->069cea95-d59e-c14d-b46c-da43e70daf46@10.31.128.214@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.247419:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705060838208, offset 192 00000400:00000200:12.0:1616513091.247425:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.214@tcp1 00000400:00000200:12.0:1616513091.247431:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.214@tcp1 routed destination 00000400:00000200:12.0:1616513091.247457:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.214@tcp1(10.31.128.214@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.247463:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.247468:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.247471:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.247473:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.247477:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.247483:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.247495:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.247501:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.247504:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513091.247510:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513091.247514:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e908 00000800:00000200:10.0:1616513091.247518:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.247520:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.271354:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.271417:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.271427:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.271449:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.189@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.271459:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.189@tcp1 of length 224 into portal 26 MB=0x604a2ab52ee40 00000400:00000200:28.0:1616513091.271466:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.189@tcp1 of length 224/224 into md 0xf97cdf09 [1] + 0 00000400:00000200:28.0:1616513091.271470:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42110 00000800:00000200:24.0:1616513091.271471:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513091.271473:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.271476:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513091.271481:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42110 00000800:00000200:24.0:1616513091.271482:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513091.271484:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513091.271487:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513091.271488:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.271491:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513091.271491:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.189@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.271499:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.189@tcp1 of length 224 into portal 12 MB=0x604a2ab52eec0 00000400:00000200:24.0:1616513091.271506:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.189@tcp1 of length 224/224 into md 0x865be291 [1] + 28104 00000400:00000200:24.0:1616513091.271516:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.271519:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.271525:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.271528:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.271530:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.271560:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946565815872 00010000:00000200:0.0:1616513091.271571:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1693946565815872/t0(0) o400->90a497c6-15f2-bcff-6931-5c6b91746641@10.31.140.189@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.271586:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946565815872, offset 192 00000100:00000200:12.0:1616513091.271590:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946565816000 00000400:00000200:0.0:1616513091.271592:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.189@tcp1 00000400:00000200:0.0:1616513091.271600:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.189@tcp1 routed destination 00010000:00000200:12.0:1616513091.271603:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e1b00 x1693946565816000/t0(0) o400->1079e2e3-1a1b-69bb-7509-30d309c79d64@10.31.140.189@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513091.271615:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.189@tcp1(10.31.140.189@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:12.0:1616513091.271617:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946565816000, offset 192 00000800:00000200:0.0:1616513091.271622:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513091.271623:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.189@tcp1 00000800:00000200:0.0:1616513091.271628:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.271630:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:12.0:1616513091.271631:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.189@tcp1 routed destination 00000800:00000200:0.0:1616513091.271633:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.271637:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.271643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.271644:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513091.271645:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.189@tcp1(10.31.140.189@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.271652:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.271658:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.271660:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.271663:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.271667:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.271676:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.271689:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.271694:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513091.271696:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.271697:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513091.271702:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513091.271704:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513091.271706:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ef68 00000400:00000200:16.0:1616513091.271707:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513091.271708:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513091.271710:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.271712:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513091.271713:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513091.271719:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000800:00000200:16.0:1616513091.271722:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.284046:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513091.284056:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.284063:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513091.284070:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.144@tcp1 : PUT - for me 00000400:00000200:10.0:1616513091.284078:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.144@tcp1 of length 224 into portal 26 MB=0x603fcdda072c0 00000400:00000200:10.0:1616513091.284083:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.144@tcp1 of length 224/224 into md 0xf97cdf19 [1] + 0 00000400:00000200:10.0:1616513091.284087:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42880 00000400:00000200:10.0:1616513091.284089:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513091.284091:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513091.284094:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42880 00000800:00000200:10.0:1616513091.284096:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.284098:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513091.284099:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.284101:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.284159:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234445185728 00010000:00000200:0.0:1616513091.284170:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1693234445185728/t0(0) o400->98c57aab-4822-3fa1-9eb8-ede743fa804a@10.31.138.144@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.284185:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234445185728, offset 192 00000400:00000200:0.0:1616513091.284191:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.144@tcp1 00000400:00000200:0.0:1616513091.284199:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.144@tcp1 routed destination 00000400:00000200:0.0:1616513091.284214:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.144@tcp1(10.31.138.144@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.284220:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.284226:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.284229:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.284231:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.284235:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.284242:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513091.284294:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.284350:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.284357:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.284360:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.284366:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.284372:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05430 00000800:00000200:20.0:1616513091.284375:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.284378:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.284384:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.284392:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.144@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.284400:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.144@tcp1 of length 224 into portal 12 MB=0x603fcdda07380 00000800:00000200:24.0:1616513091.284403:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.284406:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.144@tcp1 of length 224/224 into md 0x865be291 [1] + 28328 00000400:00000200:20.0:1616513091.284413:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.284415:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.284421:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.284424:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.284426:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.284485:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234445185920 00010000:00000200:12.0:1616513091.284498:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dd580 x1693234445185920/t0(0) o400->be67e0c6-d5e0-3e96-06a0-2dbabba9a232@10.31.138.144@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.284533:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234445185920, offset 192 00000400:00000200:12.0:1616513091.284539:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.144@tcp1 00000400:00000200:12.0:1616513091.284546:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.144@tcp1 routed destination 00000400:00000200:12.0:1616513091.284561:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.144@tcp1(10.31.138.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.284568:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.284573:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.284576:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.284578:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.284583:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.284587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.284596:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.284600:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513091.284601:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513091.284607:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513091.284609:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e220 00000800:00000200:2.0:1616513091.284612:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.284614:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.291375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.291432:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.291456:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.291465:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.119@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.291475:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.119@tcp1 of length 224 into portal 26 MB=0x5f914b6dac380 00000400:00000200:16.0:1616513091.291481:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.119@tcp1 of length 224/224 into md 0xf97cdf39 [1] + 0 00000400:00000200:16.0:1616513091.291486:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43100 00000400:00000200:16.0:1616513091.291489:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.291492:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513091.291497:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43100 00000800:00000200:16.0:1616513091.291500:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.291502:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.291503:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513091.291509:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.291512:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513091.291512:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513091.291521:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.119@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.291530:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.119@tcp1 of length 224 into portal 12 MB=0x5f914b6dac400 00000400:00000200:24.0:1616513091.291536:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.119@tcp1 of length 224/224 into md 0x865be291 [1] + 28552 00000100:00000200:0.0:1616513091.291538:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681242246005632 00000400:00000200:24.0:1616513091.291542:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.291544:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:0.0:1616513091.291548:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1681242246005632/t0(0) o400->5448e928-bc47-748e-6ea1-c3252e7d29ad@10.31.143.119@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513091.291551:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.291554:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.291556:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.291562:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681242246005632, offset 192 00000400:00000200:0.0:1616513091.291569:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.119@tcp1 00000400:00000200:0.0:1616513091.291576:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.119@tcp1 routed destination 00000400:00000200:0.0:1616513091.291591:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.119@tcp1(10.31.143.119@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.291597:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.291602:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.291605:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.291607:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.291611:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513091.291612:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681242246005760 00000100:00000200:0.0:1616513091.291617:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:12.0:1616513091.291623:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e1680 x1681242246005760/t0(0) o400->d7e2f68e-c909-4a63-e74b-89d4783e9f70@10.31.143.119@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513091.291628:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513091.291636:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681242246005760, offset 192 00000400:00000200:12.0:1616513091.291642:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.119@tcp1 00000400:00000200:12.0:1616513091.291649:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.119@tcp1 routed destination 00000400:00000200:12.0:1616513091.291663:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.119@tcp1(10.31.143.119@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.291669:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.291674:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.291677:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.291679:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.291683:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513091.291685:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513091.291689:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513091.291692:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.291695:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.291696:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513091.291698:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513091.291702:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:10.0:1616513091.291702:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.291704:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.291707:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb043b8 00000400:00000200:10.0:1616513091.291709:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513091.291711:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513091.291712:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fc28 00000800:00000200:10.0:1616513091.291714:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513091.291716:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.291964:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513091.292016:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.292026:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.292035:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.143@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.292045:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.143@tcp1 of length 224 into portal 26 MB=0x5ffd400661a40 00000400:00000200:2.0:1616513091.292051:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.143@tcp1 of length 224/224 into md 0xf97cdf49 [1] + 0 00000400:00000200:2.0:1616513091.292057:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42e58 00000400:00000200:2.0:1616513091.292059:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.292062:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513091.292067:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42e58 00000800:00000200:2.0:1616513091.292070:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.292073:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.292075:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.292077:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.292080:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660888394304 00010000:00000200:0.0:1616513091.292085:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688660888394304/t0(0) o400->95323e88-88d1-1aef-9ba4-feef67d39ee3@10.31.129.143@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.292095:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660888394304, offset 192 00000400:00000200:0.0:1616513091.292100:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.143@tcp1 00000400:00000200:0.0:1616513091.292105:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.143@tcp1 routed destination 00000400:00000200:0.0:1616513091.292117:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.143@tcp1(10.31.129.143@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.292123:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.292127:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.292129:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.292131:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.292133:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.292137:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513091.292139:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.292151:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.292158:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.292160:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513091.292162:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.292167:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.292173:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a90 00000800:00000200:20.0:1616513091.292176:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.292418:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.292427:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.292439:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513091.292444:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513091.292446:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.143@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.292453:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.143@tcp1 of length 224 into portal 12 MB=0x5ffd400661ec0 00000400:00000200:16.0:1616513091.292458:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.143@tcp1 of length 224/224 into md 0x865be291 [1] + 28776 00000400:00000200:16.0:1616513091.292464:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.292467:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513091.292472:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.292474:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.292476:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.292484:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660888395456 00010000:00000200:12.0:1616513091.292490:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd2400 x1688660888395456/t0(0) o400->fe71adfe-c998-2ced-bea1-b8300df64c9f@10.31.129.143@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.292500:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660888395456, offset 192 00000400:00000200:12.0:1616513091.292505:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.143@tcp1 00000400:00000200:12.0:1616513091.292510:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.143@tcp1 routed destination 00000400:00000200:12.0:1616513091.292523:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.143@tcp1(10.31.129.143@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.292529:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.292534:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.292536:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.292537:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.292540:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.292545:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513091.292552:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513091.292554:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513091.292556:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513091.292560:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513091.292561:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513091.292562:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e990 00000800:00000200:10.0:1616513091.292565:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.320731:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513091.320749:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513091.320786:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.320796:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.320805:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.182@tcp1 : PUT - for me 00000800:00000200:24.0:1616513091.320806:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513091.320814:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.182@tcp1 of length 224 into portal 26 MB=0x602196858e500 00000800:00000200:24.0:1616513091.320816:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513091.320821:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.182@tcp1 of length 224/224 into md 0xf97cdf79 [1] + 0 00000400:00000200:24.0:1616513091.320824:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.182@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.320828:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43a08 00000400:00000200:2.0:1616513091.320831:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.320833:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.182@tcp1 of length 224 into portal 12 MB=0x602196858e5c0 00000100:00000200:2.0:1616513091.320834:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513091.320838:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513091.320838:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43a08 00000400:00000200:24.0:1616513091.320840:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.182@tcp1 of length 224/224 into md 0x865be291 [1] + 29000 00000800:00000200:2.0:1616513091.320841:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.320844:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513091.320845:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513091.320845:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513091.320848:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.320854:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.320857:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.320859:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.320862:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.320917:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691158008358144 00000100:00000200:12.0:1616513091.320918:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691158008358336 00010000:00000200:0.0:1616513091.320928:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1691158008358144/t0(0) o400->bed933f2-e71a-560d-232a-6a55737fcc96@10.31.140.182@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:12.0:1616513091.320930:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e7500 x1691158008358336/t0(0) o400->b1bbb45e-853e-c974-8243-2f76df63ff59@10.31.140.182@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513091.320941:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691158008358144, offset 192 00000100:00000200:12.0:1616513091.320943:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691158008358336, offset 192 00000400:00000200:0.0:1616513091.320947:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.182@tcp1 00000400:00000200:12.0:1616513091.320952:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.182@tcp1 00000400:00000200:0.0:1616513091.320954:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.182@tcp1 routed destination 00000400:00000200:0.0:1616513091.320968:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.182@tcp1(10.31.140.182@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.320979:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513091.320981:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.182@tcp1 routed destination 00000800:00000200:0.0:1616513091.320988:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.320990:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.320992:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513091.320995:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.182@tcp1(10.31.140.182@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.320997:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513091.321001:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513091.321002:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.321003:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513091.321006:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.321009:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.321011:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.321015:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.321021:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513091.321056:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.321063:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.321066:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.321073:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.321078:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:22.0:1616513091.321080:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513091.321084:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.321087:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513091.321088:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.321091:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.321098:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.321102:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f760 00000800:00000200:22.0:1616513091.321106:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.321109:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.325758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513091.325814:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.325823:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.325832:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.156@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.325841:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.156@tcp1 of length 224 into portal 26 MB=0x603e14cd16180 00000400:00000200:24.0:1616513091.325848:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.156@tcp1 of length 224/224 into md 0xf97cdf89 [1] + 0 00000400:00000200:24.0:1616513091.325854:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42550 00000400:00000200:24.0:1616513091.325857:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.325860:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513091.325864:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42550 00000800:00000200:24.0:1616513091.325867:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.325867:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.325870:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.325874:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.325877:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.325878:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.325886:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.156@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.325895:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.156@tcp1 of length 224 into portal 12 MB=0x603e14cd16200 00000400:00000200:20.0:1616513091.325901:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.156@tcp1 of length 224/224 into md 0x865be291 [1] + 29224 00000400:00000200:20.0:1616513091.325906:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.325909:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.325916:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.325919:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.325921:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.325927:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116051579264 00010000:00000200:0.0:1616513091.325938:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1693116051579264/t0(0) o400->022893e1-17fb-2f3c-8f2a-58b54c3ba264@10.31.136.156@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.325953:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693116051579264, offset 192 00000400:00000200:0.0:1616513091.325959:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.156@tcp1 00000400:00000200:0.0:1616513091.325966:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.156@tcp1 routed destination 00000100:00000200:12.0:1616513091.325978:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116051579392 00000400:00000200:0.0:1616513091.325981:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.156@tcp1(10.31.136.156@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.325988:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513091.325990:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dd100 x1693116051579392/t0(0) o400->37df4017-cd1d-715c-c56c-af9f07176f29@10.31.136.156@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.325993:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.325996:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.325998:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.326003:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513091.326005:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693116051579392, offset 192 00000100:00000200:0.0:1616513091.326010:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513091.326011:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.156@tcp1 00000400:00000200:12.0:1616513091.326018:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.156@tcp1 routed destination 00000800:00000200:11.2:1616513091.326021:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513091.326033:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.156@tcp1(10.31.136.156@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.326040:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.326046:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.326048:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.326050:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.326055:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.326061:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513091.326074:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.326081:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513091.326084:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513091.326085:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513091.326090:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513091.326096:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb055c8 00000800:00000200:16.0:1616513091.326099:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513091.326115:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.326124:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.326127:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.326133:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.326137:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ed48 00000800:00000200:18.0:1616513091.326141:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.326144:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.330774:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.330827:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513091.330832:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513091.330836:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513091.330846:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.106@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.330855:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.106@tcp1 of length 224 into portal 12 MB=0x6018373b38900 00000400:00000200:22.0:1616513091.330862:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.106@tcp1 of length 224/224 into md 0xf964d519 [1] + 92152 00000400:00000200:22.0:1616513091.330867:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.330869:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513091.330878:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.330881:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513091.330881:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.330885:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513091.330906:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.330915:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.330924:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.106@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.330933:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.106@tcp1 of length 224 into portal 26 MB=0x6018373b38880 00000400:00000200:20.0:1616513091.330939:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.106@tcp1 of length 224/224 into md 0xf97cdf99 [1] + 0 00000400:00000200:20.0:1616513091.330944:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43cb0 00000100:00000200:18.0:1616513091.330946:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513953753344 00000400:00000200:20.0:1616513091.330947:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.330950:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.330954:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43cb0 00000800:00000200:20.0:1616513091.330957:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:18.0:1616513091.330958:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a29200 x1690513953753344/t0(0) o400->a2353f6c-4c68-935e-f342-8a3ce86206b8@10.31.140.106@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513091.330962:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.330963:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.330967:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:18.0:1616513091.330972:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513953753344, offset 192 00000400:00000200:18.0:1616513091.330979:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.106@tcp1 00000400:00000200:18.0:1616513091.330986:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.106@tcp1 routed destination 00000400:00000200:18.0:1616513091.331001:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.106@tcp1(10.31.140.106@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513091.331008:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513091.331013:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513091.331016:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513091.331018:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.331018:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513953753216 00000800:00000200:18.0:1616513091.331023:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.331028:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:0.0:1616513091.331029:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1690513953753216/t0(0) o400->45127511-8866-17b1-b6a6-5696be9084e2@10.31.140.106@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.331044:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513953753216, offset 192 00000400:00000200:0.0:1616513091.331050:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.106@tcp1 00000400:00000200:0.0:1616513091.331057:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.106@tcp1 routed destination 00000400:00000200:0.0:1616513091.331072:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.106@tcp1(10.31.140.106@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.331079:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513091.331082:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513091.331085:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513091.331088:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513091.331088:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513091.331091:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513091.331091:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.331096:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513091.331098:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513091.331101:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513091.331102:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cdb18 00000100:00000200:0.0:1616513091.331103:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513091.331106:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.331109:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513091.331112:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.331116:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.331119:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.331124:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.331129:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:18.0:1616513091.331132:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.331134:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.335755:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513091.335776:0:50680:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513091.335812:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.335821:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.335831:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.86@tcp1 : PUT - for me 00000800:00000200:14.0:1616513091.335835:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513091.335840:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.86@tcp1 of length 224 into portal 12 MB=0x6048d8e583cc0 00000800:00000200:14.0:1616513091.335844:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513091.335846:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.86@tcp1 of length 224/224 into md 0x865be291 [1] + 29448 00000400:00000200:28.0:1616513091.335851:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.335854:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513091.335854:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.86@tcp1 : PUT - for me 00000800:00000200:28.0:1616513091.335860:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.335863:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513091.335863:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.86@tcp1 of length 224 into portal 26 MB=0x6048d8e583c40 00000800:00000200:28.0:1616513091.335864:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.335867:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513091.335869:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.86@tcp1 of length 224/224 into md 0xf97cdfa9 [1] + 0 00000400:00000200:14.0:1616513091.335874:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a436d8 00000400:00000200:14.0:1616513091.335877:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.335879:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513091.335884:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a436d8 00000800:00000200:14.0:1616513091.335887:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.335890:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513091.335890:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513091.335895:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513091.335925:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855885311168 00010000:00000200:12.0:1616513091.335938:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeeda400 x1693855885311168/t0(0) o400->11e70adf-d10d-f32e-d59b-ccb9cafb99e7@10.31.140.86@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513091.335950:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855885311040 00000100:00000200:12.0:1616513091.335952:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855885311168, offset 192 00000400:00000200:12.0:1616513091.335958:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.86@tcp1 00010000:00000200:0.0:1616513091.335962:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1693855885311040/t0(0) o400->67eccd96-2e82-fa01-5e6a-5d96905f9fce@10.31.140.86@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513091.335966:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.86@tcp1 routed destination 00000100:00000200:0.0:1616513091.335977:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855885311040, offset 192 00000400:00000200:12.0:1616513091.335980:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.86@tcp1(10.31.140.86@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513091.335983:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.86@tcp1 00000800:00000200:12.0:1616513091.335987:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513091.335990:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.86@tcp1 routed destination 00000800:00000200:12.0:1616513091.335998:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.336001:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513091.336003:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513091.336006:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.86@tcp1(10.31.140.86@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.336007:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513091.336012:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513091.336015:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.336018:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.336021:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.336023:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.336028:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.336034:0:50680:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513091.336035:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513091.336070:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.336076:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.336079:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.336086:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.336090:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e660 00000800:00000200:18.0:1616513091.336091:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513091.336094:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.336097:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513091.336097:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.336100:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.336107:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.336112:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb044c8 00000800:00000200:18.0:1616513091.336116:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.336119:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.337946:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.338002:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.338011:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513091.338017:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513091.338020:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.223@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.338028:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.223@tcp1 of length 224 into portal 26 MB=0x5ffd3d41a0140 00000400:00000200:22.0:1616513091.338035:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.223@tcp1 of length 224/224 into md 0xf97cdfb9 [1] + 0 00000400:00000200:22.0:1616513091.338040:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43210 00000400:00000200:22.0:1616513091.338042:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.338045:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513091.338049:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43210 00000800:00000200:22.0:1616513091.338052:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.338055:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.338056:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.338110:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660145209664 00010000:00000200:0.0:1616513091.338121:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688660145209664/t0(0) o400->3a858f36-7b1e-3189-1a19-0e45f2f5e8ff@10.31.128.223@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.338134:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660145209664, offset 192 00000400:00000200:0.0:1616513091.338140:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.223@tcp1 00000400:00000200:0.0:1616513091.338147:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.223@tcp1 routed destination 00000400:00000200:0.0:1616513091.338160:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.223@tcp1(10.31.128.223@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.338167:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.338172:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.338174:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.338176:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513091.338178:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.338181:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.338187:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.338187:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513091.338202:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.338208:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.338210:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.338216:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.338221:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:28.0:1616513091.338224:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.338227:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513091.338231:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.338242:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.338250:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.223@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.338257:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.223@tcp1 of length 224 into portal 12 MB=0x5ffd3d41a04c0 00000400:00000200:18.0:1616513091.338263:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.223@tcp1 of length 224/224 into md 0xf964d519 [1] + 92376 00000400:00000200:18.0:1616513091.338268:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.338270:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513091.338278:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.338281:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.338282:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.338285:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513091.338299:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660145210560 00010000:00000200:22.0:1616513091.338309:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd701200 x1688660145210560/t0(0) o400->01ccde45-4733-0d5a-bd7a-13a07cc3c091@10.31.128.223@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513091.338322:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660145210560, offset 192 00000400:00000200:22.0:1616513091.338328:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.223@tcp1 00000400:00000200:22.0:1616513091.338334:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.223@tcp1 routed destination 00000400:00000200:22.0:1616513091.338348:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.223@tcp1(10.31.128.223@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513091.338354:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513091.338362:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513091.338364:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.338366:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.338370:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.338375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.338386:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.338391:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.338393:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513091.338395:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513091.338400:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.338404:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4abb0 00000800:00000200:14.0:1616513091.338407:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:23.0:1616513091.338592:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928416, portal 4 00000100:00000200:23.0:1616513091.338598:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928416, offset 0 00000400:00000200:23.0:1616513091.338605:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:23.0:1616513091.338613:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:23.0:1616513091.338627:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513091.338634:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:23.0:1616513091.338641:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513091.338645:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:23.0:1616513091.338647:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:23.0:1616513091.338653:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513091.338701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513091.338721:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513091.338726:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513091.338729:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513091.338737:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513091.338744:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bf9f80 x1693067725928416/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513091.338758:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3eff0 00000800:00000200:1.0:1616513091.338763:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513091.338766:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513091.338914:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513091.338923:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513091.338929:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513091.338933:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513091.338938:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513091.338947:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6277e0 00000400:00000200:1.0:1616513091.338953:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700015 [1] + 192 00000400:00000200:1.0:1616513091.338959:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513091.338963:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bf9f80 x1693067725928416/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513091.338977:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513091.338980:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513091.338982:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:23.0:1616513091.338985:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bf9f80 x1693067725928416/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513091.338998:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bf9f80 x1693067725928416/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513091.339008:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f4b8 00000800:00000200:7.2:1616513091.349801:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513091.349858:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.349868:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.349877:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.183@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.349886:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.183@tcp1 of length 224 into portal 12 MB=0x5ffd43efd7400 00000400:00000200:18.0:1616513091.349892:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.183@tcp1 of length 224/224 into md 0xf964d519 [1] + 92600 00000400:00000200:18.0:1616513091.349897:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.349900:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513091.349907:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.349910:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.349911:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.349914:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513091.349973:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661938500608 00010000:00000200:22.0:1616513091.349985:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd706300 x1688661938500608/t0(0) o400->bfa366ce-2e50-33e1-cba6-52436515e3bb@10.31.131.183@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513091.349999:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661938500608, offset 192 00000400:00000200:22.0:1616513091.350005:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.183@tcp1 00000400:00000200:22.0:1616513091.350013:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.183@tcp1 routed destination 00000400:00000200:22.0:1616513091.350028:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.183@tcp1(10.31.131.183@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513091.350035:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513091.350040:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513091.350043:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513091.350045:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513091.350050:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.350057:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.350113:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.350120:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.350122:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.350129:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.350133:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a880 00000800:00000200:24.0:1616513091.350137:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.350140:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.351078:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.351139:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.351148:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.351157:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.183@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.351165:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.183@tcp1 of length 224 into portal 26 MB=0x5ffd43efd7340 00000400:00000200:28.0:1616513091.351172:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.183@tcp1 of length 224/224 into md 0xf97cdfd9 [1] + 0 00000400:00000200:28.0:1616513091.351177:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43d38 00000400:00000200:28.0:1616513091.351180:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.351183:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513091.351187:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43d38 00000800:00000200:28.0:1616513091.351190:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.351193:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.351194:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.351198:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.351252:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661938500416 00010000:00000200:0.0:1616513091.351263:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688661938500416/t0(0) o400->8642c475-c73f-3758-9e5a-9953a03220f6@10.31.131.183@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.351279:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661938500416, offset 192 00000400:00000200:0.0:1616513091.351285:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.183@tcp1 00000400:00000200:0.0:1616513091.351292:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.183@tcp1 routed destination 00000400:00000200:0.0:1616513091.351307:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.183@tcp1(10.31.131.183@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.351314:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513091.351319:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.351322:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.351324:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.351329:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.351334:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513091.351347:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513091.351390:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.351397:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.351399:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.351406:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.351411:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04110 00000800:00000200:22.0:1616513091.351414:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.351417:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.370979:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513091.371034:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.371043:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.371052:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.28@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.371060:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.28@tcp1 of length 224 into portal 26 MB=0x601447c360440 00000400:00000200:18.0:1616513091.371066:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.28@tcp1 of length 224/224 into md 0xf97cdff9 [1] + 0 00000400:00000200:18.0:1616513091.371070:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43540 00000400:00000200:18.0:1616513091.371073:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.371076:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513091.371079:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43540 00000800:00000200:18.0:1616513091.371082:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.371084:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.371086:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.371091:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.371101:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.371110:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.28@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.371119:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.28@tcp1 of length 224 into portal 12 MB=0x601447c3606c0 00000400:00000200:2.0:1616513091.371126:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.28@tcp1 of length 224/224 into md 0xf964d519 [1] + 92824 00000400:00000200:2.0:1616513091.371131:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.371134:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513091.371141:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690243513582656 00000800:00000200:2.0:1616513091.371142:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.371145:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.371147:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.371151:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513091.371152:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1690243513582656/t0(0) o400->77ca8235-98a2-66f1-c522-4ae34b6c91cd@10.31.132.28@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.371165:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690243513582656, offset 192 00000400:00000200:0.0:1616513091.371170:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.28@tcp1 00000400:00000200:0.0:1616513091.371177:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.28@tcp1 routed destination 00000400:00000200:0.0:1616513091.371191:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.28@tcp1(10.31.132.28@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.371197:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.371202:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.371205:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513091.371207:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690243513583296 00000800:00000200:0.0:1616513091.371207:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.371211:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.371218:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:26.0:1616513091.371219:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7980 x1690243513583296/t0(0) o400->80cd4fe0-bc09-7887-4693-1b062b019214@10.31.132.28@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513091.371219:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513091.371234:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690243513583296, offset 192 00000400:00000200:26.0:1616513091.371240:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.28@tcp1 00000400:00000200:26.0:1616513091.371247:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.28@tcp1 routed destination 00000400:00000200:26.0:1616513091.371262:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.28@tcp1(10.31.132.28@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.371269:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.371274:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.371277:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513091.371277:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513091.371279:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.371284:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513091.371284:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.371287:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513091.371289:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513091.371293:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.371299:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:14.0:1616513091.371302:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513091.371304:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.371307:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513091.371309:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.371313:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.371318:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513091.371325:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.371329:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af100 00000800:00000200:14.0:1616513091.371333:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.392462:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.392525:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.392534:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.392543:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.69@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.392552:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.69@tcp1 of length 224 into portal 26 MB=0x6021666ac9700 00000400:00000200:20.0:1616513091.392559:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.69@tcp1 of length 224/224 into md 0xf97ce039 [1] + 0 00000400:00000200:20.0:1616513091.392564:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43078 00000400:00000200:20.0:1616513091.392567:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.392569:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.392574:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43078 00000800:00000200:20.0:1616513091.392577:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.392580:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.392582:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.392584:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.392589:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.392596:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.69@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.392601:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.69@tcp1 of length 224 into portal 12 MB=0x6021666ac9780 00000800:00000200:28.0:1616513091.392604:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.392606:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.69@tcp1 of length 224/224 into md 0x865be291 [1] + 29672 00000400:00000200:20.0:1616513091.392610:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.392612:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.392618:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.392619:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.392621:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.392634:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145095386880 00010000:00000200:0.0:1616513091.392643:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1691145095386880/t0(0) o400->da85af6f-5a81-7a13-2207-ac9945df2459@10.31.141.69@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.392656:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691145095386880, offset 192 00000400:00000200:0.0:1616513091.392661:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.69@tcp1 00000400:00000200:0.0:1616513091.392667:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.69@tcp1 routed destination 00000400:00000200:0.0:1616513091.392682:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.69@tcp1(10.31.141.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:12.0:1616513091.392684:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145095387008 00000800:00000200:0.0:1616513091.392687:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.392692:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:12.0:1616513091.392695:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d8900 x1691145095387008/t0(0) o400->e1053dae-4510-7685-6008-ad607fd66383@10.31.141.69@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.392695:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.392697:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.392701:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.392707:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.392707:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513091.392708:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691145095387008, offset 192 00000400:00000200:12.0:1616513091.392714:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.69@tcp1 00000400:00000200:12.0:1616513091.392720:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.69@tcp1 routed destination 00000400:00000200:12.0:1616513091.392734:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.69@tcp1(10.31.141.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.392740:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.392746:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.392748:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.392750:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.392754:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513091.392760:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.392767:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513091.392770:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513091.392771:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513091.392777:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513091.392782:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:16.0:1616513091.392786:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.392801:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.392857:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.392864:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.392867:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.392874:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.392878:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f540 00000800:00000200:22.0:1616513091.392882:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.392885:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.403346:0:50687:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513091.403402:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.403411:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.403420:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.234@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.403430:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.234@tcp1 of length 224 into portal 26 MB=0x60182c5dd1040 00000400:00000200:18.0:1616513091.403450:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.234@tcp1 of length 224/224 into md 0xf97ce049 [1] + 0 00000400:00000200:18.0:1616513091.403455:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42198 00000800:00000200:14.0:1616513091.403456:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513091.403457:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.403460:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513091.403464:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42198 00000800:00000200:18.0:1616513091.403467:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.403470:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.403471:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.403528:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511037239360 00010000:00000200:0.0:1616513091.403540:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1690511037239360/t0(0) o400->6c682c76-c3c6-d486-1322-ceb649696edc@10.31.132.234@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.403554:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511037239360, offset 192 00000400:00000200:0.0:1616513091.403560:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.234@tcp1 00000400:00000200:0.0:1616513091.403568:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.234@tcp1 routed destination 00000400:00000200:0.0:1616513091.403583:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.234@tcp1(10.31.132.234@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.403590:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.403595:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.403598:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.403600:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.403605:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.403612:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513091.403650:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.403707:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.403713:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.403716:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.403723:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.403728:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:20.0:1616513091.403731:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.403734:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.403931:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.403984:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.403994:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.404003:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.234@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.404012:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.234@tcp1 of length 224 into portal 12 MB=0x60182c5dd1180 00000400:00000200:28.0:1616513091.404018:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.234@tcp1 of length 224/224 into md 0x865be291 [1] + 29896 00000400:00000200:28.0:1616513091.404024:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.404026:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513091.404032:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.404035:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.404037:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.404040:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.404095:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511037239680 00010000:00000200:12.0:1616513091.404108:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed9680 x1690511037239680/t0(0) o400->05bfb7e1-9d46-9cc3-d4ea-d804490e5e27@10.31.132.234@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.404122:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511037239680, offset 192 00000400:00000200:12.0:1616513091.404128:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.234@tcp1 00000400:00000200:12.0:1616513091.404136:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.234@tcp1 routed destination 00000400:00000200:12.0:1616513091.404152:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.234@tcp1(10.31.132.234@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.404159:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.404164:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.404167:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.404169:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.404174:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.404179:0:50687:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.404233:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.404239:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.404242:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.404249:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.404253:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ea18 00000800:00000200:22.0:1616513091.404257:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.404260:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.419187:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.419243:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.419253:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.419261:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.157@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.419270:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.157@tcp1 of length 224 into portal 26 MB=0x5ffd3d650d680 00000400:00000200:20.0:1616513091.419276:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.157@tcp1 of length 224/224 into md 0xf97ce059 [1] + 0 00000400:00000200:20.0:1616513091.419282:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42ff0 00000400:00000200:20.0:1616513091.419285:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.419287:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.419292:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42ff0 00000800:00000200:20.0:1616513091.419295:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.419297:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.419298:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.419300:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.419370:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660182357632 00010000:00000200:0.0:1616513091.419380:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688660182357632/t0(0) o400->7944be01-a757-f4bc-b214-fd3f6efce2fd@10.31.131.157@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.419394:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660182357632, offset 192 00000400:00000200:0.0:1616513091.419400:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.157@tcp1 00000400:00000200:0.0:1616513091.419407:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.157@tcp1 routed destination 00000800:00000200:11.2:1616513091.419409:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:0.0:1616513091.419420:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.157@tcp1(10.31.131.157@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.419427:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.419432:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.419437:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.419454:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513091.419459:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.419465:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513091.419466:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513091.419474:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.419482:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.157@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.419490:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.157@tcp1 of length 224 into portal 12 MB=0x5ffd3d650da00 00000400:00000200:28.0:1616513091.419496:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.157@tcp1 of length 224/224 into md 0x865be291 [1] + 30120 00000400:00000200:28.0:1616513091.419501:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.419503:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513091.419509:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513091.419512:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513091.419514:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.419516:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.419520:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.419522:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.419522:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513091.419527:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.419531:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:28.0:1616513091.419534:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.419570:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660182358528 00010000:00000200:12.0:1616513091.419581:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed8000 x1688660182358528/t0(0) o400->b29e1a80-959d-dbdc-f8eb-59d235790d60@10.31.131.157@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.419595:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660182358528, offset 192 00000400:00000200:12.0:1616513091.419601:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.157@tcp1 00000400:00000200:12.0:1616513091.419608:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.157@tcp1 routed destination 00000400:00000200:12.0:1616513091.419621:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.157@tcp1(10.31.131.157@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.419628:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.419633:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.419635:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.419637:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.419642:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.419656:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.419711:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.419718:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.419721:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513091.419728:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.419732:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777edd0 00000800:00000200:14.0:1616513091.419736:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.419739:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.438069:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.438125:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.438135:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513091.438144:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.84@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.438154:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.84@tcp1 of length 224 into portal 12 MB=0x5ffdfb17566c0 00000400:00000200:22.0:1616513091.438161:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.84@tcp1 of length 224/224 into md 0xf964d519 [1] + 93048 00000400:00000200:22.0:1616513091.438166:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.438169:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513091.438177:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.438179:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.438180:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.438186:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.438189:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.438190:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.438198:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.84@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.438208:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.84@tcp1 of length 224 into portal 26 MB=0x5ffdfb1756640 00000400:00000200:2.0:1616513091.438215:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.84@tcp1 of length 224/224 into md 0xf97ce079 [1] + 0 00000400:00000200:2.0:1616513091.438219:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a434b8 00000400:00000200:2.0:1616513091.438222:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.438225:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513091.438229:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a434b8 00000800:00000200:2.0:1616513091.438232:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.438235:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.438237:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.438242:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711103604416 00010000:00000200:26.0:1616513091.438254:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2b180 x1688711103604416/t0(0) o400->f1f46fb5-73f7-a108-e2f9-36aec3c0ad9a@10.31.133.84@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.438268:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688711103604416, offset 192 00000400:00000200:26.0:1616513091.438274:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.84@tcp1 00000400:00000200:26.0:1616513091.438282:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.84@tcp1 routed destination 00000100:00000200:0.0:1616513091.438294:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711103604288 00000400:00000200:26.0:1616513091.438297:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.84@tcp1(10.31.133.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.438304:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:0.0:1616513091.438305:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688711103604288/t0(0) o400->2388fca9-3f50-6b4c-5456-6339a3a9e2ca@10.31.133.84@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513091.438309:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.438312:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513091.438314:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513091.438319:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.438319:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688711103604288, offset 192 00000400:00000200:0.0:1616513091.438325:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.84@tcp1 00000400:00000200:0.0:1616513091.438333:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.84@tcp1 routed destination 00000400:00000200:0.0:1616513091.438348:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.84@tcp1(10.31.133.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.438354:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513091.438360:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.438363:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.438365:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513091.438366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.438370:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.438375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513091.438376:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513091.438387:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.438394:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.438397:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513091.438398:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513091.438403:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.438409:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:18.0:1616513091.438412:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513091.438423:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.438430:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.438446:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.438453:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.438457:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aea18 00000800:00000200:20.0:1616513091.438461:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.438464:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.467452:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513091.467481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513091.467510:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.467519:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.467528:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.54@tcp1 : PUT - for me 00000800:00000200:24.0:1616513091.467540:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513091.467540:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.54@tcp1 of length 224 into portal 26 MB=0x5f90972a97f80 00000400:00000200:2.0:1616513091.467547:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.54@tcp1 of length 224/224 into md 0xf97ce099 [1] + 0 00000800:00000200:24.0:1616513091.467550:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513091.467552:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43320 00000400:00000200:2.0:1616513091.467554:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.467557:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513091.467559:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.54@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.467561:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43320 00000800:00000200:2.0:1616513091.467564:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.467566:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.467567:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:24.0:1616513091.467569:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.54@tcp1 of length 224 into portal 12 MB=0x5f90972a98000 00000800:00000200:2.0:1616513091.467570:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513091.467575:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.54@tcp1 of length 224/224 into md 0x865be291 [1] + 30344 00000400:00000200:24.0:1616513091.467580:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.467583:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.467589:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.467592:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.467594:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.467597:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.467629:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681193857286016 00010000:00000200:0.0:1616513091.467640:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1681193857286016/t0(0) o400->54b8b453-9715-6d6a-7a0d-f6d43fe7a0ae@10.31.131.54@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513091.467655:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681193857286144 00000100:00000200:0.0:1616513091.467655:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681193857286016, offset 192 00000400:00000200:0.0:1616513091.467661:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.54@tcp1 00010000:00000200:12.0:1616513091.467668:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e4380 x1681193857286144/t0(0) o400->24a33bb0-e96b-c365-ce3f-5905f6bcc7b7@10.31.131.54@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513091.467669:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.54@tcp1 routed destination 00000100:00000200:12.0:1616513091.467682:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681193857286144, offset 192 00000400:00000200:0.0:1616513091.467683:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.54@tcp1(10.31.131.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513091.467688:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.54@tcp1 00000800:00000200:0.0:1616513091.467690:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513091.467695:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.54@tcp1 routed destination 00000800:00000200:0.0:1616513091.467703:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.467706:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.467708:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513091.467710:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.54@tcp1(10.31.131.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.467713:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513091.467717:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513091.467718:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.467720:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513091.467723:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.467725:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.467729:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513091.467731:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513091.467734:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513091.467738:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.467740:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.467747:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513091.467748:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513091.467752:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:20.0:1616513091.467756:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.467759:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513091.467812:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.467819:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.467822:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.467828:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.467832:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e198 00000800:00000200:22.0:1616513091.467836:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.467839:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.481546:0:50695:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.481603:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.481613:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.481623:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.93@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.481633:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.93@tcp1 of length 224 into portal 26 MB=0x5ffdd3da45480 00000400:00000200:14.0:1616513091.481640:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.93@tcp1 of length 224/224 into md 0xf97ce0a9 [1] + 0 00000400:00000200:14.0:1616513091.481645:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43ba0 00000400:00000200:14.0:1616513091.481648:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.481651:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513091.481655:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ba0 00000800:00000200:2.0:1616513091.481657:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.481659:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.481661:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.481663:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.481730:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700570588288 00010000:00000200:0.0:1616513091.481742:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688700570588288/t0(0) o400->06f93139-11b8-a2b4-0e48-30b364cb8c05@10.31.141.93@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.481756:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700570588288, offset 192 00000400:00000200:0.0:1616513091.481763:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.93@tcp1 00000400:00000200:0.0:1616513091.481771:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.93@tcp1 routed destination 00000400:00000200:0.0:1616513091.481786:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.93@tcp1(10.31.141.93@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.481793:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.481799:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.481802:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.481804:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.481809:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513091.481816:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513091.481817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.481873:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.481880:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.481882:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.481889:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.481895:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:24.0:1616513091.481898:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.481901:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.481992:0:50695:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.482048:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.482057:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513091.482066:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.93@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.482075:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.93@tcp1 of length 224 into portal 12 MB=0x5ffdd3da45500 00000400:00000200:22.0:1616513091.482081:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.93@tcp1 of length 224/224 into md 0xf964d519 [1] + 93272 00000400:00000200:22.0:1616513091.482087:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.482089:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513091.482099:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.482102:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.482104:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.482107:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.482161:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700570588416 00010000:00000200:26.0:1616513091.482173:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2f080 x1688700570588416/t0(0) o400->5b7b7b22-0ab7-8c32-400d-1808ac60fa1b@10.31.141.93@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.482187:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700570588416, offset 192 00000400:00000200:26.0:1616513091.482192:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.93@tcp1 00000400:00000200:26.0:1616513091.482199:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.93@tcp1 routed destination 00000400:00000200:26.0:1616513091.482213:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.93@tcp1(10.31.141.93@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.482219:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.482224:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.482227:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.482229:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.482234:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.482238:0:50695:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.482248:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.482254:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513091.482256:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513091.482259:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513091.482263:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513091.482267:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afba0 00000800:00000200:2.0:1616513091.482271:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.484517:0:50695:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.484573:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.484582:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513091.484591:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.116@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.484600:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.116@tcp1 of length 224 into portal 12 MB=0x5fc62e836c240 00000400:00000200:22.0:1616513091.484606:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.116@tcp1 of length 224/224 into md 0xf964d519 [1] + 93496 00000400:00000200:22.0:1616513091.484611:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.484614:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513091.484620:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.484623:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.484624:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.484627:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.484683:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1684876616450624 00010000:00000200:26.0:1616513091.484695:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2bf00 x1684876616450624/t0(0) o400->32f476ce-635f-1547-f6e2-819c91148dfa@10.31.143.116@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.484709:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1684876616450624, offset 192 00000800:00000200:11.2:1616513091.484714:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513091.484715:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.116@tcp1 00000400:00000200:26.0:1616513091.484723:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.116@tcp1 routed destination 00000400:00000200:26.0:1616513091.484740:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.116@tcp1(10.31.143.116@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.484746:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513091.484752:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.484755:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513091.484757:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513091.484762:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.484770:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.484779:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.484788:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.116@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.484797:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.116@tcp1 of length 224 into portal 26 MB=0x5fc62e836c1c0 00000400:00000200:20.0:1616513091.484804:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.116@tcp1 of length 224/224 into md 0xf97ce0b9 [1] + 0 00000400:00000200:20.0:1616513091.484808:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a423b8 00000400:00000200:20.0:1616513091.484811:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.484817:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.484822:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a423b8 00000800:00000200:20.0:1616513091.484824:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513091.484827:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513091.484827:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:20.0:1616513091.484834:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513091.484835:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513091.484837:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513091.484838:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513091.484845:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513091.484849:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa90 00000800:00000200:16.0:1616513091.484853:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.484885:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1684876616450496 00010000:00000200:0.0:1616513091.484908:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1684876616450496/t0(0) o400->30bf7d93-bfc5-b49c-99a5-d5d6dc2a3ac2@10.31.143.116@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.484921:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1684876616450496, offset 192 00000400:00000200:0.0:1616513091.484927:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.116@tcp1 00000400:00000200:0.0:1616513091.484933:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.116@tcp1 routed destination 00000400:00000200:0.0:1616513091.484947:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.116@tcp1(10.31.143.116@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.484953:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513091.484959:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.484961:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.484963:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.484968:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513091.484974:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513091.484975:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.484999:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.485005:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.485008:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513091.485015:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.485020:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:14.0:1616513091.485023:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.485026:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.513166:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.513229:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.513238:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.513248:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.237@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.513257:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.237@tcp1 of length 224 into portal 26 MB=0x604f619e75f80 00000400:00000200:28.0:1616513091.513263:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.237@tcp1 of length 224/224 into md 0xf97ce0c9 [1] + 0 00000400:00000200:28.0:1616513091.513271:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42f68 00000400:00000200:28.0:1616513091.513274:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.513277:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513091.513282:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42f68 00000800:00000200:20.0:1616513091.513283:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.513285:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.513287:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513091.513290:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.513293:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.513294:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.513303:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.237@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.513311:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.237@tcp1 of length 224 into portal 12 MB=0x604f619e76280 00000400:00000200:20.0:1616513091.513318:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.237@tcp1 of length 224/224 into md 0x865be291 [1] + 30568 00000400:00000200:20.0:1616513091.513323:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.513326:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.513333:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.513336:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.513338:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.513348:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304903323520 00010000:00000200:0.0:1616513091.513360:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1694304903323520/t0(0) o400->e9cd042d-4f64-d7ac-ea5c-3f56ea54045e@10.31.140.237@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.513375:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694304903323520, offset 192 00000400:00000200:0.0:1616513091.513383:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.237@tcp1 00000400:00000200:0.0:1616513091.513390:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.237@tcp1 routed destination 00000100:00000200:12.0:1616513091.513401:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304903324288 00000400:00000200:0.0:1616513091.513406:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.237@tcp1(10.31.140.237@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.513412:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513091.513413:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dde80 x1694304903324288/t0(0) o400->8ca55b01-bdbf-51c3-0ebc-04bc6de78f19@10.31.140.237@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.513418:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.513421:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.513423:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513091.513428:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694304903324288, offset 192 00000800:00000200:0.0:1616513091.513428:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.513433:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513091.513449:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.237@tcp1 00000100:00000200:0.0:1616513091.513451:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513091.513456:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.237@tcp1 routed destination 00000400:00000200:12.0:1616513091.513472:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.237@tcp1(10.31.140.237@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.513478:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.513484:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.513487:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513091.513488:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513091.513489:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.513494:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513091.513495:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.513498:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513091.513500:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513091.513502:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513091.513505:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.513510:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:24.0:1616513091.513514:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513091.513559:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.513567:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.513570:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.513576:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.513581:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ebb0 00000800:00000200:22.0:1616513091.513585:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.513588:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.538155:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.538210:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.538219:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.538229:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.219@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.538237:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.219@tcp1 of length 224 into portal 26 MB=0x5ffd548ace700 00000400:00000200:20.0:1616513091.538244:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.219@tcp1 of length 224/224 into md 0xf97ce0d9 [1] + 0 00000400:00000200:20.0:1616513091.538250:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43650 00000400:00000200:20.0:1616513091.538252:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.538255:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.538260:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43650 00000800:00000200:20.0:1616513091.538263:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.538264:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.538266:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.538268:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.538326:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666395961088 00010000:00000200:0.0:1616513091.538337:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688666395961088/t0(0) o400->256076a9-dcdb-8584-68b0-8eb1a65323b6@10.31.132.219@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.538352:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666395961088, offset 192 00000400:00000200:0.0:1616513091.538359:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.219@tcp1 00000400:00000200:0.0:1616513091.538366:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.219@tcp1 routed destination 00000400:00000200:0.0:1616513091.538381:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.219@tcp1(10.31.132.219@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.538388:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.538394:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.538397:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.538399:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.538404:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.538409:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.538411:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513091.538463:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.538469:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.538472:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.538479:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.538485:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:28.0:1616513091.538488:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.538491:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.538932:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.538942:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.538950:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513091.538952:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513091.538959:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.219@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.538968:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.219@tcp1 of length 224 into portal 12 MB=0x5ffd548ace880 00000400:00000200:16.0:1616513091.538974:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.219@tcp1 of length 224/224 into md 0x865be291 [1] + 30792 00000400:00000200:16.0:1616513091.538980:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.538982:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513091.538991:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.538994:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.538996:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.539055:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666395961472 00010000:00000200:12.0:1616513091.539067:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd3f00 x1688666395961472/t0(0) o400->8955c610-dc21-15f7-1c95-a49c1ed9e52b@10.31.132.219@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.539082:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666395961472, offset 192 00000400:00000200:12.0:1616513091.539088:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.219@tcp1 00000400:00000200:12.0:1616513091.539095:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.219@tcp1 routed destination 00000400:00000200:12.0:1616513091.539110:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.219@tcp1(10.31.132.219@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.539117:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.539123:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.539126:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.539128:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.539133:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.539140:0:50701:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.539196:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.539205:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.539208:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513091.539215:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.539218:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e3b8 00000800:00000200:14.0:1616513091.539222:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.539225:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.541566:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.541624:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.541633:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513091.541642:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.98@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.541652:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.98@tcp1 of length 224 into portal 26 MB=0x5ffde90f8d600 00000400:00000200:22.0:1616513091.541658:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.98@tcp1 of length 224/224 into md 0xf97ce0e9 [1] + 0 00000400:00000200:22.0:1616513091.541663:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42aa0 00000400:00000200:22.0:1616513091.541665:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.541668:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513091.541672:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42aa0 00000800:00000200:22.0:1616513091.541675:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.541678:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.541678:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.541682:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.541739:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706263602688 00010000:00000200:0.0:1616513091.541750:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688706263602688/t0(0) o400->668b9c9d-3eae-3d50-392e-2c9076e6ea27@10.31.136.98@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.541764:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706263602688, offset 192 00000400:00000200:0.0:1616513091.541770:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.98@tcp1 00000400:00000200:0.0:1616513091.541778:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.98@tcp1 routed destination 00000400:00000200:0.0:1616513091.541792:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.98@tcp1(10.31.136.98@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.541803:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.541809:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.541811:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.541813:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.541818:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.541824:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.541825:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513091.541877:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.541884:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.541887:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.541893:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.541899:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb043b8 00000800:00000200:28.0:1616513091.541902:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.541905:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.541947:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.542000:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.542009:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.542018:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.98@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.542026:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.98@tcp1 of length 224 into portal 12 MB=0x5ffde90f8d700 00000400:00000200:14.0:1616513091.542033:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.98@tcp1 of length 224/224 into md 0xf964d519 [1] + 93720 00000400:00000200:14.0:1616513091.542038:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.542041:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513091.542048:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.542051:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513091.542053:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.542056:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.542112:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706263602944 00010000:00000200:26.0:1616513091.542124:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07080 x1688706263602944/t0(0) o400->20d1ada1-ed40-b41f-86ec-1c7c22eb853b@10.31.136.98@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.542139:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706263602944, offset 192 00000400:00000200:26.0:1616513091.542145:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.98@tcp1 00000400:00000200:26.0:1616513091.542152:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.98@tcp1 routed destination 00000400:00000200:26.0:1616513091.542167:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.98@tcp1(10.31.136.98@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.542174:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.542179:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.542182:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.542184:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.542189:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.542194:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.542205:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.542210:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.542213:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513091.542215:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513091.542219:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.542223:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afed0 00000800:00000200:18.0:1616513091.542227:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.545593:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.545647:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.545656:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.545665:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.100@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.545674:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.100@tcp1 of length 224 into portal 26 MB=0x5ffd3ef467e80 00000400:00000200:20.0:1616513091.545682:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.100@tcp1 of length 224/224 into md 0xf97ce109 [1] + 0 00000400:00000200:20.0:1616513091.545687:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43ed0 00000400:00000200:20.0:1616513091.545690:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.545693:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.545697:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43ed0 00000800:00000200:20.0:1616513091.545700:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.545700:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.545703:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.545707:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.545710:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.545711:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.545719:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.100@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.545728:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.100@tcp1 of length 224 into portal 12 MB=0x5ffd3ef467f00 00000400:00000200:16.0:1616513091.545734:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.100@tcp1 of length 224/224 into md 0x865be291 [1] + 31016 00000400:00000200:16.0:1616513091.545739:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.545742:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513091.545749:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.545752:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.545754:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.545761:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660601110144 00010000:00000200:0.0:1616513091.545772:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688660601110144/t0(0) o400->85c5dd08-e71f-f1d5-3c30-264972142100@10.31.128.100@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.545786:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660601110144, offset 192 00000400:00000200:0.0:1616513091.545792:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.100@tcp1 00000400:00000200:0.0:1616513091.545800:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.100@tcp1 routed destination 00000100:00000200:12.0:1616513091.545813:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660601110272 00000400:00000200:0.0:1616513091.545815:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.100@tcp1(10.31.128.100@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.545821:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513091.545825:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd5580 x1688660601110272/t0(0) o400->acd42b41-e05c-87d3-c22c-4cbe04c424c0@10.31.128.100@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.545827:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.545830:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.545832:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.545837:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513091.545839:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660601110272, offset 192 00000800:00000200:11.2:1616513091.545842:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.545844:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513091.545847:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.100@tcp1 00000400:00000200:12.0:1616513091.545854:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.100@tcp1 routed destination 00000400:00000200:12.0:1616513091.545869:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.100@tcp1(10.31.128.100@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.545876:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.545882:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.545884:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.545886:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.545891:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513091.545896:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513091.545896:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513091.545903:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.545905:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513091.545906:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513091.545912:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.545917:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000800:00000200:24.0:1616513091.545921:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513091.545949:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.545956:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.545959:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513091.545965:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.545969:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f5c8 00000800:00000200:14.0:1616513091.545973:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.545976:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.570301:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.570356:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.570365:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.570375:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.189@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.570385:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.189@tcp1 of length 224 into portal 12 MB=0x5ffd43027f680 00000400:00000200:16.0:1616513091.570392:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.189@tcp1 of length 224/224 into md 0x865be291 [1] + 31240 00000400:00000200:16.0:1616513091.570397:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.570400:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513091.570406:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.570409:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.570410:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513091.570414:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.570470:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661689628288 00010000:00000200:12.0:1616513091.570483:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd4380 x1688661689628288/t0(0) o400->5531dd4c-ce48-5cf2-31d3-76308b5c4034@10.31.136.189@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.570497:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661689628288, offset 192 00000400:00000200:12.0:1616513091.570503:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.189@tcp1 00000400:00000200:12.0:1616513091.570512:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.189@tcp1 routed destination 00000400:00000200:12.0:1616513091.570527:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.189@tcp1(10.31.136.189@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.570533:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513091.570539:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.570542:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513091.570544:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513091.570548:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.570553:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.570566:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.570573:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.570575:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.570582:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.570586:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e7f8 00000800:00000200:20.0:1616513091.570590:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.570593:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.571592:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.571647:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.571657:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513091.571666:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.189@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.571675:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.189@tcp1 of length 224 into portal 26 MB=0x5ffd43027f540 00000400:00000200:22.0:1616513091.571682:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.189@tcp1 of length 224/224 into md 0xf97ce119 [1] + 0 00000400:00000200:22.0:1616513091.571687:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a435c8 00000400:00000200:22.0:1616513091.571689:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.571692:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513091.571696:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a435c8 00000800:00000200:22.0:1616513091.571700:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.571701:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.571703:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.571706:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.571790:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661689627968 00010000:00000200:0.0:1616513091.571801:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688661689627968/t0(0) o400->d6b7e835-5a65-4c1e-5bca-143806a48925@10.31.136.189@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.571816:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661689627968, offset 192 00000400:00000200:0.0:1616513091.571822:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.189@tcp1 00000400:00000200:0.0:1616513091.571829:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.189@tcp1 routed destination 00000400:00000200:0.0:1616513091.571844:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.189@tcp1(10.31.136.189@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.571851:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513091.571857:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.571860:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.571864:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.571869:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513091.571876:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513091.571879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.571933:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.571940:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.571943:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513091.571949:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.571955:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:14.0:1616513091.571958:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.571961:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.587773:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.587791:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.587802:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.587811:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.110@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.587820:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.110@tcp1 of length 224 into portal 26 MB=0x604a298f4b680 00000400:00000200:28.0:1616513091.587827:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.110@tcp1 of length 224/224 into md 0xf97ce139 [1] + 0 00000400:00000200:28.0:1616513091.587831:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a427f8 00000400:00000200:28.0:1616513091.587834:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.587837:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513091.587842:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a427f8 00000800:00000200:28.0:1616513091.587845:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.587849:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.587851:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.587854:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.587859:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513091.587859:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513091.587866:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.110@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.587871:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.110@tcp1 of length 224 into portal 12 MB=0x604a298f4b700 00000400:00000200:28.0:1616513091.587877:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.110@tcp1 of length 224/224 into md 0x865be291 [1] + 31464 00000400:00000200:28.0:1616513091.587880:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.587883:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513091.587888:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.587890:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.587892:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.587907:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946257651328 00010000:00000200:0.0:1616513091.587919:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1693946257651328/t0(0) o400->a263e7dd-640d-7626-72c4-b58091b16fc7@10.31.140.110@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.587933:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946257651328, offset 192 00000400:00000200:0.0:1616513091.587939:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.110@tcp1 00000400:00000200:0.0:1616513091.587947:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.110@tcp1 routed destination 00000100:00000200:12.0:1616513091.587955:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946257651456 00000400:00000200:0.0:1616513091.587962:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.110@tcp1(10.31.140.110@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:12.0:1616513091.587968:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedf500 x1693946257651456/t0(0) o400->da279668-b182-c0fd-90d2-3e86d9cb9083@10.31.140.110@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.587968:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.587974:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.587977:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.587979:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513091.587982:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946257651456, offset 192 00000800:00000200:0.0:1616513091.587984:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513091.587989:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.110@tcp1 00000800:00000200:11.2:1616513091.587989:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.587991:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513091.587996:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.110@tcp1 routed destination 00000400:00000200:12.0:1616513091.588011:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.110@tcp1(10.31.140.110@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.588018:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.588024:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.588029:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.588031:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.588036:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513091.588043:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513091.588043:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513091.588052:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.588054:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.588055:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513091.588061:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.588066:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:24.0:1616513091.588070:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513091.588100:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.588107:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.588110:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.588117:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.588121:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fcb0 00000800:00000200:18.0:1616513091.588127:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.588129:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.592639:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.592705:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.592721:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.592730:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.176@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.592740:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.176@tcp1 of length 224 into portal 26 MB=0x5ffe3a425dd40 00000400:00000200:14.0:1616513091.592747:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.176@tcp1 of length 224/224 into md 0xf97ce149 [1] + 0 00000400:00000200:14.0:1616513091.592754:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42990 00000400:00000200:14.0:1616513091.592757:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.592760:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513091.592764:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42990 00000800:00000200:14.0:1616513091.592767:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.592770:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513091.592772:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.592774:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.592779:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513091.592785:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513091.592786:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.176@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.592792:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.176@tcp1 of length 224 into portal 12 MB=0x5ffe3a425df00 00000400:00000200:14.0:1616513091.592797:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.176@tcp1 of length 224/224 into md 0xf964d519 [1] + 93944 00000400:00000200:14.0:1616513091.592801:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.592804:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513091.592809:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.592811:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.592813:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.592830:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728060157248 00010000:00000200:0.0:1616513091.592841:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688728060157248/t0(0) o400->45e98a45-024c-c433-4d6a-bb427475c8cc@10.31.141.176@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.592855:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688728060157248, offset 192 00000400:00000200:0.0:1616513091.592861:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.176@tcp1 00000400:00000200:0.0:1616513091.592869:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.176@tcp1 routed destination 00000100:00000200:26.0:1616513091.592875:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728060157696 00000400:00000200:0.0:1616513091.592884:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.176@tcp1(10.31.141.176@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:26.0:1616513091.592888:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01200 x1688728060157696/t0(0) o400->f43d59a9-581f-dbfe-d2a4-63d313d143b9@10.31.141.176@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.592891:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.592897:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.592899:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.592901:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513091.592902:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688728060157696, offset 192 00000800:00000200:0.0:1616513091.592906:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513091.592909:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.176@tcp1 00000800:00000200:11.2:1616513091.592913:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.592913:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513091.592916:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.176@tcp1 routed destination 00000400:00000200:26.0:1616513091.592931:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.176@tcp1(10.31.141.176@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.592937:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.592943:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.592946:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.592950:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.592955:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.592960:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513091.592968:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.592975:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.592978:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.592984:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.592990:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:28.0:1616513091.592993:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.592996:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513091.593015:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.593023:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.593026:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513091.593028:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513091.593032:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.593036:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae880 00000800:00000200:22.0:1616513091.593040:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:2.0:1616513091.594518:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928432, portal 4 00000100:00000200:2.0:1616513091.594524:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928432, offset 0 00000400:00000200:2.0:1616513091.594530:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:2.0:1616513091.594536:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:2.0:1616513091.594549:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513091.594555:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:2.0:1616513091.594562:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513091.594565:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:2.0:1616513091.594567:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:2.0:1616513091.594572:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513091.594582:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513091.594597:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513091.594605:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513091.594608:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513091.594615:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513091.594623:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9642d00 x1693067725928432/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513091.594635:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eede48 00000800:00000200:27.0:1616513091.594641:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513091.594644:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513091.594797:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513091.594806:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513091.594812:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:3.0:1616513091.594817:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513091.594820:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513091.594829:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6277f0 00000400:00000200:27.0:1616513091.594836:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371670dd [1] + 192 00000400:00000200:27.0:1616513091.594842:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513091.594846:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9642d00 x1693067725928432/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513091.594862:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513091.594865:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513091.594866:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:2.0:1616513091.594871:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9642d00 x1693067725928432/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:2.0:1616513091.594884:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9642d00 x1693067725928432/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513098 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513091.594894:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eecb28 00000800:00000200:7.2:1616513091.596294:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513091.596347:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.596356:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.596365:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.223@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.596375:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.223@tcp1 of length 224 into portal 26 MB=0x6001580304d00 00000400:00000200:18.0:1616513091.596381:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.223@tcp1 of length 224/224 into md 0xf97ce179 [1] + 0 00000400:00000200:18.0:1616513091.596386:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42908 00000400:00000200:18.0:1616513091.596389:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.596392:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513091.596396:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42908 00000800:00000200:18.0:1616513091.596399:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.596400:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.596401:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.596407:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.596410:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.596411:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.596419:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.223@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.596429:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.223@tcp1 of length 224 into portal 12 MB=0x6001580304d80 00000400:00000200:14.0:1616513091.596450:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.223@tcp1 of length 224/224 into md 0xf964d519 [1] + 94168 00000400:00000200:14.0:1616513091.596455:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.596458:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513091.596464:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.596468:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.596469:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.596478:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688942205226240 00010000:00000200:0.0:1616513091.596490:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688942205226240/t0(0) o400->fa6e06a8-3068-70d7-a9ae-e3d3132748be@10.31.136.223@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.596506:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688942205226240, offset 192 00000400:00000200:0.0:1616513091.596512:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.223@tcp1 00000400:00000200:0.0:1616513091.596519:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.223@tcp1 routed destination 00000100:00000200:26.0:1616513091.596527:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688942205226368 00000400:00000200:0.0:1616513091.596534:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.223@tcp1(10.31.136.223@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:26.0:1616513091.596539:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01b00 x1688942205226368/t0(0) o400->d2ab37c8-a4f3-5e31-6a36-f36098e15346@10.31.136.223@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.596541:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.596546:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.596549:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.596551:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513091.596554:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688942205226368, offset 192 00000800:00000200:0.0:1616513091.596556:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513091.596560:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.223@tcp1 00000800:00000200:11.2:1616513091.596561:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.596563:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513091.596567:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.223@tcp1 routed destination 00000400:00000200:26.0:1616513091.596582:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.223@tcp1(10.31.136.223@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.596589:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.596595:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.596597:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.596600:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.596604:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.596609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513091.596614:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.596620:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513091.596623:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513091.596624:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513091.596629:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513091.596631:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.596633:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513091.596634:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513091.596635:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:16.0:1616513091.596638:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513091.596640:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513091.596641:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513091.596644:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af6d8 00000800:00000200:22.0:1616513091.596648:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.606269:0:5832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.606324:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.606333:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.606342:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.143@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.606352:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.143@tcp1 of length 224 into portal 26 MB=0x601859b589480 00000400:00000200:28.0:1616513091.606358:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.143@tcp1 of length 224/224 into md 0xf97ce189 [1] + 0 00000400:00000200:28.0:1616513091.606363:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42c38 00000400:00000200:28.0:1616513091.606366:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.606369:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513091.606373:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42c38 00000800:00000200:28.0:1616513091.606377:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.606377:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.606379:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.606381:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.606398:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690523208815744 00010000:00000200:0.0:1616513091.606409:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1690523208815744/t0(0) o400->bedfa7f6-fd29-787c-d91f-3d1703e1d454@10.31.140.143@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.606424:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690523208815744, offset 192 00000400:00000200:0.0:1616513091.606430:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.143@tcp1 00000400:00000200:0.0:1616513091.606451:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.143@tcp1 routed destination 00000400:00000200:0.0:1616513091.606466:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.143@tcp1(10.31.140.143@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.606473:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.606479:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.606482:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.606484:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.606489:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.606494:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.606496:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513091.606547:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.606556:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513091.606559:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513091.606565:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513091.606571:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:16.0:1616513091.606574:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.606577:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.606712:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513091.606765:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.606775:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.606784:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.143@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.606793:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.143@tcp1 of length 224 into portal 12 MB=0x601859b589680 00000400:00000200:2.0:1616513091.606799:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.143@tcp1 of length 224/224 into md 0xf964d519 [1] + 94392 00000400:00000200:2.0:1616513091.606804:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.606807:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513091.606814:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.606817:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.606818:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.606821:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.606877:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690523208816256 00010000:00000200:26.0:1616513091.606889:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5a00 x1690523208816256/t0(0) o400->c78117b0-b081-7fd7-61ae-82c794136bd7@10.31.140.143@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.606903:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690523208816256, offset 192 00000400:00000200:26.0:1616513091.606909:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.143@tcp1 00000400:00000200:26.0:1616513091.606917:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.143@tcp1 routed destination 00000400:00000200:26.0:1616513091.606932:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.143@tcp1(10.31.140.143@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.606939:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.606950:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.606953:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.606955:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.606960:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.606964:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.606975:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.606982:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.606984:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.606991:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.606995:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae088 00000800:00000200:18.0:1616513091.606999:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.607002:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.609843:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513091.609858:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.609866:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.609875:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.204@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.609883:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.204@tcp1 of length 224 into portal 26 MB=0x5ffe04a485440 00000400:00000200:2.0:1616513091.609889:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.204@tcp1 of length 224/224 into md 0xf97ce199 [1] + 0 00000400:00000200:2.0:1616513091.609893:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43430 00000400:00000200:2.0:1616513091.609896:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.609899:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513091.609903:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43430 00000800:00000200:2.0:1616513091.609905:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.609908:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.609910:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.609912:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.609966:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688713667564608 00010000:00000200:0.0:1616513091.609977:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688713667564608/t0(0) o400->99943782-e2fc-33b6-6bab-3f38eff0f0b1@10.31.142.204@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.609990:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688713667564608, offset 192 00000400:00000200:0.0:1616513091.609996:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.204@tcp1 00000400:00000200:0.0:1616513091.610003:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.204@tcp1 routed destination 00000400:00000200:0.0:1616513091.610019:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.204@tcp1(10.31.142.204@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.610025:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.610030:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.610033:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.610035:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.610039:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.610044:0:5832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.610045:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513091.610098:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.610105:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.610108:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.610114:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.610120:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:20.0:1616513091.610123:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.610126:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.610135:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513091.610185:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.610194:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.610202:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.204@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.610210:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.204@tcp1 of length 224 into portal 12 MB=0x5ffe04a486140 00000400:00000200:18.0:1616513091.610216:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.204@tcp1 of length 224/224 into md 0xf964d519 [1] + 94616 00000400:00000200:18.0:1616513091.610220:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.610223:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513091.610229:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.610232:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.610235:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.610238:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.610252:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688713667567936 00010000:00000200:26.0:1616513091.610263:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd702d00 x1688713667567936/t0(0) o400->aef79f64-cb92-53e7-2945-c13d80c8e4a0@10.31.142.204@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.610276:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688713667567936, offset 192 00000400:00000200:26.0:1616513091.610284:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.204@tcp1 00000400:00000200:26.0:1616513091.610291:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.204@tcp1 routed destination 00000400:00000200:26.0:1616513091.610305:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.204@tcp1(10.31.142.204@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.610311:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.610316:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.610319:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.610321:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.610325:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.610330:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.610342:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.610349:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.610352:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.610354:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513091.610360:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.610364:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af3a8 00000800:00000200:14.0:1616513091.610368:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.610670:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.610683:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.610692:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513091.610693:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.610698:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513091.610701:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.610702:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.71@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.610705:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.71@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.610713:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.71@tcp1 of length 224 into portal 12 MB=0x5ffdd7c963f80 00000400:00000200:22.0:1616513091.610714:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.71@tcp1 of length 224 into portal 26 MB=0x5ffdd7c963f00 00000400:00000200:18.0:1616513091.610718:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.71@tcp1 of length 224/224 into md 0xf964d519 [1] + 94840 00000400:00000200:18.0:1616513091.610722:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.610725:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.71@tcp1 of length 224/224 into md 0xf97ce1a9 [1] + 0 00000400:00000200:22.0:1616513091.610729:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42cc0 00000100:00000200:18.0:1616513091.610731:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:22.0:1616513091.610732:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.610736:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513091.610736:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.610738:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.610739:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513091.610740:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42cc0 00000800:00000200:22.0:1616513091.610745:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:26.0:1616513091.610747:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701626630016 00000800:00000200:22.0:1616513091.610748:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.610750:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:26.0:1616513091.610753:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd700900 x1688701626630016/t0(0) o400->6588c286-c3ca-ecea-7bf7-877bf5a4971e@10.31.136.71@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513091.610754:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701626629888 00010000:00000200:0.0:1616513091.610760:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688701626629888/t0(0) o400->f36c9818-6fae-50a9-4135-7894e9327ae4@10.31.136.71@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513091.610763:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701626630016, offset 192 00000400:00000200:26.0:1616513091.610768:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.71@tcp1 00000100:00000200:0.0:1616513091.610771:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701626629888, offset 192 00000400:00000200:26.0:1616513091.610773:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.71@tcp1 routed destination 00000400:00000200:0.0:1616513091.610775:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.71@tcp1 00000400:00000200:26.0:1616513091.610785:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.71@tcp1(10.31.136.71@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.610790:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513091.610792:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.71@tcp1 routed destination 00000800:00000200:26.0:1616513091.610795:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.610797:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513091.610799:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513091.610801:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513091.610805:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.71@tcp1(10.31.136.71@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513091.610807:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.610810:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513091.610815:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.610817:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.610819:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.610822:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513091.610826:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513091.610827:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.610836:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.610839:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.610841:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513091.610843:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513091.610847:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.610851:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04bb0 00000800:00000200:14.0:1616513091.610854:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513091.610860:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.610866:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513091.610869:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513091.610876:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513091.610880:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa08 00000800:00000200:16.0:1616513091.610884:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.610887:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.627402:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.627457:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.627467:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513091.627473:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.627477:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.95@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.627486:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.95@tcp1 of length 224 into portal 12 MB=0x604b950dde240 00000400:00000200:20.0:1616513091.627493:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.95@tcp1 of length 224/224 into md 0x865be291 [1] + 31688 00000400:00000200:20.0:1616513091.627498:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.627501:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.627509:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.627514:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.627516:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.627552:0:6562:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:12.0:1616513091.627576:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694043832443456 00010000:00000200:12.0:1616513091.627590:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0df080 x1694043832443456/t0(0) o400->6c95b6db-9f12-7407-b047-997a115a726c@10.31.143.95@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513091.627605:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513091.627605:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694043832443456, offset 192 00000400:00000200:12.0:1616513091.627612:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.95@tcp1 00000800:00000200:18.0:1616513091.627614:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513091.627620:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.95@tcp1 routed destination 00000400:00000200:18.0:1616513091.627623:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.95@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.627631:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.95@tcp1 of length 224 into portal 26 MB=0x604b950dde0c0 00000400:00000200:12.0:1616513091.627635:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.95@tcp1(10.31.143.95@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513091.627638:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.95@tcp1 of length 224/224 into md 0xf97ce1b9 [1] + 0 00000800:00000200:12.0:1616513091.627641:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513091.627642:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42330 00000400:00000200:18.0:1616513091.627645:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.627647:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:12.0:1616513091.627647:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.627650:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513091.627651:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42330 00000800:00000200:12.0:1616513091.627652:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513091.627654:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.627657:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513091.627657:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513091.627658:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.627661:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.627662:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.627677:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.627684:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513091.627687:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513091.627694:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513091.627698:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eff0 00000800:00000200:16.0:1616513091.627701:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.627704:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.627716:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694043832443072 00010000:00000200:0.0:1616513091.627727:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1694043832443072/t0(0) o400->62f18ca8-8e12-8716-351b-8b600aa35d85@10.31.143.95@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.627740:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694043832443072, offset 192 00000400:00000200:0.0:1616513091.627746:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.95@tcp1 00000400:00000200:0.0:1616513091.627753:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.95@tcp1 routed destination 00000400:00000200:0.0:1616513091.627767:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.95@tcp1(10.31.143.95@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.627773:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513091.627779:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.627784:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.627786:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.627791:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.627796:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513091.627797:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513091.627852:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.627859:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513091.627861:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513091.627867:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513091.627873:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafdc0 00000800:00000200:2.0:1616513091.627876:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.627879:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.633955:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513091.634009:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.634018:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.634026:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.48@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.634036:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.48@tcp1 of length 224 into portal 26 MB=0x6033cb2bbf280 00000400:00000200:18.0:1616513091.634042:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.48@tcp1 of length 224/224 into md 0xf97ce1d9 [1] + 0 00000400:00000200:18.0:1616513091.634046:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42660 00000400:00000200:18.0:1616513091.634049:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.634052:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513091.634058:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42660 00000800:00000200:18.0:1616513091.634061:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.634062:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.634063:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.634066:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.634122:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692409091846784 00010000:00000200:0.0:1616513091.634133:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1692409091846784/t0(0) o400->7db39798-c431-06e7-3db5-3b3a6ecf144f@10.31.138.48@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.634146:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692409091846784, offset 192 00000400:00000200:0.0:1616513091.634152:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.48@tcp1 00000400:00000200:0.0:1616513091.634159:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.48@tcp1 routed destination 00000400:00000200:0.0:1616513091.634173:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.48@tcp1(10.31.138.48@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.634179:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.634185:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.634188:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.634190:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.634194:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.634198:0:5832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.634201:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513091.634253:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.634261:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.634264:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.634271:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.634276:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafa90 00000800:00000200:28.0:1616513091.634280:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.634283:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.634346:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.634399:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.634409:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.634418:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.48@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.634427:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.48@tcp1 of length 224 into portal 12 MB=0x6033cb2bbf480 00000400:00000200:16.0:1616513091.634453:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.48@tcp1 of length 224/224 into md 0x865be291 [1] + 31912 00000400:00000200:16.0:1616513091.634459:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.634461:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.634467:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.634469:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.634472:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.634474:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.634491:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692409091847296 00010000:00000200:12.0:1616513091.634503:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd2880 x1692409091847296/t0(0) o400->11cafca1-9e11-4a82-47b3-6ca3db646940@10.31.138.48@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.634518:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692409091847296, offset 192 00000400:00000200:12.0:1616513091.634524:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.48@tcp1 00000400:00000200:12.0:1616513091.634532:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.48@tcp1 routed destination 00000400:00000200:12.0:1616513091.634547:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.48@tcp1(10.31.138.48@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.634553:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.634560:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.634562:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.634565:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.634570:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.634574:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.634625:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.634631:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513091.634634:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513091.634640:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513091.634643:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:2.0:1616513091.634647:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.634649:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.662166:0:50714:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.662226:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.662236:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513091.662246:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.86@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.662255:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.86@tcp1 of length 224 into portal 26 MB=0x5ffdd521ca0c0 00000400:00000200:22.0:1616513091.662262:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.86@tcp1 of length 224/224 into md 0xf97ce1f9 [1] + 0 00000400:00000200:22.0:1616513091.662267:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a43f58 00000400:00000200:22.0:1616513091.662270:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.662273:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513091.662277:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a43f58 00000800:00000200:18.0:1616513091.662280:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.662281:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.662284:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513091.662286:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.662289:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.662289:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513091.662297:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.86@tcp1 : PUT - for me 00000400:00000200:18.0:1616513091.662305:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.86@tcp1 of length 224 into portal 12 MB=0x5ffdd521ca180 00000400:00000200:18.0:1616513091.662311:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.86@tcp1 of length 224/224 into md 0xf964d519 [1] + 95064 00000400:00000200:18.0:1616513091.662315:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.662318:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513091.662325:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.662328:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.662330:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.662342:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700914016448 00010000:00000200:0.0:1616513091.662353:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688700914016448/t0(0) o400->dc11262b-f1d7-7af7-6f43-0407ae59bd3d@10.31.142.86@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.662367:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700914016448, offset 192 00000400:00000200:0.0:1616513091.662372:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.86@tcp1 00000400:00000200:0.0:1616513091.662379:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.86@tcp1 routed destination 00000100:00000200:26.0:1616513091.662389:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700914016640 00000400:00000200:0.0:1616513091.662394:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.86@tcp1(10.31.142.86@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.662400:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:26.0:1616513091.662401:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd701680 x1688700914016640/t0(0) o400->1367e22a-8d7d-1d58-ce03-c280907a454b@10.31.142.86@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.662405:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.662408:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.662410:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513091.662414:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700914016640, offset 192 00000800:00000200:0.0:1616513091.662415:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513091.662419:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.86@tcp1 00000800:00000200:11.2:1616513091.662420:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.662422:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513091.662426:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.86@tcp1 routed destination 00000400:00000200:26.0:1616513091.662452:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.86@tcp1(10.31.142.86@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.662458:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.662464:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.662468:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.662470:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513091.662475:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513091.662475:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.662479:0:50714:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513091.662483:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.662486:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.662492:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.662498:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf760 00000800:00000200:28.0:1616513091.662502:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.662505:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513091.662552:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.662560:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.662563:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513091.662564:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513091.662569:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.662573:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af870 00000800:00000200:14.0:1616513091.662577:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.670933:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513091.670989:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.670999:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.671007:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.192@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.671017:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.192@tcp1 of length 224 into portal 26 MB=0x5fffee9e0cb00 00000400:00000200:24.0:1616513091.671023:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.192@tcp1 of length 224/224 into md 0xf97ce209 [1] + 0 00000400:00000200:24.0:1616513091.671028:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42bb0 00000400:00000200:24.0:1616513091.671031:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.671033:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513091.671038:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42bb0 00000800:00000200:24.0:1616513091.671041:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.671044:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513091.671044:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.671049:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.671101:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688845194152704 00000800:00000200:11.2:1616513091.671113:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:0.0:1616513091.671113:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688845194152704/t0(0) o400->4c179e76-a76a-a491-a1e5-72b8425e0f21@10.31.142.192@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.671127:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688845194152704, offset 192 00000400:00000200:0.0:1616513091.671133:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.192@tcp1 00000400:00000200:0.0:1616513091.671141:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.192@tcp1 routed destination 00000400:00000200:0.0:1616513091.671156:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.192@tcp1(10.31.142.192@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.671162:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.671168:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513091.671170:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.671171:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.671173:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513091.671178:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513091.671179:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513091.671184:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513091.671188:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.192@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.671197:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.192@tcp1 of length 224 into portal 12 MB=0x5fffee9e0d700 00000400:00000200:28.0:1616513091.671203:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.192@tcp1 of length 224/224 into md 0x865be291 [1] + 32136 00000400:00000200:28.0:1616513091.671208:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.671211:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513091.671218:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513091.671221:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513091.671223:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.671226:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.671234:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.671237:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.671238:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513091.671242:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688845194155776 00000400:00000200:20.0:1616513091.671244:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.671249:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf188 00000800:00000200:20.0:1616513091.671253:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:12.0:1616513091.671254:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedf080 x1688845194155776/t0(0) o400->198a7d33-68a2-4315-c38f-1f01137f3596@10.31.142.192@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.671279:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688845194155776, offset 192 00000400:00000200:12.0:1616513091.671285:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.192@tcp1 00000400:00000200:12.0:1616513091.671293:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.192@tcp1 routed destination 00000400:00000200:12.0:1616513091.671307:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.192@tcp1(10.31.142.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.671314:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.671320:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.671322:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.671324:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.671329:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.671336:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.671391:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.671398:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.671400:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.671407:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.671411:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e440 00000800:00000200:18.0:1616513091.671415:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.671418:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.675334:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.675390:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.675399:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513091.675408:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.230@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.675418:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.230@tcp1 of length 224 into portal 26 MB=0x603e45273fc00 00000400:00000200:22.0:1616513091.675425:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.230@tcp1 of length 224/224 into md 0xf97ce219 [1] + 0 00000400:00000200:22.0:1616513091.675429:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d5645a42000 00000400:00000200:22.0:1616513091.675432:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513091.675446:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513091.675456:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513091.675460:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5645a42000 00000800:00000200:22.0:1616513091.675463:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.675466:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.675468:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.675477:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513091.675524:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693129031023616 00000800:00000200:24.0:1616513091.675534:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:0.0:1616513091.675535:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1693129031023616/t0(0) o400->052d2873-22f7-b71d-f5aa-614131cea104@10.31.136.230@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:24.0:1616513091.675543:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513091.675549:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693129031023616, offset 192 00000400:00000200:24.0:1616513091.675552:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.230@tcp1 : PUT - for me 00000400:00000200:0.0:1616513091.675557:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.230@tcp1 00000400:00000200:24.0:1616513091.675561:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.230@tcp1 of length 224 into portal 12 MB=0x603e45273fcc0 00000400:00000200:0.0:1616513091.675564:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.230@tcp1 routed destination 00000400:00000200:24.0:1616513091.675567:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.230@tcp1 of length 224/224 into md 0x865be291 [1] + 32360 00000400:00000200:24.0:1616513091.675573:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.675575:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513091.675579:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.230@tcp1(10.31.136.230@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513091.675585:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513091.675586:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513091.675588:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.675589:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.675592:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513091.675592:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.675594:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.675597:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.675601:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513091.675603:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693129031023808 00000800:00000200:11.2:1616513091.675605:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.675608:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:12.0:1616513091.675616:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e5580 x1693129031023808/t0(0) o400->7c2f102e-f499-5ca2-6441-2e2078904b9a@10.31.136.230@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.675630:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693129031023808, offset 192 00000400:00000200:12.0:1616513091.675636:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.230@tcp1 00000400:00000200:12.0:1616513091.675644:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.230@tcp1 routed destination 00000800:00000200:16.0:1616513091.675657:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513091.675659:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.230@tcp1(10.31.136.230@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513091.675664:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513091.675665:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513091.675668:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513091.675671:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.675674:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513091.675675:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:12.0:1616513091.675676:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513091.675681:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae220 00000800:00000200:12.0:1616513091.675681:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513091.675684:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513091.675686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513091.675687:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513091.675700:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.675707:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.675710:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.675716:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.675720:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ec38 00000800:00000200:18.0:1616513091.675724:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.675727:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.679537:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.679590:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.679599:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.679609:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.92@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.679618:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.92@tcp1 of length 224 into portal 26 MB=0x5ffdd322b23c0 00000400:00000200:14.0:1616513091.679625:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.92@tcp1 of length 224/224 into md 0xf97ce229 [1] + 0 00000400:00000200:14.0:1616513091.679630:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04880 00000400:00000200:14.0:1616513091.679633:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.679635:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513091.679640:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04880 00000800:00000200:14.0:1616513091.679643:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.679644:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.679645:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513091.679648:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.679702:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700378096576 00010000:00000200:0.0:1616513091.679712:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688700378096576/t0(0) o400->ef7a8992-afa4-de15-5395-866c2a92d151@10.31.128.92@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.679725:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700378096576, offset 192 00000400:00000200:0.0:1616513091.679732:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.92@tcp1 00000400:00000200:0.0:1616513091.679739:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.92@tcp1 routed destination 00000400:00000200:0.0:1616513091.679753:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.92@tcp1(10.31.128.92@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.679759:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.679764:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.679767:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.679769:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.679773:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.679779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.679780:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513091.679826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513091.679834:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.679840:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.679843:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.679849:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.679855:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf650 00000800:00000200:28.0:1616513091.679858:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.679861:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.679867:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.679874:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.110@tcp1 : PUT - for me 00000800:00000200:18.0:1616513091.679877:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513091.679883:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.110@tcp1 of length 224 into portal 12 MB=0x5fdb00abbb080 00000800:00000200:18.0:1616513091.679886:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513091.679889:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.110@tcp1 of length 224/224 into md 0x865be291 [1] + 32584 00000800:00000200:24.0:1616513091.679893:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513091.679894:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.110@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.679899:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.679901:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513091.679903:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.110@tcp1 of length 224 into portal 26 MB=0x5fdb00abbb000 00000800:00000200:28.0:1616513091.679907:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513091.679909:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.110@tcp1 of length 224/224 into md 0xf97ce259 [1] + 0 00000800:00000200:28.0:1616513091.679910:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.679911:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513091.679918:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05100 00000400:00000200:18.0:1616513091.679920:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513091.679923:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513091.679927:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05100 00000100:00000200:12.0:1616513091.679927:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1686307419697280 00000800:00000200:18.0:1616513091.679930:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.679932:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.679933:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513091.679939:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00010000:00000200:12.0:1616513091.679939:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedd580 x1686307419697280/t0(0) o400->897fda83-ff83-1335-3e7b-d215d9ffec9a@10.31.143.110@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513091.679940:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1686307419697152 00000800:00000200:18.0:1616513091.679942:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.679943:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:0.0:1616513091.679946:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1686307419697152/t0(0) o400->5828b0ac-0eef-ee0d-9d74-8601536988c9@10.31.143.110@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513091.679952:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1686307419697280, offset 192 00000400:00000200:2.0:1616513091.679952:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.92@tcp1 : PUT - for me 00000400:00000200:12.0:1616513091.679958:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.110@tcp1 00000400:00000200:2.0:1616513091.679960:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.92@tcp1 of length 224 into portal 12 MB=0x5ffdd322b2740 00000100:00000200:0.0:1616513091.679960:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1686307419697152, offset 192 00000400:00000200:12.0:1616513091.679965:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.110@tcp1 routed destination 00000400:00000200:0.0:1616513091.679965:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.110@tcp1 00000400:00000200:2.0:1616513091.679967:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.92@tcp1 of length 224/224 into md 0xf964d519 [1] + 95288 00000400:00000200:2.0:1616513091.679972:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:0.0:1616513091.679973:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.110@tcp1 routed destination 00000100:00000200:2.0:1616513091.679975:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513091.679979:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.110@tcp1(10.31.143.110@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513091.679986:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513091.679986:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.110@tcp1(10.31.143.110@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.679987:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513091.679989:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.679991:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513091.679991:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513091.679992:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.679995:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.679996:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.679997:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.679998:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.680000:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513091.680001:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513091.680002:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513091.680006:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513091.680007:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.680007:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513091.680015:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.680019:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.680021:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513091.680023:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513091.680027:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.680030:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f188 00000800:00000200:14.0:1616513091.680036:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.680042:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700378097472 00010000:00000200:26.0:1616513091.680052:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d1680 x1688700378097472/t0(0) o400->da196e83-c34b-40d9-7571-6ec4375de116@10.31.128.92@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513091.680063:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:26.0:1616513091.680064:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700378097472, offset 192 00000400:00000200:26.0:1616513091.680069:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.92@tcp1 00000800:00000200:20.0:1616513091.680070:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.680072:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513091.680073:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513091.680074:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.92@tcp1 routed destination 00000400:00000200:20.0:1616513091.680079:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.680085:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf540 00000400:00000200:26.0:1616513091.680086:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.92@tcp1(10.31.128.92@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513091.680088:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513091.680091:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.680096:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.680098:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.680100:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.680104:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.680109:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.680119:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.680124:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.680128:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513091.680129:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.680135:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.680139:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afdc0 00000800:00000200:22.0:1616513091.680143:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.704620:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.704674:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.704684:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.704694:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.23@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.704704:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.23@tcp1 of length 224 into portal 26 MB=0x5ffdc604db5c0 00000400:00000200:16.0:1616513091.704711:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.23@tcp1 of length 224/224 into md 0xf97ce279 [1] + 0 00000400:00000200:16.0:1616513091.704716:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04ee0 00000400:00000200:16.0:1616513091.704719:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.704722:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513091.704727:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04ee0 00000800:00000200:28.0:1616513091.704728:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.704730:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.704733:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513091.704735:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513091.704738:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.704739:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.704748:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.23@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.704757:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.23@tcp1 of length 224 into portal 12 MB=0x5ffdc604db640 00000400:00000200:28.0:1616513091.704764:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.23@tcp1 of length 224/224 into md 0x865be291 [1] + 32808 00000400:00000200:28.0:1616513091.704769:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.704772:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513091.704779:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.704782:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.704784:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.704791:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696857146816 00010000:00000200:0.0:1616513091.704802:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688696857146816/t0(0) o400->61b4f832-4e76-eefc-d50f-0342a1a7159c@10.31.143.23@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.704815:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696857146816, offset 192 00000400:00000200:0.0:1616513091.704821:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.23@tcp1 00000400:00000200:0.0:1616513091.704829:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.23@tcp1 routed destination 00000100:00000200:12.0:1616513091.704841:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696857146944 00000400:00000200:0.0:1616513091.704843:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.23@tcp1(10.31.143.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.704849:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513091.704853:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed8480 x1688696857146944/t0(0) o400->28bc00db-ce1a-4364-81f5-030a9109e61f@10.31.143.23@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.704856:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.704858:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.704860:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.704865:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513091.704866:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696857146944, offset 192 00000800:00000200:11.2:1616513091.704870:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513091.704872:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.23@tcp1 00000100:00000200:0.0:1616513091.704872:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513091.704879:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.23@tcp1 routed destination 00000400:00000200:12.0:1616513091.704893:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.23@tcp1(10.31.143.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.704899:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.704905:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.704908:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.704910:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.704915:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.704921:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513091.704923:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.704930:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.704933:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.704933:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.704940:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.704945:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae550 00000800:00000200:20.0:1616513091.704949:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513091.704974:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.704980:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.704983:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.704990:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.704993:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f100 00000800:00000200:18.0:1616513091.704997:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.705000:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.723146:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.723202:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.723212:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.723221:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.181@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.723231:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.181@tcp1 of length 224 into portal 26 MB=0x5ffd4588d4ec0 00000400:00000200:28.0:1616513091.723238:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.181@tcp1 of length 224/224 into md 0xf97ce289 [1] + 0 00000400:00000200:28.0:1616513091.723243:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb042a8 00000400:00000200:28.0:1616513091.723246:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.723249:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513091.723253:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb042a8 00000800:00000200:28.0:1616513091.723256:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.723257:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.723259:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.723264:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.723267:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.723267:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.723278:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.181@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.723288:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.181@tcp1 of length 224 into portal 12 MB=0x5ffd4588d5300 00000400:00000200:24.0:1616513091.723294:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.181@tcp1 of length 224/224 into md 0x865be291 [1] + 33032 00000400:00000200:24.0:1616513091.723300:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.723302:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.723309:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.723312:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.723314:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.723321:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662367358656 00010000:00000200:0.0:1616513091.723333:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688662367358656/t0(0) o400->b82bf216-417c-8669-bc66-dd0e37f65758@10.31.136.181@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.723347:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662367358656, offset 192 00000400:00000200:0.0:1616513091.723354:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.181@tcp1 00000400:00000200:0.0:1616513091.723362:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.181@tcp1 routed destination 00000100:00000200:12.0:1616513091.723373:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662367359744 00000400:00000200:0.0:1616513091.723376:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.181@tcp1(10.31.136.181@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.723383:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513091.723387:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e6300 x1688662367359744/t0(0) o400->4524a749-d2ab-f95f-a012-b4d4f179268b@10.31.136.181@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.723389:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.723392:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.723394:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.723399:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513091.723401:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662367359744, offset 192 00000800:00000200:11.2:1616513091.723403:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.723406:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513091.723407:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.181@tcp1 00000800:00000200:20.0:1616513091.723414:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513091.723415:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.181@tcp1 routed destination 00000800:00000200:20.0:1616513091.723422:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.723424:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513091.723424:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513091.723430:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.181@tcp1(10.31.136.181@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513091.723431:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.723458:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae3b8 00000800:00000200:20.0:1616513091.723462:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513091.723463:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.723469:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.723471:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.723474:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.723479:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.723484:0:50721:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.723540:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.723547:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513091.723550:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513091.723557:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513091.723561:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e770 00000800:00000200:2.0:1616513091.723565:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.723568:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.723574:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513091.723576:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.723578:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.723579:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.723582:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.746650:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513091.746705:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.746715:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.746724:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.59@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.746734:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.59@tcp1 of length 224 into portal 26 MB=0x6019aa3101f40 00000400:00000200:24.0:1616513091.746741:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.59@tcp1 of length 224/224 into md 0xf97ce2a9 [1] + 0 00000400:00000200:24.0:1616513091.746746:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05870 00000400:00000200:24.0:1616513091.746749:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.746752:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513091.746756:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05870 00000800:00000200:16.0:1616513091.746759:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.746760:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.746762:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.746764:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.746822:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690613532598080 00010000:00000200:0.0:1616513091.746834:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1690613532598080/t0(0) o400->efbd7a37-e4ce-ebf7-9ea6-7e62fa388310@10.31.141.59@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.746849:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690613532598080, offset 192 00000400:00000200:0.0:1616513091.746855:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.59@tcp1 00000400:00000200:0.0:1616513091.746863:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.59@tcp1 routed destination 00000400:00000200:0.0:1616513091.746879:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.59@tcp1(10.31.141.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.746885:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.746891:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.746894:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.746896:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.746901:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.746905:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.746910:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513091.746958:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.746965:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.746968:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513091.746970:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513091.746974:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.746981:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf980 00000800:00000200:28.0:1616513091.746985:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.747381:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.747391:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.747399:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513091.747401:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513091.747409:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.59@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.747418:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.59@tcp1 of length 224 into portal 12 MB=0x6019aa3102b00 00000400:00000200:16.0:1616513091.747424:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.59@tcp1 of length 224/224 into md 0x865be291 [1] + 33256 00000400:00000200:16.0:1616513091.747429:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.747432:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513091.747451:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.747454:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.747456:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.747516:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690613532601088 00010000:00000200:12.0:1616513091.747529:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affd4800 x1690613532601088/t0(0) o400->cd812888-adde-4ffb-a51d-5831a473b86f@10.31.141.59@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.747544:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690613532601088, offset 192 00000400:00000200:12.0:1616513091.747550:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.59@tcp1 00000400:00000200:12.0:1616513091.747557:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.59@tcp1 routed destination 00000400:00000200:12.0:1616513091.747572:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.59@tcp1(10.31.141.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.747579:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.747585:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.747587:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.747590:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.747595:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.747600:0:50724:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.747658:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.747665:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.747674:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513091.747681:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.747685:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e550 00000800:00000200:14.0:1616513091.747689:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.747692:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.765331:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.765385:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.765395:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.765403:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.117@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.765413:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.117@tcp1 of length 224 into portal 26 MB=0x60055051e5800 00000400:00000200:20.0:1616513091.765420:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.117@tcp1 of length 224/224 into md 0xf97ce2b9 [1] + 0 00000400:00000200:20.0:1616513091.765425:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05cb0 00000400:00000200:20.0:1616513091.765428:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.765432:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513091.765445:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513091.765445:0:50727:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513091.765451:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05cb0 00000800:00000200:20.0:1616513091.765455:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.765458:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.765460:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513091.765501:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.765511:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513091.765521:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.117@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.765530:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.117@tcp1 of length 224 into portal 12 MB=0x60055051e5bc0 00000100:00000200:0.0:1616513091.765533:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689215018358784 00000400:00000200:22.0:1616513091.765537:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.117@tcp1 of length 224/224 into md 0xf964d519 [1] + 95512 00000400:00000200:22.0:1616513091.765542:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00010000:00000200:0.0:1616513091.765543:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1689215018358784/t0(0) o400->ca3255e5-e393-0c8e-52a2-8529dd78d2c9@10.31.141.117@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:22.0:1616513091.765545:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513091.765553:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.765556:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513091.765557:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513091.765557:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689215018358784, offset 192 00000800:00000200:18.0:1616513091.765560:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513091.765563:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.117@tcp1 00000400:00000200:0.0:1616513091.765570:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.117@tcp1 routed destination 00000400:00000200:0.0:1616513091.765584:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.117@tcp1(10.31.141.117@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.765591:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.765596:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.765598:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.765600:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.765606:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.765610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.765612:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513091.765617:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689215018359744 00010000:00000200:26.0:1616513091.765630:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2a400 x1689215018359744/t0(0) o400->c9536446-939b-5c2b-45e8-e22552fa6e80@10.31.141.117@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.765645:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689215018359744, offset 192 00000400:00000200:26.0:1616513091.765651:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.117@tcp1 00000400:00000200:26.0:1616513091.765659:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.117@tcp1 routed destination 00000800:00000200:24.0:1616513091.765664:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.765671:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513091.765673:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.117@tcp1(10.31.141.117@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513091.765674:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513091.765680:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:24.0:1616513091.765680:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513091.765686:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513091.765686:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafed0 00000800:00000200:26.0:1616513091.765689:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513091.765689:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513091.765691:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513091.765692:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513091.765696:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.765700:0:50727:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.765761:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.765769:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.765771:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513091.765778:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513091.765782:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af4b8 00000800:00000200:14.0:1616513091.765786:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.765789:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.767322:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.767376:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.767385:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.767394:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.124@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.767403:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.124@tcp1 of length 224 into portal 26 MB=0x5ffd4effba780 00000400:00000200:28.0:1616513091.767410:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.124@tcp1 of length 224/224 into md 0xf97ce2c9 [1] + 0 00000400:00000200:28.0:1616513091.767415:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04198 00000400:00000200:28.0:1616513091.767417:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.767420:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513091.767424:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04198 00000800:00000200:28.0:1616513091.767427:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.767429:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.767430:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.767433:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.767488:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664907949952 00010000:00000200:0.0:1616513091.767499:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688664907949952/t0(0) o400->85f576a3-ede2-4424-5506-15f2aca4fc82@10.31.132.124@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.767513:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664907949952, offset 192 00000400:00000200:0.0:1616513091.767519:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.124@tcp1 00000400:00000200:0.0:1616513091.767528:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.124@tcp1 routed destination 00000400:00000200:0.0:1616513091.767542:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.124@tcp1(10.31.132.124@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.767549:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.767554:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.767557:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.767559:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.767564:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.767568:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.767571:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513091.767626:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.767633:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.767635:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.767642:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.767647:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faeff0 00000800:00000200:24.0:1616513091.767653:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.767656:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513091.768591:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513091.768602:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.768610:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.768619:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.124@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.768628:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.124@tcp1 of length 224 into portal 12 MB=0x5ffd4effba980 00000400:00000200:20.0:1616513091.768635:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.124@tcp1 of length 224/224 into md 0x865be291 [1] + 33480 00000400:00000200:20.0:1616513091.768640:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.768642:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513091.768649:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.768652:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.768653:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.768657:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.768712:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664907950464 00010000:00000200:12.0:1616513091.768724:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dc800 x1688664907950464/t0(0) o400->a038d71d-e571-56ed-e77b-36255396fbde@10.31.132.124@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.768739:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664907950464, offset 192 00000400:00000200:12.0:1616513091.768745:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.124@tcp1 00000400:00000200:12.0:1616513091.768753:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.124@tcp1 routed destination 00000400:00000200:12.0:1616513091.768768:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.124@tcp1(10.31.132.124@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.768774:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513091.768780:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.768783:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.768785:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513091.768790:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.768795:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.768849:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.768856:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.768858:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513091.768865:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.768869:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f8f8 00000800:00000200:22.0:1616513091.768873:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.768876:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.778047:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.778101:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.778111:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.778120:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.159@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.778129:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.159@tcp1 of length 224 into portal 26 MB=0x5ffd58621a480 00000400:00000200:14.0:1616513091.778136:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.159@tcp1 of length 224/224 into md 0xf97ce2d9 [1] + 0 00000400:00000200:14.0:1616513091.778140:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb057e8 00000400:00000200:14.0:1616513091.778143:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.778146:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513091.778150:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb057e8 00000800:00000200:14.0:1616513091.778153:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.778155:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.778156:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513091.778159:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.778213:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667427021952 00000800:00000200:11.2:1616513091.778219:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:0.0:1616513091.778224:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688667427021952/t0(0) o400->c3bc8d36-d50a-f9c0-ec0d-5c83fac4b217@10.31.138.159@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.778239:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667427021952, offset 192 00000400:00000200:0.0:1616513091.778247:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.159@tcp1 00000400:00000200:0.0:1616513091.778254:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.159@tcp1 routed destination 00000400:00000200:0.0:1616513091.778269:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.159@tcp1(10.31.138.159@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.778275:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513091.778280:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.778281:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.778284:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513091.778286:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513091.778289:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.778297:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.159@tcp1 : PUT - for me 00000800:00000200:0.0:1616513091.778299:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:0.0:1616513091.778305:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513091.778306:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.159@tcp1 of length 224 into portal 12 MB=0x5ffd58621a500 00000400:00000200:24.0:1616513091.778312:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.159@tcp1 of length 224/224 into md 0x865be291 [1] + 33704 00000400:00000200:24.0:1616513091.778317:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.778320:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.778326:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513091.778329:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513091.778331:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.778333:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.778337:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.778339:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513091.778339:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.778344:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.778349:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae770 00000800:00000200:24.0:1616513091.778352:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.778390:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667427022080 00010000:00000200:12.0:1616513091.778402:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e6c00 x1688667427022080/t0(0) o400->f49d1c3c-c710-20fd-8ef3-429dd7d48871@10.31.138.159@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513091.778417:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667427022080, offset 192 00000400:00000200:12.0:1616513091.778423:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.159@tcp1 00000800:00000200:11.2:1616513091.778426:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:12.0:1616513091.778430:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.159@tcp1 routed destination 00000800:00000200:20.0:1616513091.778444:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.778454:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513091.778455:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513091.778459:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.159@tcp1(10.31.138.159@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513091.778462:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.180@tcp1 : PUT - for me 00000800:00000200:12.0:1616513091.778466:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513091.778472:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.180@tcp1 of length 224 into portal 26 MB=0x6001bc4489c80 00000800:00000200:12.0:1616513091.778472:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.778475:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513091.778477:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513091.778478:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.180@tcp1 of length 224/224 into md 0xf97ce309 [1] + 0 00000800:00000200:12.0:1616513091.778482:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513091.778483:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb053a8 00000400:00000200:20.0:1616513091.778485:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:7.2:1616513091.778486:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513091.778488:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513091.778492:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb053a8 00000800:00000200:20.0:1616513091.778496:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.778498:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.778500:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.778507:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969117473920 00010000:00000200:0.0:1616513091.778513:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688969117473920/t0(0) o400->92a58925-9817-3b0a-ca62-00f42701b264@10.31.128.180@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.778524:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688969117473920, offset 192 00000400:00000200:0.0:1616513091.778530:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.180@tcp1 00000400:00000200:0.0:1616513091.778536:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.180@tcp1 routed destination 00000800:00000200:22.0:1616513091.778540:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.778547:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.778550:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513091.778550:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.180@tcp1(10.31.128.180@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.778555:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513091.778557:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513091.778560:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513091.778561:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f3a8 00000800:00000200:0.0:1616513091.778563:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.778564:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513091.778565:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.778568:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513091.778568:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.778571:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.778572:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513091.778580:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.778583:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.778585:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513091.778589:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513091.778590:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.778593:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faef68 00000800:00000200:24.0:1616513091.778595:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.778629:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513091.778639:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.778647:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513091.778649:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513091.778656:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.180@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.778667:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.180@tcp1 of length 224 into portal 12 MB=0x6001bc448a0c0 00000400:00000200:2.0:1616513091.778673:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.180@tcp1 of length 224/224 into md 0xf964d519 [1] + 95736 00000400:00000200:2.0:1616513091.778679:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.778681:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513091.778688:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.778691:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.778693:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.778750:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969117475008 00010000:00000200:26.0:1616513091.778761:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d1b00 x1688969117475008/t0(0) o400->5ddffc0f-bcb9-8276-1e33-721fa6df31ba@10.31.128.180@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.778785:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688969117475008, offset 192 00000400:00000200:26.0:1616513091.778790:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.180@tcp1 00000400:00000200:26.0:1616513091.778797:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.180@tcp1 routed destination 00000400:00000200:26.0:1616513091.778811:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.180@tcp1(10.31.128.180@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.778817:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.778823:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.778825:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.778827:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.778839:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.778839:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.778848:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.778852:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.778854:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513091.778857:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513091.778859:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.778862:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae550 00000800:00000200:22.0:1616513091.778864:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.794457:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.794512:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.794521:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.794530:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.181@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.794539:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.181@tcp1 of length 224 into portal 26 MB=0x5ffe3d6e275c0 00000400:00000200:14.0:1616513091.794546:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.181@tcp1 of length 224/224 into md 0xf97ce339 [1] + 0 00000400:00000200:14.0:1616513091.794556:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04330 00000400:00000200:14.0:1616513091.794558:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.794561:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513091.794565:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04330 00000800:00000200:2.0:1616513091.794566:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.794568:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.794571:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513091.794573:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513091.794576:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.794577:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.794586:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.181@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.794594:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.181@tcp1 of length 224 into portal 12 MB=0x5ffe3d6e27840 00000400:00000200:2.0:1616513091.794601:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.181@tcp1 of length 224/224 into md 0xf964d519 [1] + 95960 00000400:00000200:2.0:1616513091.794606:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.794608:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513091.794615:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.794619:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.794621:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.794631:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728911377856 00010000:00000200:0.0:1616513091.794642:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688728911377856/t0(0) o400->d0cdf219-1b98-5503-0cd0-a2fe9dedca60@10.31.141.181@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.794656:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688728911377856, offset 192 00000400:00000200:0.0:1616513091.794662:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.181@tcp1 00000400:00000200:0.0:1616513091.794669:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.181@tcp1 routed destination 00000100:00000200:26.0:1616513091.794679:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688728911378496 00000400:00000200:0.0:1616513091.794684:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.181@tcp1(10.31.141.181@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:26.0:1616513091.794692:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4800 x1688728911378496/t0(0) o400->41f2213d-82a6-da2e-076d-e6cfa1d1acd1@10.31.141.181@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513091.794695:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.794701:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.794703:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513091.794706:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688728911378496, offset 192 00000800:00000200:0.0:1616513091.794706:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.794710:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513091.794712:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.181@tcp1 00000800:00000200:11.2:1616513091.794715:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.794717:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513091.794719:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.181@tcp1 routed destination 00000400:00000200:26.0:1616513091.794746:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.181@tcp1(10.31.141.181@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.794752:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.794757:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.794760:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.794762:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.794766:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.794771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513091.794783:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.794796:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513091.794799:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513091.794806:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513091.794811:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafb18 00000800:00000200:16.0:1616513091.794815:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513091.794818:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513091.794823:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.794830:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513091.794833:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513091.794834:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513091.794839:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.794843:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af7e8 00000800:00000200:22.0:1616513091.794847:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.795886:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513091.795938:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.795947:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.795956:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.170@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.795966:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.170@tcp1 of length 224 into portal 26 MB=0x60185c7a96700 00000400:00000200:2.0:1616513091.795972:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.170@tcp1 of length 224/224 into md 0xf97ce349 [1] + 0 00000400:00000200:2.0:1616513091.795977:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05ba0 00000400:00000200:2.0:1616513091.795980:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513091.795983:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513091.795987:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ba0 00000800:00000200:2.0:1616513091.795990:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.795991:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.795992:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.795994:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.796060:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690523952310016 00010000:00000200:0.0:1616513091.796071:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1690523952310016/t0(0) o400->a37d0c70-43e9-074a-0843-94765e538b98@10.31.129.170@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.796085:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690523952310016, offset 192 00000800:00000200:7.2:1616513091.796090:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513091.796091:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.170@tcp1 00000400:00000200:0.0:1616513091.796099:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.170@tcp1 routed destination 00000400:00000200:0.0:1616513091.796114:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.170@tcp1(10.31.129.170@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.796121:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.796126:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.796129:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.796131:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.796136:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.796141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.796142:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513091.796144:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.796153:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.796162:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.170@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.796170:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.170@tcp1 of length 224 into portal 12 MB=0x60185c7a96b40 00000400:00000200:14.0:1616513091.796177:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.170@tcp1 of length 224/224 into md 0xf964d519 [1] + 96184 00000400:00000200:14.0:1616513091.796182:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8441eaa110 00000400:00000200:14.0:1616513091.796184:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.796187:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513091.796191:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8441eaa110 00000800:00000200:28.0:1616513091.796193:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513091.796194:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.796197:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513091.796199:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513091.796199:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513091.796202:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513091.796202:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513091.796205:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513091.796208:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.796213:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae110 00000800:00000200:28.0:1616513091.796216:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513091.796254:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690523952311104 00010000:00000200:26.0:1616513091.796265:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a20e850 x1690523952311104/t0(0) o400->b7ad6a1c-572b-d41b-d386-756ab93f10ab@10.31.129.170@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.796279:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690523952311104, offset 192 00000400:00000200:26.0:1616513091.796285:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.170@tcp1 00000400:00000200:26.0:1616513091.796292:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.170@tcp1 routed destination 00000400:00000200:26.0:1616513091.796306:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.170@tcp1(10.31.129.170@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.796313:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.796319:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.796321:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.796323:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.796328:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.796332:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513091.796342:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.796347:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.796349:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513091.796351:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513091.796356:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.796360:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae110 00000800:00000200:18.0:1616513091.796364:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.796708:0:184144:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 12 00000800:00000200:11.2:1616513091.820768:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513091.820822:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513091.820832:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513091.820841:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:16.0:1616513091.820850:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x603991341a8c0 00000400:00000200:16.0:1616513091.820856:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0x865be291 [1] + 33928 00000400:00000200:16.0:1616513091.820862:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513091.820864:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513091.820872:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.820875:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.820877:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513091.820881:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513091.820936:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805848213696 00010000:00000200:12.0:1616513091.821006:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b194c80 x1692805848213696/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:717/0 lens 336/336 e 0 to 0 dl 1616513097 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513091.821021:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805848213696, offset 192 00000400:00000200:12.0:1616513091.821027:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:12.0:1616513091.821035:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:12.0:1616513091.821050:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.821057:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513091.821062:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.821065:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513091.821067:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513091.821072:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.821076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.821087:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.821094:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513091.821097:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.821104:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.821108:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f7e8 00000800:00000200:24.0:1616513091.821112:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.821115:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.847230:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.847284:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.847294:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.847303:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.27@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.847312:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.27@tcp1 of length 224 into portal 26 MB=0x600bd38f660c0 00000400:00000200:14.0:1616513091.847319:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.27@tcp1 of length 224/224 into md 0xf97ce359 [1] + 0 00000400:00000200:14.0:1616513091.847323:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05e48 00000400:00000200:14.0:1616513091.847326:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.847329:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513091.847333:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05e48 00000800:00000200:14.0:1616513091.847336:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513091.847339:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.847339:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513091.847343:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513091.847405:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689662564753600 00010000:00000200:0.0:1616513091.847416:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1689662564753600/t0(0) o400->95e94b92-b535-2591-2749-c2ded9bc53f7@10.31.132.27@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.847431:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689662564753600, offset 192 00000400:00000200:0.0:1616513091.847452:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.27@tcp1 00000400:00000200:0.0:1616513091.847460:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.27@tcp1 routed destination 00000400:00000200:0.0:1616513091.847475:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.27@tcp1(10.31.132.27@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.847482:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513091.847488:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.847491:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.847498:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.847503:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.847508:0:50738:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513091.847511:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513091.847563:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513091.847570:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513091.847576:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513091.847583:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513091.847588:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faeaa0 00000800:00000200:2.0:1616513091.847592:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.847595:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513091.847674:0:50738:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513091.847685:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.847693:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513091.847695:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513091.847702:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.27@tcp1 : PUT - for me 00000400:00000200:22.0:1616513091.847711:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.27@tcp1 of length 224 into portal 12 MB=0x600bd38f66180 00000400:00000200:22.0:1616513091.847717:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.27@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 0 00000400:00000200:22.0:1616513091.847722:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.847725:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513091.847732:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513091.847735:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.847737:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.847797:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689662564753792 00010000:00000200:26.0:1616513091.847809:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a29680 x1689662564753792/t0(0) o400->470e4938-2e8e-0c0a-cae5-6034a0864d01@10.31.132.27@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.847824:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689662564753792, offset 192 00000400:00000200:26.0:1616513091.847830:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.27@tcp1 00000400:00000200:26.0:1616513091.847837:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.27@tcp1 routed destination 00000400:00000200:26.0:1616513091.847852:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.27@tcp1(10.31.132.27@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.847858:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513091.847864:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.847866:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513091.847868:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513091.847873:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.847921:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.847977:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.847983:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.847986:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.847993:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.847998:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae770 00000800:00000200:20.0:1616513091.848003:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.848005:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.857599:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513091.857623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513091.857653:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513091.857662:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513091.857670:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.31@tcp1 : PUT - for me 00000400:00000200:2.0:1616513091.857679:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.31@tcp1 of length 224 into portal 12 MB=0x60184c08e94c0 00000800:00000200:24.0:1616513091.857683:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513091.857685:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.31@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 224 00000400:00000200:2.0:1616513091.857691:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513091.857693:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000100:00000200:2.0:1616513091.857693:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513091.857695:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513091.857700:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513091.857703:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.31@tcp1 : PUT - for me 00000800:00000200:2.0:1616513091.857703:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513091.857705:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513091.857707:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513091.857712:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.31@tcp1 of length 224 into portal 26 MB=0x60184c08e9440 00000400:00000200:24.0:1616513091.857719:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.31@tcp1 of length 224/224 into md 0xf97ce369 [1] + 0 00000400:00000200:24.0:1616513091.857724:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04b28 00000400:00000200:24.0:1616513091.857726:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.857729:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513091.857733:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04b28 00000800:00000200:24.0:1616513091.857736:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.857739:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.857741:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513091.857760:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519538144448 00010000:00000200:26.0:1616513091.857771:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d2d00 x1690519538144448/t0(0) o400->cf2be6bf-9f6e-0d9b-d990-8d0fd875f6d4@10.31.132.31@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513091.857785:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690519538144448, offset 192 00000400:00000200:26.0:1616513091.857790:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.31@tcp1 00000400:00000200:26.0:1616513091.857798:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.31@tcp1 routed destination 00000100:00000200:0.0:1616513091.857798:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519538144320 00010000:00000200:0.0:1616513091.857809:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1690519538144320/t0(0) o400->90fb1254-eaeb-029b-b3c7-0f4bc541767d@10.31.132.31@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513091.857811:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.31@tcp1(10.31.132.31@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.857817:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.857822:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513091.857824:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690519538144320, offset 192 00000800:00000200:26.0:1616513091.857825:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.857827:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513091.857830:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.31@tcp1 00000800:00000200:26.0:1616513091.857833:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513091.857837:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513091.857837:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.31@tcp1 routed destination 00000800:00000200:14.0:1616513091.857848:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513091.857853:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.31@tcp1(10.31.132.31@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513091.857854:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513091.857857:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513091.857859:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513091.857863:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513091.857865:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.857874:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.857876:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513091.857877:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae220 00000800:00000200:14.0:1616513091.857880:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513091.857881:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513091.857883:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513091.857886:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.857887:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513091.857946:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513091.857953:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513091.857956:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513091.857963:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513091.857968:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae2a8 00000800:00000200:20.0:1616513091.857972:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513091.857975:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:31.0:1616513091.871497:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513091.871504:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513091.890566:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513091.890622:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513091.890630:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513091.890639:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.176@tcp1 : PUT - for me 00000400:00000200:28.0:1616513091.890648:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.176@tcp1 of length 224 into portal 26 MB=0x5ffd44d6476c0 00000400:00000200:28.0:1616513091.890654:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.176@tcp1 of length 224/224 into md 0xf97ce379 [1] + 0 00000400:00000200:28.0:1616513091.890658:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05980 00000400:00000200:28.0:1616513091.890661:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513091.890663:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513091.890668:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05980 00000800:00000200:28.0:1616513091.890672:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.890675:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.890676:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513091.890682:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513091.890685:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.890686:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513091.890695:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.176@tcp1 : PUT - for me 00000400:00000200:24.0:1616513091.890704:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.176@tcp1 of length 224 into portal 12 MB=0x5ffd44d647780 00000400:00000200:24.0:1616513091.890711:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.176@tcp1 of length 224/224 into md 0x865be291 [1] + 34264 00000400:00000200:24.0:1616513091.890716:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513091.890718:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513091.890725:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513091.890728:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513091.890730:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.890731:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662180132544 00010000:00000200:0.0:1616513091.890742:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688662180132544/t0(0) o400->5f995169-9276-9cbd-038b-a07188c5f7f9@10.31.136.176@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.890755:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662180132544, offset 192 00000400:00000200:0.0:1616513091.890760:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.176@tcp1 00000400:00000200:0.0:1616513091.890768:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.176@tcp1 routed destination 00000400:00000200:0.0:1616513091.890781:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.176@tcp1(10.31.136.176@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.890787:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:12.0:1616513091.890790:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662180132736 00000800:00000200:0.0:1616513091.890793:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.890796:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513091.890798:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513091.890802:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:12.0:1616513091.890803:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e0d80 x1688662180132736/t0(0) o400->3d290ce4-b897-5343-f1fc-ecf9ae767233@10.31.136.176@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513091.890809:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513091.890817:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662180132736, offset 192 00000400:00000200:12.0:1616513091.890824:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.176@tcp1 00000400:00000200:12.0:1616513091.890831:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.176@tcp1 routed destination 00000800:00000200:7.2:1616513091.890844:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513091.890846:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.176@tcp1(10.31.136.176@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513091.890853:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513091.890859:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513091.890861:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513091.890865:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513091.890870:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513091.890895:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.890901:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.890904:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.890910:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.890915:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae330 00000800:00000200:18.0:1616513091.890918:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513091.890919:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513091.890921:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513091.890977:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513091.890991:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513091.890994:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513091.890996:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513091.891001:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513091.891005:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa08 00000800:00000200:16.0:1616513091.891009:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.922500:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.922556:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.922565:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.922574:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.162@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.922584:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.162@tcp1 of length 224 into portal 12 MB=0x5fff593e0c700 00000400:00000200:14.0:1616513091.922590:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.162@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 448 00000400:00000200:14.0:1616513091.922595:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.922598:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513091.922605:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513091.922608:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513091.922610:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513091.922612:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513091.922627:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688805096605440 00010000:00000200:26.0:1616513091.922640:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01f80 x1688805096605440/t0(0) o400->311aef76-3062-5161-b447-c77fe446d5e9@10.31.129.162@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513091.922652:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513091.922654:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688805096605440, offset 192 00000400:00000200:26.0:1616513091.922660:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.162@tcp1 00000400:00000200:26.0:1616513091.922668:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.162@tcp1 routed destination 00000400:00000200:26.0:1616513091.922683:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.162@tcp1(10.31.129.162@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.922689:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.922695:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.922698:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.922701:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.922705:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513091.922708:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513091.922710:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513091.922717:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:18.0:1616513091.922721:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513091.922726:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.162@tcp1 : PUT - for me 00000800:00000200:18.0:1616513091.922728:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513091.922731:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513091.922734:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.162@tcp1 of length 224 into portal 26 MB=0x5fff593e0c280 00000400:00000200:18.0:1616513091.922740:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:24.0:1616513091.922741:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.162@tcp1 of length 224/224 into md 0xf97ce399 [1] + 0 00000400:00000200:24.0:1616513091.922746:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb045d8 00000400:00000200:24.0:1616513091.922748:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513091.922749:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae6e8 00000100:00000200:24.0:1616513091.922752:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513091.922752:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.922755:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513091.922756:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb045d8 00000800:00000200:24.0:1616513091.922759:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.922762:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.922762:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513091.922767:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513091.922823:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688805096604288 00010000:00000200:0.0:1616513091.922835:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688805096604288/t0(0) o400->35eb5abe-8b14-5829-7137-e3ac52372ef1@10.31.129.162@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.922851:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688805096604288, offset 192 00000400:00000200:0.0:1616513091.922857:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.162@tcp1 00000400:00000200:0.0:1616513091.922864:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.162@tcp1 routed destination 00000400:00000200:0.0:1616513091.922879:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.162@tcp1(10.31.129.162@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.922885:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.922891:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.922894:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.922896:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.922901:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.922906:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.922908:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513091.922918:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.922925:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.922928:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.922934:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.922940:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faff58 00000800:00000200:28.0:1616513091.922943:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.922946:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513091.973376:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513091.973432:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513091.973448:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513091.973455:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513091.973465:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.148@tcp1 : PUT - for me 00000400:00000200:14.0:1616513091.973475:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.148@tcp1 of length 224 into portal 12 MB=0x6059532bd5780 00000400:00000200:14.0:1616513091.973481:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.148@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 672 00000800:00000200:22.0:1616513091.973486:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513091.973487:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.973489:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513091.973507:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513091.973509:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513091.973511:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513091.973527:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513091.973537:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513091.973546:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.148@tcp1 : PUT - for me 00000400:00000200:20.0:1616513091.973554:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.148@tcp1 of length 224 into portal 26 MB=0x6059532bd54c0 00000400:00000200:20.0:1616513091.973561:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.148@tcp1 of length 224/224 into md 0xf97ce3b9 [1] + 0 00000400:00000200:20.0:1616513091.973566:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04aa0 00000400:00000200:20.0:1616513091.973569:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513091.973572:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:26.0:1616513091.973574:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694988219799424 00000400:00000200:20.0:1616513091.973577:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04aa0 00000800:00000200:20.0:1616513091.973580:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513091.973582:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513091.973582:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:26.0:1616513091.973587:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e02400 x1694988219799424/t0(0) o400->3096852a-d478-bcda-6061-52e006ab11f0@10.31.143.148@tcp1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513091.973587:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513091.973602:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694988219799424, offset 192 00000400:00000200:26.0:1616513091.973608:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.148@tcp1 00000400:00000200:26.0:1616513091.973615:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.148@tcp1 routed destination 00000400:00000200:26.0:1616513091.973630:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.148@tcp1(10.31.143.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513091.973636:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513091.973643:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513091.973646:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513091.973648:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513091.973653:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513091.973653:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694988219798720 00000800:00000200:7.2:1616513091.973657:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513091.973664:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1694988219798720/t0(0) o400->148f74ad-1e5d-7107-d860-6e82ab16f877@10.31.143.148@tcp1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513091.973669:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513091.973676:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:0.0:1616513091.973678:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694988219798720, offset 192 00000400:00000200:18.0:1616513091.973679:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513091.973685:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.148@tcp1 00000400:00000200:18.0:1616513091.973686:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513091.973689:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af980 00000400:00000200:0.0:1616513091.973693:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.148@tcp1 routed destination 00000800:00000200:18.0:1616513091.973694:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513091.973697:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513091.973708:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.148@tcp1(10.31.143.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513091.973714:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513091.973720:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513091.973723:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513091.973725:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513091.973730:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513091.973735:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513091.973737:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513091.973748:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513091.973755:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513091.973758:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513091.973765:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513091.973771:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafa08 00000800:00000200:28.0:1616513091.973774:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513091.973777:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:9.2:1616513091.988354:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d73e8234a00] (20)++ 00000800:00000200:22.0:1616513091.988411:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d73e8234a00] (21)++ 00000800:00000200:22.0:1616513091.988421:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.145@o2ib1 00000400:00000200:22.0:1616513091.988429:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.145@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513091.988453:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.145@o2ib1 of length 224 into portal 26 MB=0x604a7cdf5ce80 00000400:00000200:22.0:1616513091.988460:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.145@o2ib1 of length 224/224 into md 0xf97ce3c9 [1] + 0 00000400:00000200:22.0:1616513091.988465:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb046e8 00000800:00000200:14.0:1616513091.988465:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d73e8234a00] (22)++ 00000400:00000200:22.0:1616513091.988468:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513091.988470:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513091.988474:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb046e8 00000800:00000200:14.0:1616513091.988476:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.145@o2ib1 00000800:00000200:22.0:1616513091.988477:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d73e8234a00] (23)++ 00000800:00000200:22.0:1616513091.988480:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d73e8234a00] (24)-- 00000800:00000200:22.0:1616513091.988482:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d73e8234a00] (23)-- 00000800:00000200:22.0:1616513091.988485:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d73e8234a00] (22)-- 00000400:00000200:14.0:1616513091.988485:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.145@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513091.988493:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.145@o2ib1 of length 224 into portal 12 MB=0x604a7cdf5cf00 00000400:00000200:14.0:1616513091.988512:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.145@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 896 00000400:00000200:14.0:1616513091.988517:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513091.988520:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513091.988525:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693968621751936 00000800:00000200:14.0:1616513091.988529:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d73e8234a00] (21)++ 00000800:00000200:14.0:1616513091.988532:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d73e8234a00] (22)-- 00000800:00000200:14.0:1616513091.988534:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d73e8234a00] (21)-- 00010000:00000200:0.0:1616513091.988536:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1693968621751936/t0(0) o400->38e040f4-5633-3a58-2c85-36fa51fe8599@10.242.46.145@o2ib1:48/0 lens 224/224 e 0 to 0 dl 1616513183 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513091.988551:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693968621751936, offset 192 00000100:00000200:30.0:1616513091.988552:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693968621752064 00000400:00000200:0.0:1616513091.988557:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.145@o2ib1 00010000:00000200:30.0:1616513091.988565:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e02d00 x1693968621752064/t0(0) o400->8c16e8b8-ca0f-855d-1df8-2db1ee57cac5@10.242.46.145@o2ib1:717/0 lens 224/224 e 0 to 0 dl 1616513097 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513091.988565:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.145@o2ib1 local destination 00000100:00000200:30.0:1616513091.988580:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693968621752064, offset 192 00000400:00000200:0.0:1616513091.988580:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.145@o2ib1(10.242.46.145@o2ib1:10.242.46.145@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513091.988586:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.145@o2ib1 00000800:00000200:0.0:1616513091.988587:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.145@o2ib1 00000800:00000200:0.0:1616513091.988593:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f00] -> 10.242.46.145@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513091.988594:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.145@o2ib1 local destination 00000800:00000200:0.0:1616513091.988596:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d73e8234a00] (20)++ 00000800:00000200:0.0:1616513091.988599:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d73e8234a00] (21)++ 00000800:00000200:0.0:1616513091.988604:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d73e8234a00] (22)-- 00000400:00000200:30.0:1616513091.988607:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.145@o2ib1(10.242.46.145@o2ib1:10.242.46.145@o2ib1) : PUT try# 0 00000800:00000200:9.2:1616513091.988609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d73e8234a00] (21)++ 00000100:00000200:0.0:1616513091.988611:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513091.988614:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.145@o2ib1 00000800:00000200:30.0:1616513091.988620:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f00] -> 10.242.46.145@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513091.988623:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d73e8234a00] (22)++ 00000800:00000200:30.0:1616513091.988625:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d73e8234a00] (23)++ 00000800:00000200:30.0:1616513091.988629:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d73e8234a00] (24)-- 00000800:00000200:2.0:1616513091.988663:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d73e8234a00] (23)++ 00000800:00000200:2.0:1616513091.988670:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d73e8234a00] (24)-- 00000400:00000200:2.0:1616513091.988673:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513091.988674:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d73e8234a00] (23)++ 00000800:00000200:22.0:1616513091.988680:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d73e8234a00] (24)-- 00000400:00000200:2.0:1616513091.988680:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.145@o2ib1: PUT: OK 00000400:00000200:22.0:1616513091.988682:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513091.988685:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d73e8234a00] (23)-- 00000400:00000200:2.0:1616513091.988685:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafe48 00000400:00000200:22.0:1616513091.988687:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.145@o2ib1: PUT: OK 00000800:00000200:2.0:1616513091.988689:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d73e8234a00] (22)-- 00000400:00000200:22.0:1616513091.988690:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eebb0 00000800:00000200:22.0:1616513091.988693:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d73e8234a00] (21)-- 00000800:00000200:7.2:1616513092.022420:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513092.022475:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.022485:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513092.022494:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.51@tcp1 : PUT - for me 00000400:00000200:14.0:1616513092.022503:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.51@tcp1 of length 224 into portal 12 MB=0x5ffdf5b3c3480 00000400:00000200:14.0:1616513092.022510:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.51@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 1120 00000400:00000200:14.0:1616513092.022515:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513092.022518:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513092.022535:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.022535:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.022538:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513092.022540:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.022600:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709657015424 00010000:00000200:30.0:1616513092.022612:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03a80 x1688709657015424/t0(0) o400->d760409c-cdfe-e3c5-461e-d37d5df1c00c@10.31.142.51@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.022626:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688709657015424, offset 192 00000400:00000200:30.0:1616513092.022632:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.51@tcp1 00000400:00000200:30.0:1616513092.022641:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.51@tcp1 routed destination 00000400:00000200:30.0:1616513092.022655:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.51@tcp1(10.31.142.51@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.022661:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513092.022667:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.022669:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513092.022671:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513092.022676:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.022681:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513092.022733:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513092.022739:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513092.022742:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513092.022748:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.022751:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef760 00000800:00000200:2.0:1616513092.022755:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.022758:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513092.023568:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513092.023622:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.023631:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513092.023640:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.51@tcp1 : PUT - for me 00000400:00000200:24.0:1616513092.023650:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.51@tcp1 of length 224 into portal 26 MB=0x5ffdf5b3c3280 00000400:00000200:24.0:1616513092.023656:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.51@tcp1 of length 224/224 into md 0xf97ce3d9 [1] + 0 00000400:00000200:24.0:1616513092.023661:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04660 00000400:00000200:24.0:1616513092.023663:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.023666:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513092.023671:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04660 00000800:00000200:24.0:1616513092.023673:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513092.023676:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513092.023676:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513092.023680:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.023733:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709657014912 00010000:00000200:0.0:1616513092.023744:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688709657014912/t0(0) o400->33910d81-60b0-c655-967c-83f199936bbf@10.31.142.51@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.023757:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688709657014912, offset 192 00000400:00000200:0.0:1616513092.023763:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.51@tcp1 00000400:00000200:0.0:1616513092.023770:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.51@tcp1 routed destination 00000400:00000200:0.0:1616513092.023784:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.51@tcp1(10.31.142.51@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.023790:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.023795:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.023798:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.023800:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.023804:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.023808:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513092.023810:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513092.023819:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.023826:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513092.023829:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.023836:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.023841:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae7f8 00000800:00000200:28.0:1616513092.023845:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.023848:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.026354:0:50751:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513092.026407:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.026416:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513092.026425:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.95@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.026446:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.95@tcp1 of length 224 into portal 26 MB=0x5ffddb6486f40 00000400:00000200:18.0:1616513092.026453:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.95@tcp1 of length 224/224 into md 0xf97ce3f9 [1] + 0 00000400:00000200:18.0:1616513092.026458:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04dd0 00000800:00000200:14.0:1616513092.026460:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513092.026461:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.026464:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513092.026468:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04dd0 00000800:00000200:18.0:1616513092.026471:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.026474:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.026476:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.026531:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702594608960 00010000:00000200:0.0:1616513092.026542:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688702594608960/t0(0) o400->cd3217e0-d5f6-ffc2-2f38-5ea0487b6d72@10.31.136.95@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.026557:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702594608960, offset 192 00000400:00000200:0.0:1616513092.026563:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.95@tcp1 00000400:00000200:0.0:1616513092.026570:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.95@tcp1 routed destination 00000400:00000200:0.0:1616513092.026585:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.95@tcp1(10.31.136.95@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.026592:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.026597:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.026602:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.026604:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.026609:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.026614:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.026615:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513092.026667:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513092.026674:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513092.026676:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513092.026683:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.026688:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafd38 00000800:00000200:2.0:1616513092.026700:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.026703:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513092.026708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513092.026719:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.026727:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513092.026729:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513092.026736:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.95@tcp1 : PUT - for me 00000400:00000200:14.0:1616513092.026744:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.95@tcp1 of length 224 into portal 12 MB=0x5ffddb6487140 00000400:00000200:14.0:1616513092.026751:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.95@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 1344 00000400:00000200:14.0:1616513092.026756:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513092.026758:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513092.026765:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.026768:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.026770:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.026832:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702594609472 00010000:00000200:30.0:1616513092.026844:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03f00 x1688702594609472/t0(0) o400->47b9a780-a488-629e-8a5f-8ae6c84f7850@10.31.136.95@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.026858:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702594609472, offset 192 00000400:00000200:30.0:1616513092.026865:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.95@tcp1 00000400:00000200:30.0:1616513092.026872:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.95@tcp1 routed destination 00000400:00000200:30.0:1616513092.026887:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.95@tcp1(10.31.136.95@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.026894:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513092.026899:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.026902:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513092.026904:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513092.026909:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.026914:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.026967:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.026974:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513092.026976:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513092.026983:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513092.026987:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efba0 00000800:00000200:20.0:1616513092.026991:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513092.026994:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.035627:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.035681:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.035690:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.035699:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.179@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.035708:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.179@tcp1 of length 224 into portal 26 MB=0x601830e1b7e40 00000400:00000200:28.0:1616513092.035715:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.179@tcp1 of length 224/224 into md 0xf97ce419 [1] + 0 00000400:00000200:28.0:1616513092.035720:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05d38 00000400:00000200:28.0:1616513092.035722:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.035725:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513092.035731:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05d38 00000800:00000200:16.0:1616513092.035734:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.035735:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.035737:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.035739:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.035762:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513092.035794:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512249290304 00010000:00000200:0.0:1616513092.035805:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1690512249290304/t0(0) o400->500ce3b0-5678-c3e1-2c42-594e428c9282@10.31.141.179@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513092.035815:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513092.035819:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512249290304, offset 192 00000800:00000200:20.0:1616513092.035824:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513092.035826:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.179@tcp1 00000400:00000200:20.0:1616513092.035833:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.179@tcp1 : PUT - for me 00000400:00000200:0.0:1616513092.035833:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.179@tcp1 routed destination 00000400:00000200:20.0:1616513092.035843:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.179@tcp1 of length 224 into portal 12 MB=0x601830e1b8000 00000400:00000200:0.0:1616513092.035848:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.179@tcp1(10.31.141.179@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513092.035849:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.179@tcp1 of length 224/224 into md 0x865be291 [1] + 34488 00000400:00000200:20.0:1616513092.035855:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513092.035855:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:20.0:1616513092.035857:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513092.035860:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.035863:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.035865:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513092.035871:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513092.035872:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513092.035873:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513092.035875:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513092.035877:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.035879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.035880:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513092.035930:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512249290752 00000800:00000200:2.0:1616513092.035934:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513092.035941:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:12.0:1616513092.035942:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d9200 x1690512249290752/t0(0) o400->f7d7ed87-d065-01f4-a21d-45dc1ef7693c@10.31.141.179@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513092.035944:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513092.035951:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.035956:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faee58 00000100:00000200:12.0:1616513092.035959:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512249290752, offset 192 00000800:00000200:2.0:1616513092.035960:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.035963:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513092.035966:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.179@tcp1 00000400:00000200:12.0:1616513092.035973:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.179@tcp1 routed destination 00000400:00000200:12.0:1616513092.035988:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.179@tcp1(10.31.141.179@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.035995:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513092.036000:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.036003:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.036005:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.036010:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.036014:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.036024:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.036029:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513092.036031:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513092.036033:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513092.036038:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.036042:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e000 00000800:00000200:16.0:1616513092.036046:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.088344:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513092.088398:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.088407:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513092.088417:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.56@tcp1 : PUT - for me 00000400:00000200:24.0:1616513092.088427:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.56@tcp1 of length 224 into portal 26 MB=0x603e1707b0400 00000400:00000200:24.0:1616513092.088463:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.56@tcp1 of length 224/224 into md 0xf97ce439 [1] + 0 00000800:00000200:20.0:1616513092.088463:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513092.088468:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04cc0 00000400:00000200:24.0:1616513092.088471:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.088474:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513092.088478:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04cc0 00000800:00000200:24.0:1616513092.088481:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.088484:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.088486:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.088543:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116649899008 00010000:00000200:0.0:1616513092.088554:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1693116649899008/t0(0) o400->cac6dfe1-37f3-7bbf-a612-76dd2fbb6e7f@10.31.136.56@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.088567:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693116649899008, offset 192 00000400:00000200:0.0:1616513092.088573:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.56@tcp1 00000400:00000200:0.0:1616513092.088580:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.56@tcp1 routed destination 00000400:00000200:0.0:1616513092.088594:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.56@tcp1(10.31.136.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.088600:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.088606:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.088609:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.088611:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.088615:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513092.088621:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513092.088624:0:50757:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.088677:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.088684:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513092.088687:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513092.088694:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513092.088699:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf6d8 00000800:00000200:18.0:1616513092.088702:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.088706:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513092.089137:0:50757:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513092.089190:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513092.089200:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513092.089209:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.56@tcp1 : PUT - for me 00000400:00000200:2.0:1616513092.089219:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.56@tcp1 of length 224 into portal 12 MB=0x603e1707b0540 00000400:00000200:2.0:1616513092.089225:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.56@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 1568 00000400:00000200:2.0:1616513092.089230:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513092.089232:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513092.089239:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513092.089242:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513092.089244:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.089247:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.089305:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693116649899328 00010000:00000200:30.0:1616513092.089318:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3a80 x1693116649899328/t0(0) o400->e900800c-def2-a987-7410-2f9d4eb22728@10.31.136.56@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.089332:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693116649899328, offset 192 00000400:00000200:30.0:1616513092.089338:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.56@tcp1 00000400:00000200:30.0:1616513092.089346:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.56@tcp1 routed destination 00000400:00000200:30.0:1616513092.089361:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.56@tcp1(10.31.136.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.089368:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513092.089373:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.089376:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513092.089380:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513092.089385:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.089389:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.089400:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.089407:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513092.089410:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.089417:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.089421:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee660 00000800:00000200:28.0:1616513092.089425:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.089428:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.096636:0:50757:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513092.096689:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.096701:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513092.096709:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.137@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.096718:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.137@tcp1 of length 224 into portal 26 MB=0x604abbe58a800 00000400:00000200:18.0:1616513092.096725:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.137@tcp1 of length 224/224 into md 0xf97ce449 [1] + 0 00000400:00000200:18.0:1616513092.096730:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05540 00000400:00000200:18.0:1616513092.096732:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.096735:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513092.096739:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05540 00000800:00000200:18.0:1616513092.096742:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.096745:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513092.096745:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.096749:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.096803:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693985539663872 00010000:00000200:0.0:1616513092.096814:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1693985539663872/t0(0) o400->d3d99564-1151-0d18-5b82-fd49adc5f913@10.31.140.137@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.096828:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693985539663872, offset 192 00000400:00000200:0.0:1616513092.096834:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.137@tcp1 00000400:00000200:0.0:1616513092.096842:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.137@tcp1 routed destination 00000400:00000200:0.0:1616513092.096857:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.137@tcp1(10.31.140.137@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.096864:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.096870:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.096872:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.096875:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.096879:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.096883:0:50757:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.096886:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513092.096936:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513092.096950:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513092.096952:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513092.096959:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.096964:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae088 00000800:00000200:2.0:1616513092.096968:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.096971:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513092.097208:0:50757:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513092.097218:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.097226:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513092.097228:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513092.097235:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.137@tcp1 : PUT - for me 00000400:00000200:14.0:1616513092.097244:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.137@tcp1 of length 224 into portal 12 MB=0x604abbe58ac80 00000400:00000200:14.0:1616513092.097252:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.137@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 1792 00000400:00000200:14.0:1616513092.097257:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513092.097260:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513092.097266:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.097269:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.097271:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.097329:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693985539665024 00010000:00000200:30.0:1616513092.097341:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e04380 x1693985539665024/t0(0) o400->3e2f924e-2f83-62d0-62bf-38b5d47588f2@10.31.140.137@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.097355:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693985539665024, offset 192 00000400:00000200:30.0:1616513092.097362:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.137@tcp1 00000400:00000200:30.0:1616513092.097369:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.137@tcp1 routed destination 00000400:00000200:30.0:1616513092.097384:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.137@tcp1(10.31.140.137@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.097391:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513092.097396:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.097399:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513092.097401:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513092.097405:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.097412:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.097464:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.097471:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513092.097474:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513092.097480:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513092.097484:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee4c8 00000800:00000200:20.0:1616513092.097488:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513092.097491:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.107432:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.107502:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.107511:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.107520:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.21@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.107528:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.21@tcp1 of length 224 into portal 26 MB=0x5ffe6374a1180 00000400:00000200:28.0:1616513092.107535:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.21@tcp1 of length 224/224 into md 0xf97ce459 [1] + 0 00000400:00000200:28.0:1616513092.107540:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04c38 00000400:00000200:28.0:1616513092.107542:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.107545:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513092.107550:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04c38 00000800:00000200:28.0:1616513092.107553:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.107555:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513092.107555:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513092.107562:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.107565:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.107566:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513092.107574:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.21@tcp1 : PUT - for me 00000400:00000200:16.0:1616513092.107583:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.21@tcp1 of length 224 into portal 12 MB=0x5ffe6374a1480 00000400:00000200:16.0:1616513092.107589:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.21@tcp1 of length 224/224 into md 0x865be291 [1] + 34712 00000400:00000200:16.0:1616513092.107597:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513092.107600:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513092.107607:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.107610:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.107612:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.107613:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739118715264 00010000:00000200:0.0:1616513092.107624:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688739118715264/t0(0) o400->f1233654-c51f-92da-d3bd-66d33d26ef97@10.31.142.21@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.107638:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688739118715264, offset 192 00000400:00000200:0.0:1616513092.107644:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.21@tcp1 00000400:00000200:0.0:1616513092.107652:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.21@tcp1 routed destination 00000400:00000200:0.0:1616513092.107666:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.21@tcp1(10.31.142.21@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:12.0:1616513092.107671:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739118716032 00000800:00000200:0.0:1616513092.107673:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.107679:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.107681:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00010000:00000200:12.0:1616513092.107683:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b193600 x1688739118716032/t0(0) o400->b23e4aba-a8fa-eb46-b6c1-76a173068c46@10.31.142.21@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513092.107684:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.107688:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.107694:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.107695:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:12.0:1616513092.107697:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688739118716032, offset 192 00000400:00000200:12.0:1616513092.107704:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.21@tcp1 00000400:00000200:12.0:1616513092.107711:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.21@tcp1 routed destination 00000400:00000200:12.0:1616513092.107726:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.21@tcp1(10.31.142.21@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.107732:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513092.107738:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.107741:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.107743:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.107747:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513092.107747:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513092.107753:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513092.107754:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513092.107756:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513092.107763:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513092.107765:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513092.107769:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf430 00000800:00000200:24.0:1616513092.107772:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513092.107772:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513092.107775:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513092.107775:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513092.107776:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513092.107782:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513092.107786:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eb28 00000800:00000200:24.0:1616513092.107790:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.144150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513092.144206:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.144216:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513092.144225:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.218@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.144233:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.218@tcp1 of length 224 into portal 26 MB=0x5ffe9224d1000 00000400:00000200:18.0:1616513092.144240:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.218@tcp1 of length 224/224 into md 0xf97ce469 [1] + 0 00000400:00000200:18.0:1616513092.144245:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb054b8 00000400:00000200:18.0:1616513092.144247:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.144250:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513092.144254:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb054b8 00000800:00000200:18.0:1616513092.144257:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.144260:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513092.144260:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513092.144267:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.144270:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.144271:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513092.144280:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.218@tcp1 : PUT - for me 00000400:00000200:14.0:1616513092.144288:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.218@tcp1 of length 224 into portal 12 MB=0x5ffe9224d1080 00000400:00000200:14.0:1616513092.144294:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.218@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 2016 00000400:00000200:14.0:1616513092.144300:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513092.144302:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513092.144309:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.144312:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.144314:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.144340:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688751651491840 00010000:00000200:0.0:1616513092.144351:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688751651491840/t0(0) o400->e19f75dc-725e-8563-63eb-ecfb25728739@10.31.141.218@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.144364:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688751651491840, offset 192 00000400:00000200:0.0:1616513092.144372:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.218@tcp1 00000100:00000200:30.0:1616513092.144374:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688751651491968 00000400:00000200:0.0:1616513092.144379:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.218@tcp1 routed destination 00010000:00000200:30.0:1616513092.144387:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e04800 x1688751651491968/t0(0) o400->ad6bfbbc-dbe3-8418-90a6-78c63cd0a6b7@10.31.141.218@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513092.144393:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.218@tcp1(10.31.141.218@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.144399:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:30.0:1616513092.144401:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688751651491968, offset 192 00000800:00000200:0.0:1616513092.144405:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513092.144407:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.218@tcp1 00000800:00000200:0.0:1616513092.144407:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.144409:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:30.0:1616513092.144414:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.218@tcp1 routed destination 00000800:00000200:0.0:1616513092.144414:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.144419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.144421:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:30.0:1616513092.144429:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.218@tcp1(10.31.141.218@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.144449:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513092.144454:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.144457:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513092.144459:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513092.144464:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.144472:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513092.144475:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.144482:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513092.144485:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513092.144486:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513092.144491:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513092.144497:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafc28 00000800:00000200:22.0:1616513092.144501:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513092.144529:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.144536:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513092.144539:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.144545:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.144549:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efcb0 00000800:00000200:16.0:1616513092.144553:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.144556:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.158842:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.158899:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.158915:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.158925:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.178@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.158933:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.178@tcp1 of length 224 into portal 26 MB=0x5ffd5d2c03480 00000400:00000200:28.0:1616513092.158940:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.178@tcp1 of length 224/224 into md 0xf97ce479 [1] + 0 00000400:00000200:28.0:1616513092.158945:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04550 00000400:00000200:28.0:1616513092.158947:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.158952:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513092.158958:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04550 00000800:00000200:28.0:1616513092.158961:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513092.158961:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.158964:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.158970:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.158973:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.158974:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513092.158982:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.178@tcp1 : PUT - for me 00000400:00000200:24.0:1616513092.158991:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.178@tcp1 of length 224 into portal 12 MB=0x5ffd5d2c03800 00000400:00000200:24.0:1616513092.158997:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.178@tcp1 of length 224/224 into md 0x865be291 [1] + 34936 00000400:00000200:24.0:1616513092.159003:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.159005:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513092.159013:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.159016:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.159018:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.159022:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668712481920 00000100:00000200:12.0:1616513092.159033:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668712482816 00010000:00000200:0.0:1616513092.159033:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688668712481920/t0(0) o400->5d0e6a28-c459-8f2b-e3b8-8f1147b93964@10.31.129.178@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:12.0:1616513092.159046:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cb02e5100 x1688668712482816/t0(0) o400->625b221a-e15e-fca9-e02f-7286bb3c4775@10.31.129.178@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513092.159048:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668712481920, offset 192 00000400:00000200:0.0:1616513092.159054:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.178@tcp1 00000100:00000200:12.0:1616513092.159060:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668712482816, offset 192 00000400:00000200:0.0:1616513092.159062:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.178@tcp1 routed destination 00000400:00000200:12.0:1616513092.159068:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.178@tcp1 00000400:00000200:12.0:1616513092.159075:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.178@tcp1 routed destination 00000400:00000200:0.0:1616513092.159077:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.178@tcp1(10.31.129.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.159084:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513092.159090:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.178@tcp1(10.31.129.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.159097:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.159097:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.159100:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513092.159103:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.159103:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513092.159105:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.159107:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.159107:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513092.159112:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513092.159113:0:50761:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.159114:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513092.159117:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513092.159167:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.159174:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513092.159177:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513092.159183:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513092.159187:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513092.159189:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae088 00000800:00000200:14.0:1616513092.159192:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513092.159194:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513092.159195:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513092.159197:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513092.159198:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513092.159204:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513092.159208:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ee58 00000800:00000200:20.0:1616513092.159212:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.177985:0:50763:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513092.178041:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.178051:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513092.178060:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.196@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.178071:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.196@tcp1 of length 224 into portal 12 MB=0x5ffd385768000 00000400:00000200:18.0:1616513092.178077:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.196@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 2240 00000400:00000200:18.0:1616513092.178083:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.178088:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513092.178095:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.178095:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.178098:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.178100:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.178118:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658825871360 00010000:00000200:30.0:1616513092.178130:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd705a00 x1688658825871360/t0(0) o400->ab9b186b-7213-51bd-2fb5-aee0681b7c87@10.31.138.196@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.178144:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658825871360, offset 192 00000400:00000200:30.0:1616513092.178152:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.196@tcp1 00000800:00000200:11.2:1616513092.178153:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:30.0:1616513092.178160:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.196@tcp1 routed destination 00000400:00000200:30.0:1616513092.178175:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.196@tcp1(10.31.138.196@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.178181:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513092.178187:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.178190:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513092.178192:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513092.178197:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.178201:0:50763:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513092.178209:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.178219:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513092.178228:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.196@tcp1 : PUT - for me 00000400:00000200:24.0:1616513092.178237:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.196@tcp1 of length 224 into portal 26 MB=0x5ffd385767f80 00000400:00000200:24.0:1616513092.178244:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.196@tcp1 of length 224/224 into md 0xf97ce489 [1] + 0 00000400:00000200:24.0:1616513092.178249:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05298 00000400:00000200:24.0:1616513092.178251:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.178254:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513092.178255:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513092.178258:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05298 00000800:00000200:24.0:1616513092.178261:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513092.178262:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513092.178264:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513092.178265:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513092.178266:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.178269:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513092.178271:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513092.178275:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee990 00000800:00000200:14.0:1616513092.178279:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.178282:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.178323:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658825871232 00010000:00000200:0.0:1616513092.178334:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688658825871232/t0(0) o400->a021c923-095e-70eb-cd66-15989cd0bb93@10.31.138.196@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.178348:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658825871232, offset 192 00000400:00000200:0.0:1616513092.178355:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.196@tcp1 00000400:00000200:0.0:1616513092.178363:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.196@tcp1 routed destination 00000400:00000200:0.0:1616513092.178378:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.196@tcp1(10.31.138.196@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.178388:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.178394:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.178397:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.178399:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.178404:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.178409:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513092.178411:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513092.178421:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.178428:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513092.178431:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.178450:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.178455:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faee58 00000800:00000200:28.0:1616513092.178459:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.178462:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.189373:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513092.189429:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.189452:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513092.189461:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.215@tcp1 : PUT - for me 00000400:00000200:24.0:1616513092.189471:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.215@tcp1 of length 224 into portal 12 MB=0x5ffdaed534ec0 00000400:00000200:24.0:1616513092.189477:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.215@tcp1 of length 224/224 into md 0x865be291 [1] + 35160 00000800:00000200:16.0:1616513092.189482:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513092.189483:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.189485:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513092.189494:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.189497:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.189499:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513092.189513:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688690633166528 00010000:00000200:12.0:1616513092.189525:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce9200 x1688690633166528/t0(0) o400->77ffb38c-36a1-e68c-e8ba-0acee89bee19@10.31.140.215@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513092.189538:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688690633166528, offset 192 00000400:00000200:12.0:1616513092.189544:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.215@tcp1 00000800:00000200:7.2:1616513092.189547:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513092.189552:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.215@tcp1 routed destination 00000400:00000200:12.0:1616513092.189566:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.215@tcp1(10.31.140.215@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.189572:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513092.189577:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.189580:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513092.189582:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513092.189586:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513092.189604:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.189614:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513092.189619:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:22.0:1616513092.189623:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.215@tcp1 : PUT - for me 00000800:00000200:18.0:1616513092.189627:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:18.0:1616513092.189630:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513092.189632:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.215@tcp1 of length 224 into portal 26 MB=0x5ffdaed534e40 00000400:00000200:18.0:1616513092.189637:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513092.189639:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.215@tcp1 of length 224/224 into md 0xf97ce499 [1] + 0 00000400:00000200:18.0:1616513092.189640:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e880 00000400:00000200:22.0:1616513092.189643:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb047f8 00000800:00000200:18.0:1616513092.189644:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513092.189646:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513092.189647:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:22.0:1616513092.189649:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513092.189653:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb047f8 00000800:00000200:22.0:1616513092.189656:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.189659:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.189661:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.189728:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688690633166400 00010000:00000200:0.0:1616513092.189739:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688690633166400/t0(0) o400->79b42a36-d807-9f9f-d207-00e776fbedf9@10.31.140.215@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.189753:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688690633166400, offset 192 00000400:00000200:0.0:1616513092.189760:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.215@tcp1 00000400:00000200:0.0:1616513092.189768:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.215@tcp1 routed destination 00000400:00000200:0.0:1616513092.189783:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.215@tcp1(10.31.140.215@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.189789:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.189795:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.189798:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.189800:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.189805:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.189811:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513092.189811:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513092.189823:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.189830:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513092.189833:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.189839:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.189845:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae7f8 00000800:00000200:28.0:1616513092.189848:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.189851:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.204587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513092.204644:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513092.204654:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513092.204663:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.60@tcp1 : PUT - for me 00000400:00000200:2.0:1616513092.204672:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.60@tcp1 of length 224 into portal 26 MB=0x5f91dbfb89500 00000400:00000200:2.0:1616513092.204679:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.60@tcp1 of length 224/224 into md 0xf97ce4a9 [1] + 0 00000400:00000200:2.0:1616513092.204684:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04088 00000400:00000200:2.0:1616513092.204686:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513092.204689:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513092.204693:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04088 00000800:00000200:18.0:1616513092.204697:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.204697:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513092.204699:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513092.204702:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.204756:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681281049466112 00010000:00000200:0.0:1616513092.204766:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1681281049466112/t0(0) o400->85817ad3-90a6-f453-8945-63fa8b4f1ea2@10.31.131.60@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.204779:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681281049466112, offset 192 00000400:00000200:0.0:1616513092.204785:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.60@tcp1 00000800:00000200:11.2:1616513092.204789:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:0.0:1616513092.204793:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.60@tcp1 routed destination 00000400:00000200:0.0:1616513092.204807:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.60@tcp1(10.31.131.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.204813:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.204818:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.204821:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.204824:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.204829:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.204834:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.204836:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513092.204844:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.204854:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513092.204863:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.60@tcp1 : PUT - for me 00000400:00000200:16.0:1616513092.204872:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.60@tcp1 of length 224 into portal 12 MB=0x5f91dbfb89580 00000400:00000200:16.0:1616513092.204879:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.60@tcp1 of length 224/224 into md 0x865be291 [1] + 35384 00000400:00000200:16.0:1616513092.204884:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513092.204887:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513092.204890:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513092.204894:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513092.204897:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513092.204897:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513092.204898:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513092.204899:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513092.204901:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513092.204906:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513092.204912:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafa08 00000100:00000200:12.0:1616513092.204914:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681281049466240 00000800:00000200:22.0:1616513092.204915:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.204918:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:12.0:1616513092.204926:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b197980 x1681281049466240/t0(0) o400->378fd162-53b1-8bce-b378-ba2bd6932fae@10.31.131.60@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513092.204939:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681281049466240, offset 192 00000400:00000200:12.0:1616513092.204945:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.60@tcp1 00000400:00000200:12.0:1616513092.204952:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.60@tcp1 routed destination 00000400:00000200:12.0:1616513092.204965:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.60@tcp1(10.31.131.60@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.204972:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513092.204977:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.204981:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.204983:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.204987:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.204993:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.205005:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.205011:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513092.205014:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.205021:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.205025:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:28.0:1616513092.205029:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.205032:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:13.2F:1616513092.215204:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4c914a000] (20)++ 00000800:00000200:27.0:1616513092.215261:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c914a000] (21)++ 00000800:00000200:27.0:1616513092.215272:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.147@o2ib1 00000400:00000200:27.0:1616513092.215281:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.147@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513092.215290:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.147@o2ib1 of length 224 into portal 12 MB=0x604a73232ed40 00000400:00000200:27.0:1616513092.215297:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.147@o2ib1 of length 224/224 into md 0x3657608d [1] + 91840 00000400:00000200:27.0:1616513092.215303:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513092.215306:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:27.0:1616513092.215315:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4c914a000] (22)++ 00000800:00000200:3.0:1616513092.215316:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c914a000] (22)++ 00000800:00000200:27.0:1616513092.215319:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4c914a000] (23)-- 00000800:00000200:27.0:1616513092.215325:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c914a000] (23)-- 00000800:00000200:27.0:1616513092.215327:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c914a000] (22)-- 00000800:00000200:3.0:1616513092.215329:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.147@o2ib1 00000400:00000200:3.0:1616513092.215338:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.147@o2ib1 : PUT - for me 00000400:00000200:3.0:1616513092.215346:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.147@o2ib1 of length 224 into portal 26 MB=0x604a73232ecc0 00000400:00000200:3.0:1616513092.215354:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.147@o2ib1 of length 224/224 into md 0xf97ce4b9 [1] + 0 00000400:00000200:3.0:1616513092.215359:0:6474:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05078 00000400:00000200:3.0:1616513092.215362:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:3.0:1616513092.215365:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:3.0:1616513092.215371:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05078 00000800:00000200:3.0:1616513092.215375:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4c914a000] (21)++ 00000800:00000200:3.0:1616513092.215378:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4c914a000] (22)-- 00000800:00000200:3.0:1616513092.215380:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c914a000] (21)-- 00000100:00000200:23.0:1616513092.215386:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693966008511808 00000100:00000200:0.0:1616513092.215399:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693966008511680 00010000:00000200:23.0:1616513092.215403:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d841ded7980 x1693966008511808/t0(0) o400->59ae3fa9-51e0-2721-5ef6-52256d6c2cc5@10.242.46.147@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:0.0:1616513092.215410:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1693966008511680/t0(0) o400->5f86a104-1c40-57bb-aa4e-fd28c28eeb2e@10.242.46.147@o2ib1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:23.0:1616513092.215419:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693966008511808, offset 192 00000100:00000200:0.0:1616513092.215425:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693966008511680, offset 192 00000400:00000200:23.0:1616513092.215428:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.147@o2ib1 00000400:00000200:0.0:1616513092.215432:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.147@o2ib1 00000400:00000200:23.0:1616513092.215452:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.147@o2ib1 local destination 00000400:00000200:0.0:1616513092.215455:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.147@o2ib1 local destination 00000400:00000200:23.0:1616513092.215467:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.147@o2ib1(10.242.46.147@o2ib1:10.242.46.147@o2ib1) : PUT try# 0 00000800:00000200:23.0:1616513092.215474:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.147@o2ib1 00000400:00000200:0.0:1616513092.215483:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.147@o2ib1(10.242.46.147@o2ib1:10.242.46.147@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.215489:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.147@o2ib1 00000800:00000200:23.0:1616513092.215491:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8dcff92a80] -> 10.242.46.147@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513092.215494:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4c914a000] (20)++ 00000800:00000200:23.0:1616513092.215496:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4c914a000] (21)++ 00000800:00000200:0.0:1616513092.215496:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8dcff92a80] -> 10.242.46.147@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.215499:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4c914a000] (22)++ 00000800:00000200:23.0:1616513092.215502:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4c914a000] (23)-- 00000800:00000200:0.0:1616513092.215503:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4c914a000] (23)++ 00000800:00000200:13.2:1616513092.215508:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4c914a000] (23)++ 00000800:00000200:0.0:1616513092.215508:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4c914a000] (23)-- 00000100:00000200:0.0:1616513092.215515:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:27.0:1616513092.215516:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c914a000] (23)++ 00000800:00000200:27.0:1616513092.215522:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4c914a000] (24)-- 00000400:00000200:27.0:1616513092.215524:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513092.215530:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.147@o2ib1: PUT: OK 00000400:00000200:27.0:1616513092.215533:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f4b8 00000800:00000200:27.0:1616513092.215537:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c914a000] (23)-- 00000800:00000200:27.0:1616513092.215538:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4c914a000] (22)++ 00000800:00000200:27.0:1616513092.215542:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4c914a000] (23)-- 00000400:00000200:27.0:1616513092.215544:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.0:1616513092.215546:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c914a000] (22)-- 00000400:00000200:27.0:1616513092.215550:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.147@o2ib1: PUT: OK 00000400:00000200:27.0:1616513092.215555:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae330 00000800:00000200:27.0:1616513092.215557:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4c914a000] (21)-- 00000800:00000200:11.2:1616513092.264733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513092.264793:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.264803:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513092.264813:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.118@tcp1 : PUT - for me 00000400:00000200:16.0:1616513092.264824:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.118@tcp1 of length 224 into portal 26 MB=0x5ffd5d40b6980 00000400:00000200:16.0:1616513092.264831:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.118@tcp1 of length 224/224 into md 0xf97ce4d9 [1] + 0 00000400:00000200:16.0:1616513092.264836:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05760 00000400:00000200:16.0:1616513092.264840:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513092.264843:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513092.264848:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513092.264850:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05760 00000800:00000200:16.0:1616513092.264854:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513092.264857:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513092.264858:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513092.264859:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513092.264861:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513092.264868:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.118@tcp1 : PUT - for me 00000400:00000200:24.0:1616513092.264878:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.118@tcp1 of length 224 into portal 12 MB=0x5ffd5d40b6a00 00000400:00000200:24.0:1616513092.264884:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.118@tcp1 of length 224/224 into md 0x865be291 [1] + 35608 00000400:00000200:24.0:1616513092.264890:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.264892:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513092.264901:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.264904:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.264906:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.264917:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668734187904 00010000:00000200:0.0:1616513092.264928:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688668734187904/t0(0) o400->583c8ad1-ce93-a6e3-50de-c66a17b1f9be@10.31.132.118@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.264942:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688668734187904, offset 192 00000400:00000200:0.0:1616513092.264948:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.118@tcp1 00000400:00000200:0.0:1616513092.264955:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.118@tcp1 routed destination 00000100:00000200:12.0:1616513092.264963:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688668734188032 00000400:00000200:0.0:1616513092.264970:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.118@tcp1(10.31.132.118@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:12.0:1616513092.264975:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cede80 x1688668734188032/t0(0) o400->41a40581-ff91-b9be-5df5-d9c0fdf39245@10.31.132.118@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513092.264976:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.264982:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.264985:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.264987:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513092.264989:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688668734188032, offset 192 00000800:00000200:0.0:1616513092.264992:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513092.264995:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.118@tcp1 00000100:00000200:0.0:1616513092.264999:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513092.265001:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513092.265002:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.118@tcp1 routed destination 00000400:00000200:12.0:1616513092.265016:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.118@tcp1(10.31.132.118@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.265022:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513092.265028:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.265030:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.265032:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.265037:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.265042:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513092.265058:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.265066:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513092.265068:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513092.265075:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513092.265081:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faeaa0 00000800:00000200:18.0:1616513092.265084:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.265087:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513092.265096:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.265104:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513092.265107:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513092.265107:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513092.265114:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513092.265118:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e5d8 00000800:00000200:20.0:1616513092.265121:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.266945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513092.267000:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.267010:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513092.267019:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.41@tcp1 : PUT - for me 00000400:00000200:22.0:1616513092.267030:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.41@tcp1 of length 224 into portal 26 MB=0x604a2ab7c8e00 00000400:00000200:22.0:1616513092.267036:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.41@tcp1 of length 224/224 into md 0xf97ce4e9 [1] + 0 00000400:00000200:22.0:1616513092.267041:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05c28 00000400:00000200:22.0:1616513092.267043:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513092.267046:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513092.267050:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05c28 00000800:00000200:22.0:1616513092.267054:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.267056:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513092.267056:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.267062:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513092.267065:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.267066:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513092.267075:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.41@tcp1 : PUT - for me 00000400:00000200:14.0:1616513092.267086:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.41@tcp1 of length 224 into portal 12 MB=0x604a2ab7c8f00 00000400:00000200:14.0:1616513092.267092:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.41@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 2464 00000400:00000200:14.0:1616513092.267097:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513092.267100:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513092.267107:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.267111:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.267113:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.267113:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946568543744 00010000:00000200:0.0:1616513092.267125:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1693946568543744/t0(0) o400->72366ffd-69b8-ae3f-2ab8-93024e9662dd@10.31.141.41@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513092.267127:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946568544000 00010000:00000200:30.0:1616513092.267137:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e04c80 x1693946568544000/t0(0) o400->3458c15e-7302-55a1-466e-01155bfbe637@10.31.141.41@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513092.267140:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946568543744, offset 192 00000400:00000200:0.0:1616513092.267147:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.41@tcp1 00000100:00000200:30.0:1616513092.267150:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946568544000, offset 192 00000400:00000200:0.0:1616513092.267154:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.41@tcp1 routed destination 00000400:00000200:30.0:1616513092.267156:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.41@tcp1 00000400:00000200:30.0:1616513092.267163:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.41@tcp1 routed destination 00000400:00000200:30.0:1616513092.267177:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.41@tcp1(10.31.141.41@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513092.267178:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.41@tcp1(10.31.141.41@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.267183:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.267185:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513092.267189:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.267191:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.267191:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.267193:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.267194:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.267196:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513092.267198:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513092.267201:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.267206:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.267207:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513092.267245:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513092.267269:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513092.267277:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513092.267280:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513092.267281:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513092.267286:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.267292:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafb18 00000800:00000200:2.0:1616513092.267295:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513092.267300:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513092.267307:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513092.267309:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513092.267316:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513092.267320:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eecc0 00000800:00000200:24.0:1616513092.267324:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.267326:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.290233:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513092.290289:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.290309:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513092.290317:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.146@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.290326:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.146@tcp1 of length 224 into portal 26 MB=0x603fce4359780 00000400:00000200:18.0:1616513092.290332:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.146@tcp1 of length 224/224 into md 0xf97ce4f9 [1] + 0 00000400:00000200:18.0:1616513092.290336:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04000 00000400:00000200:18.0:1616513092.290339:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.290342:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513092.290346:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04000 00000800:00000200:18.0:1616513092.290349:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.290351:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.290353:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.290355:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.290366:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513092.290375:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.146@tcp1 : PUT - for me 00000400:00000200:14.0:1616513092.290384:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.146@tcp1 of length 224 into portal 12 MB=0x603fce4359800 00000400:00000200:14.0:1616513092.290390:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.146@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 2688 00000400:00000200:14.0:1616513092.290396:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513092.290399:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513092.290405:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.290409:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513092.290410:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513092.290410:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234555623296 00000800:00000200:14.0:1616513092.290413:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.290420:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234555623424 00010000:00000200:0.0:1616513092.290421:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1693234555623296/t0(0) o400->e6cca58f-ba3a-af11-3e2f-bb807e5e35c0@10.31.138.146@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513092.290429:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05580 x1693234555623424/t0(0) o400->2fe51475-4b52-9e77-87f7-133d63e7d091@10.31.138.146@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.290447:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234555623424, offset 192 00000100:00000200:0.0:1616513092.290449:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234555623296, offset 192 00000400:00000200:30.0:1616513092.290453:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.146@tcp1 00000400:00000200:0.0:1616513092.290456:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.146@tcp1 00000400:00000200:30.0:1616513092.290459:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.146@tcp1 routed destination 00000400:00000200:0.0:1616513092.290463:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.146@tcp1 routed destination 00000400:00000200:30.0:1616513092.290472:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.146@tcp1(10.31.138.146@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.290478:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513092.290479:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.146@tcp1(10.31.138.146@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.290483:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.290486:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.290486:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513092.290488:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.290491:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.290492:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513092.290494:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513092.290496:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.290496:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.290501:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513092.290508:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513092.290509:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.290567:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513092.290568:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513092.290574:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513092.290575:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:16.0:1616513092.290577:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513092.290578:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513092.290579:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513092.290584:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.290584:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.290588:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef4b8 00000400:00000200:16.0:1616513092.290589:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae770 00000800:00000200:16.0:1616513092.290593:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513092.290593:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513092.290596:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.366927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513092.366986:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.366997:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513092.367006:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.249@tcp1 : PUT - for me 00000400:00000200:22.0:1616513092.367017:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.249@tcp1 of length 224 into portal 26 MB=0x60183c923ab00 00000400:00000200:22.0:1616513092.367024:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.249@tcp1 of length 224/224 into md 0xf97ce509 [1] + 0 00000400:00000200:22.0:1616513092.367029:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05188 00000400:00000200:22.0:1616513092.367032:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513092.367035:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513092.367040:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05188 00000800:00000200:14.0:1616513092.367042:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.367044:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.367047:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.367049:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.367109:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515387165440 00010000:00000200:0.0:1616513092.367121:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1690515387165440/t0(0) o400->85c0fc95-f346-7908-37d7-788d89a59cfc@10.31.141.249@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.367135:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690515387165440, offset 192 00000400:00000200:0.0:1616513092.367141:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.249@tcp1 00000400:00000200:0.0:1616513092.367150:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.249@tcp1 routed destination 00000400:00000200:0.0:1616513092.367166:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.249@tcp1(10.31.141.249@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.367172:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.367179:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.367181:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.367184:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.367189:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.367195:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.367196:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513092.367248:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.367256:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513092.367261:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513092.367268:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513092.367274:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafed0 00000800:00000200:18.0:1616513092.367277:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.367280:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513092.367342:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513092.367353:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.367361:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513092.367363:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513092.367370:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.249@tcp1 : PUT - for me 00000400:00000200:14.0:1616513092.367379:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.249@tcp1 of length 224 into portal 12 MB=0x60183c923b300 00000400:00000200:14.0:1616513092.367386:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.249@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 2912 00000400:00000200:14.0:1616513092.367391:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513092.367394:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513092.367402:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.367405:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.367407:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.367422:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515387167488 00010000:00000200:30.0:1616513092.367433:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e00d80 x1690515387167488/t0(0) o400->97978669-8a33-e723-9895-eb2f07a159ba@10.31.141.249@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.367455:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690515387167488, offset 192 00000400:00000200:30.0:1616513092.367461:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.249@tcp1 00000400:00000200:30.0:1616513092.367468:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.249@tcp1 routed destination 00000400:00000200:30.0:1616513092.367489:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.249@tcp1(10.31.141.249@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.367495:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513092.367500:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.367503:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513092.367505:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513092.367510:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.367518:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.367574:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513092.367581:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513092.367584:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513092.367590:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513092.367595:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef3a8 00000800:00000200:24.0:1616513092.367599:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.367601:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.367608:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513092.367610:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513092.367612:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513092.367613:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.367616:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.381392:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513092.381451:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.381460:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513092.381469:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.15@tcp1 : PUT - for me 00000400:00000200:20.0:1616513092.381479:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.15@tcp1 of length 224 into portal 26 MB=0x5ffd4be77ea80 00000400:00000200:20.0:1616513092.381486:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.15@tcp1 of length 224/224 into md 0xf97ce519 [1] + 0 00000400:00000200:20.0:1616513092.381490:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb058f8 00000400:00000200:20.0:1616513092.381495:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.381498:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513092.381502:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb058f8 00000800:00000200:24.0:1616513092.381505:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.381505:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.381509:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513092.381511:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513092.381514:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.381515:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513092.381524:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.15@tcp1 : PUT - for me 00000400:00000200:24.0:1616513092.381533:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.15@tcp1 of length 224 into portal 12 MB=0x5ffd4be77eb40 00000400:00000200:24.0:1616513092.381540:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.15@tcp1 of length 224/224 into md 0x865be291 [1] + 35832 00000400:00000200:24.0:1616513092.381547:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.381550:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513092.381557:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.381561:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.381563:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.381567:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664077232768 00010000:00000200:0.0:1616513092.381580:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688664077232768/t0(0) o400->3f3e18f1-3159-f8b2-829b-b6a8fe0aa931@10.31.136.15@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.381594:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664077232768, offset 192 00000400:00000200:0.0:1616513092.381601:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.15@tcp1 00000400:00000200:0.0:1616513092.381608:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.15@tcp1 routed destination 00000100:00000200:12.0:1616513092.381623:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664077232960 00000400:00000200:0.0:1616513092.381624:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.15@tcp1(10.31.136.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.381631:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:12.0:1616513092.381635:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cef500 x1688664077232960/t0(0) o400->cc9a41c6-6fbe-820c-e9dd-d468a54a26c7@10.31.136.15@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513092.381637:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.381640:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.381642:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.381647:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513092.381650:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664077232960, offset 192 00000800:00000200:7.2:1616513092.381655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.381655:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513092.381656:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.15@tcp1 00000400:00000200:12.0:1616513092.381664:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.15@tcp1 routed destination 00000800:00000200:18.0:1616513092.381670:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.381676:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513092.381679:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513092.381679:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.15@tcp1(10.31.136.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513092.381685:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513092.381685:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513092.381690:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae3b8 00000800:00000200:12.0:1616513092.381691:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513092.381693:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513092.381694:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513092.381696:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513092.381696:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.381701:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.381706:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.381761:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.381768:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513092.381771:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513092.381772:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513092.381777:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.381782:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f298 00000800:00000200:16.0:1616513092.381785:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.505981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.506001:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.506011:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.506020:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.181@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.506030:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.181@tcp1 of length 224 into portal 26 MB=0x5ffdb17c9bac0 00000400:00000200:28.0:1616513092.506036:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.181@tcp1 of length 224/224 into md 0xf97ce529 [1] + 0 00000400:00000200:28.0:1616513092.506041:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04440 00000400:00000200:28.0:1616513092.506045:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.506049:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513092.506053:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04440 00000800:00000200:24.0:1616513092.506056:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.506057:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.506060:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513092.506063:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.506065:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.506066:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513092.506075:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.181@tcp1 : PUT - for me 00000400:00000200:24.0:1616513092.506085:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.181@tcp1 of length 224 into portal 12 MB=0x5ffdb17c9bcc0 00000400:00000200:24.0:1616513092.506091:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.181@tcp1 of length 224/224 into md 0x865be291 [1] + 36056 00000400:00000200:24.0:1616513092.506097:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.506100:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513092.506107:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.506111:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.506113:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.506123:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691345570496 00010000:00000200:0.0:1616513092.506136:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688691345570496/t0(0) o400->fea2882a-9e36-7067-0f84-70789ce2f644@10.31.140.181@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.506152:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688691345570496, offset 192 00000400:00000200:0.0:1616513092.506158:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.181@tcp1 00000400:00000200:0.0:1616513092.506166:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.181@tcp1 routed destination 00000100:00000200:12.0:1616513092.506173:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688691345571008 00010000:00000200:12.0:1616513092.506187:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cec800 x1688691345571008/t0(0) o400->7558fd43-afc9-e32a-6e6d-78d56bdf0bf5@10.31.140.181@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513092.506188:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.181@tcp1(10.31.140.181@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.506195:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.506202:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.506207:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.506209:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513092.506213:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688691345571008, offset 192 00000800:00000200:0.0:1616513092.506216:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:12.0:1616513092.506219:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.181@tcp1 00000100:00000200:0.0:1616513092.506223:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513092.506224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513092.506226:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.181@tcp1 routed destination 00000400:00000200:12.0:1616513092.506242:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.181@tcp1(10.31.140.181@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.506248:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513092.506254:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.506257:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.506259:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.506264:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.506270:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513092.506281:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.506289:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513092.506292:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513092.506298:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513092.506304:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf540 00000800:00000200:22.0:1616513092.506308:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.506311:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513092.506325:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.506332:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513092.506335:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513092.506336:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513092.506342:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.506346:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f430 00000800:00000200:16.0:1616513092.506350:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.569894:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513092.569953:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.569964:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513092.569974:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.47@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.569984:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.47@tcp1 of length 224 into portal 26 MB=0x5ffe7a7a84d80 00000400:00000200:18.0:1616513092.569991:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.47@tcp1 of length 224/224 into md 0xf97ce539 [1] + 0 00000400:00000200:18.0:1616513092.569996:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05210 00000400:00000200:18.0:1616513092.570000:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.570003:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513092.570007:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05210 00000800:00000200:2.0:1616513092.570010:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.570011:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.570014:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513092.570018:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.570021:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.570022:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513092.570031:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.47@tcp1 : PUT - for me 00000400:00000200:2.0:1616513092.570040:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.47@tcp1 of length 224 into portal 12 MB=0x5ffe7a7a84fc0 00000400:00000200:2.0:1616513092.570046:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.47@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 3136 00000400:00000200:2.0:1616513092.570052:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513092.570055:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513092.570071:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513092.570074:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.570078:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.570083:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688745298906496 00000100:00000200:30.0:1616513092.570093:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688745298907072 00010000:00000200:0.0:1616513092.570097:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688745298906496/t0(0) o400->4eccf5a4-34c1-4e09-0017-b6d3695683dd@10.31.140.47@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513092.570107:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0480 x1688745298907072/t0(0) o400->a67559db-2c4d-924c-f043-263bf3346d85@10.31.140.47@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513092.570113:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688745298906496, offset 192 00000400:00000200:0.0:1616513092.570120:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.47@tcp1 00000100:00000200:30.0:1616513092.570122:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688745298907072, offset 192 00000400:00000200:30.0:1616513092.570128:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.47@tcp1 00000400:00000200:0.0:1616513092.570128:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.47@tcp1 routed destination 00000400:00000200:30.0:1616513092.570135:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.47@tcp1 routed destination 00000400:00000200:30.0:1616513092.570151:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.47@tcp1(10.31.140.47@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513092.570151:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.47@tcp1(10.31.140.47@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.570157:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.570158:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513092.570164:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.570165:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.570166:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513092.570168:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.570168:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.570170:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513092.570174:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513092.570175:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.570181:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513092.570183:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513092.570183:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513092.570236:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.570243:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.570246:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513092.570246:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513092.570253:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513092.570255:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513092.570260:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513092.570260:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae220 00000400:00000200:24.0:1616513092.570263:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513092.570264:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513092.570270:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513092.570274:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee908 00000800:00000200:24.0:1616513092.570278:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.570281:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.587896:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.587952:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.587962:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.587972:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.248@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.587981:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.248@tcp1 of length 224 into portal 12 MB=0x5ffdae5ea5900 00000400:00000200:28.0:1616513092.587988:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.248@tcp1 of length 224/224 into md 0x865be291 [1] + 36280 00000400:00000200:28.0:1616513092.587993:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.587996:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513092.588003:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.588006:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513092.588008:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.588012:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513092.588068:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688690508847360 00010000:00000200:12.0:1616513092.588082:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed8d80 x1688690508847360/t0(0) o400->78cf3851-677f-4551-44bb-2a475fc41339@10.31.140.248@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513092.588097:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688690508847360, offset 192 00000400:00000200:12.0:1616513092.588103:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.248@tcp1 00000400:00000200:12.0:1616513092.588111:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.248@tcp1 routed destination 00000800:00000200:7.2:1616513092.588118:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513092.588126:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.248@tcp1(10.31.140.248@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.588132:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513092.588138:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.588141:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513092.588144:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513092.588149:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513092.588174:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.588184:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513092.588195:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.248@tcp1 : PUT - for me 00000400:00000200:22.0:1616513092.588204:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.248@tcp1 of length 224 into portal 26 MB=0x5ffdae5ea5880 00000400:00000200:22.0:1616513092.588211:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.248@tcp1 of length 224/224 into md 0xf97ce559 [1] + 0 00000400:00000200:22.0:1616513092.588216:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04908 00000400:00000200:22.0:1616513092.588218:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513092.588221:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513092.588225:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04908 00000800:00000200:2.0:1616513092.588228:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513092.588229:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513092.588232:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:22.0:1616513092.588239:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513092.588240:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513092.588242:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513092.588243:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513092.588250:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.588254:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f078 00000800:00000200:2.0:1616513092.588258:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.588311:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688690508847232 00010000:00000200:0.0:1616513092.588323:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688690508847232/t0(0) o400->5d0b01ab-b7a9-e27c-ac4f-bf303ead5368@10.31.140.248@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.588337:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688690508847232, offset 192 00000400:00000200:0.0:1616513092.588344:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.248@tcp1 00000400:00000200:0.0:1616513092.588351:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.248@tcp1 routed destination 00000400:00000200:0.0:1616513092.588366:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.248@tcp1(10.31.140.248@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.588373:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.588378:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.588381:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.588383:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.588388:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.588394:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513092.588395:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513092.588406:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513092.588413:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513092.588416:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513092.588423:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513092.588428:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf760 00000800:00000200:24.0:1616513092.588432:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.588449:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513092.602525:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928448, portal 4 00000100:00000200:10.0:1616513092.602530:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928448, offset 0 00000400:00000200:10.0:1616513092.602535:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:10.0:1616513092.602541:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:10.0:1616513092.602551:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513092.602557:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:10.0:1616513092.602562:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513092.602565:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:10.0:1616513092.602567:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:10.0:1616513092.602571:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:27.0:1616513092.602585:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928464, portal 4 00000100:00000200:27.0:1616513092.602591:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928464, offset 0 00000400:00000200:27.0:1616513092.602599:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:27.0:1616513092.602606:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:1.2:1616513092.602618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:27.0:1616513092.602619:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513092.602627:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513092.602634:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513092.602637:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513092.602640:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:1.0:1616513092.602640:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:27.0:1616513092.602646:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:1.0:1616513092.602647:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513092.602650:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:27.2:1616513092.602654:0:6506:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:1.0:1616513092.602658:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513092.602666:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9645a00 x1693067725928448/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513099 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513092.602679:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82198 00000800:00000200:1.0:1616513092.602684:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513092.602687:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:3.0:1616513092.602717:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:3.0:1616513092.602725:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:3.0:1616513092.602728:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.0:1616513092.602729:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:3.0:1616513092.602735:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:3.0:1616513092.602741:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be40480 x1693067725928464/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513099 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513092.602754:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249c28 00000800:00000200:3.0:1616513092.602758:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:1.2:1616513092.602794:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513092.602804:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513092.602811:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513092.602814:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513092.602819:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513092.602829:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627800 00000400:00000200:1.0:1616513092.602836:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700025 [1] + 192 00000400:00000200:1.0:1616513092.602841:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513092.602846:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9645a00 x1693067725928448/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513099 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513092.602872:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513092.602875:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513092.602877:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:10.0:1616513092.602882:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9645a00 x1693067725928448/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513099 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.2:1616513092.602884:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513092.602894:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:10.0:1616513092.602895:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9645a00 x1693067725928448/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513099 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513092.602899:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:10.0:1616513092.602905:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83980 00000400:00000200:27.0:1616513092.602907:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513092.602913:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627810 00000400:00000200:27.0:1616513092.602918:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371670fd [1] + 192 00000400:00000200:27.0:1616513092.602923:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513092.602927:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be40480 x1693067725928464/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513099 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513092.602942:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513092.602945:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:31.0:1616513092.602947:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513092.602951:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:3.0:1616513092.602951:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be40480 x1693067725928464/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513099 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513092.602962:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be40480 x1693067725928464/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513099 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513092.602982:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248660 00000800:00000200:11.2:1616513092.654664:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513092.654725:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.654735:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513092.654745:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.139@tcp1 : PUT - for me 00000400:00000200:16.0:1616513092.654755:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.139@tcp1 of length 224 into portal 26 MB=0x5ffde7805a6c0 00000400:00000200:16.0:1616513092.654762:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.139@tcp1 of length 224/224 into md 0xf97ce569 [1] + 0 00000400:00000200:16.0:1616513092.654767:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb056d8 00000400:00000200:16.0:1616513092.654771:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513092.654773:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513092.654780:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513092.654781:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb056d8 00000800:00000200:16.0:1616513092.654785:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513092.654788:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513092.654790:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.654792:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513092.654793:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513092.654801:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.139@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.654810:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.139@tcp1 of length 224 into portal 12 MB=0x5ffde7805a780 00000400:00000200:28.0:1616513092.654817:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.139@tcp1 of length 224/224 into md 0x865be291 [1] + 36504 00000400:00000200:28.0:1616513092.654823:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.654826:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513092.654833:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.654837:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.654839:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.654851:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705845012160 00010000:00000200:0.0:1616513092.654865:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688705845012160/t0(0) o400->9c54cf0c-38c2-1d64-8efe-fc6026cfa11f@10.31.143.139@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.654881:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705845012160, offset 192 00000400:00000200:0.0:1616513092.654887:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.139@tcp1 00000400:00000200:0.0:1616513092.654895:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.139@tcp1 routed destination 00000100:00000200:12.0:1616513092.654909:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705845012352 00000400:00000200:0.0:1616513092.654910:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.139@tcp1(10.31.143.139@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.654917:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:12.0:1616513092.654923:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed9200 x1688705845012352/t0(0) o400->f811ad49-3796-5a84-0c50-ee7dbff9c7b0@10.31.143.139@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513092.654924:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.654927:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.654930:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.654936:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513092.654938:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705845012352, offset 192 00000100:00000200:0.0:1616513092.654943:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513092.654944:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.139@tcp1 00000400:00000200:12.0:1616513092.654952:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.139@tcp1 routed destination 00000400:00000200:12.0:1616513092.654967:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.139@tcp1(10.31.143.139@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.654976:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513092.654982:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.654985:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513092.654986:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513092.654987:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.654992:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.655007:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513092.655044:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.655051:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513092.655054:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513092.655061:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513092.655062:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513092.655067:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafdc0 00000800:00000200:20.0:1616513092.655070:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513092.655070:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513092.655073:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513092.655073:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513092.655074:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513092.655080:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513092.655084:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa90 00000800:00000200:20.0:1616513092.655088:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.656612:0:50813:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513092.656669:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513092.656679:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513092.656688:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.229@tcp1 : PUT - for me 00000400:00000200:2.0:1616513092.656698:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.229@tcp1 of length 224 into portal 12 MB=0x6050e2fbfc080 00000400:00000200:2.0:1616513092.656705:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.229@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 3360 00000400:00000200:2.0:1616513092.656710:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513092.656712:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513092.656720:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513092.656723:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.656724:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513092.656729:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.656741:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694408349040768 00010000:00000200:30.0:1616513092.656753:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3600 x1694408349040768/t0(0) o400->c1d5071c-7021-e6f0-a3f7-fd883a44eef7@10.31.142.229@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.656776:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694408349040768, offset 192 00000400:00000200:30.0:1616513092.656783:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.229@tcp1 00000400:00000200:30.0:1616513092.656791:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.229@tcp1 routed destination 00000400:00000200:30.0:1616513092.656805:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.229@tcp1(10.31.142.229@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.656812:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513092.656818:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.656821:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513092.656823:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513092.656828:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.656832:0:50813:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513092.656855:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513092.656899:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.656906:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513092.656909:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513092.656915:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513092.656917:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513092.656919:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee110 00000800:00000200:14.0:1616513092.656923:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513092.656926:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513092.656926:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513092.656935:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.229@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.656944:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.229@tcp1 of length 224 into portal 26 MB=0x6050e2fbfc000 00000400:00000200:28.0:1616513092.656950:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.229@tcp1 of length 224/224 into md 0xf97ce579 [1] + 0 00000400:00000200:28.0:1616513092.656957:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05f58 00000400:00000200:28.0:1616513092.656960:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.656963:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513092.656967:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05f58 00000800:00000200:28.0:1616513092.656970:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.656973:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513092.656975:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.656979:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.657030:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694408349040640 00010000:00000200:0.0:1616513092.657042:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1694408349040640/t0(0) o400->e2a77009-bf64-e083-c83f-a019c3b716e6@10.31.142.229@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.657056:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694408349040640, offset 192 00000400:00000200:0.0:1616513092.657062:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.229@tcp1 00000400:00000200:0.0:1616513092.657070:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.229@tcp1 routed destination 00000400:00000200:0.0:1616513092.657085:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.229@tcp1(10.31.142.229@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.657092:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.657097:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.657100:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.657102:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.657107:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.657112:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513092.657113:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513092.657164:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.657171:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513092.657173:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.657180:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.657186:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae220 00000800:00000200:16.0:1616513092.657189:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.657192:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:27.2:1616513092.660379:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4b5835000] (20)++ 00000800:00000200:27.0:1616513092.660399:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4b5835000] (21)++ 00000800:00000200:27.0:1616513092.660407:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.223@o2ib1 00000400:00000200:27.0:1616513092.660417:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.223@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513092.660426:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.223@o2ib1 of length 224 into portal 26 MB=0x5ff41b5030a00 00000400:00000200:27.0:1616513092.660446:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.223@o2ib1 of length 224/224 into md 0xf97ce5a9 [1] + 0 00000400:00000200:27.0:1616513092.660451:0:6475:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05dc0 00000800:00000200:3.0:1616513092.660452:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4b5835000] (22)++ 00000400:00000200:27.0:1616513092.660456:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513092.660459:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:3.0:1616513092.660463:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.223@o2ib1 00000400:00000200:27.0:1616513092.660465:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05dc0 00000800:00000200:27.0:1616513092.660469:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4b5835000] (23)++ 00000800:00000200:27.0:1616513092.660471:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4b5835000] (24)-- 00000400:00000200:3.0:1616513092.660472:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.223@o2ib1 : PUT - for me 00000800:00000200:27.0:1616513092.660473:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4b5835000] (23)-- 00000800:00000200:27.0:1616513092.660476:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4b5835000] (22)-- 00000400:00000200:3.0:1616513092.660480:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.223@o2ib1 of length 224 into portal 12 MB=0x5ff41b5030ac0 00000400:00000200:3.0:1616513092.660487:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.223@o2ib1 of length 224/224 into md 0x3657608d [1] + 92064 00000400:00000200:3.0:1616513092.660492:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:3.0:1616513092.660495:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:3.0:1616513092.660503:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4b5835000] (21)++ 00000800:00000200:3.0:1616513092.660506:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4b5835000] (22)-- 00000800:00000200:3.0:1616513092.660508:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4b5835000] (21)-- 00000100:00000200:0.0:1616513092.660532:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688032558385664 00010000:00000200:0.0:1616513092.660543:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688032558385664/t0(0) o400->d9720355-160a-b866-a529-6d02b6e7c8d4@10.242.46.223@o2ib1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.660558:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688032558385664, offset 192 00000400:00000200:0.0:1616513092.660566:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.223@o2ib1 00000100:00000200:23.0:1616513092.660573:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688032558385856 00000400:00000200:0.0:1616513092.660574:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.223@o2ib1 local destination 00000400:00000200:0.0:1616513092.660588:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.223@o2ib1(10.242.46.223@o2ib1:10.242.46.223@o2ib1) : PUT try# 0 00010000:00000200:23.0:1616513092.660589:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8da4dd604c80 x1688032558385856/t0(0) o400->c8ef6874-f8d5-3996-53a5-926493836775@10.242.46.223@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513092.660595:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.223@o2ib1 00000800:00000200:0.0:1616513092.660602:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4da9fce80] -> 10.242.46.223@o2ib1 (2) version: 12 00000100:00000200:23.0:1616513092.660605:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688032558385856, offset 192 00000800:00000200:0.0:1616513092.660605:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4b5835000] (20)++ 00000800:00000200:0.0:1616513092.660607:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4b5835000] (21)++ 00000400:00000200:23.0:1616513092.660612:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.223@o2ib1 00000800:00000200:0.0:1616513092.660612:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4b5835000] (22)-- 00000800:00000200:27.2:1616513092.660618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4b5835000] (21)++ 00000100:00000200:0.0:1616513092.660619:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:23.0:1616513092.660620:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.223@o2ib1 local destination 00000800:00000200:27.0:1616513092.660626:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4b5835000] (22)++ 00000800:00000200:27.0:1616513092.660631:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4b5835000] (23)-- 00000400:00000200:27.0:1616513092.660634:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:23.0:1616513092.660634:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.223@o2ib1(10.242.46.223@o2ib1:10.242.46.223@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513092.660636:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4b5835000] (22)-- 00000400:00000200:27.0:1616513092.660639:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.223@o2ib1: PUT: OK 00000800:00000200:23.0:1616513092.660641:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.223@o2ib1 00000400:00000200:27.0:1616513092.660645:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae3b8 00000800:00000200:27.0:1616513092.660648:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4b5835000] (21)-- 00000800:00000200:23.0:1616513092.660648:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4da9fce80] -> 10.242.46.223@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513092.660650:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4b5835000] (20)++ 00000800:00000200:23.0:1616513092.660653:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4b5835000] (21)++ 00000800:00000200:23.0:1616513092.660658:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4b5835000] (22)-- 00000800:00000200:27.2:1616513092.660663:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4b5835000] (21)++ 00000800:00000200:27.0:1616513092.660671:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4b5835000] (22)++ 00000800:00000200:27.0:1616513092.660674:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4b5835000] (23)-- 00000400:00000200:27.0:1616513092.660676:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513092.660679:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4b5835000] (22)-- 00000400:00000200:27.0:1616513092.660681:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.223@o2ib1: PUT: OK 00000400:00000200:27.0:1616513092.660683:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e770 00000800:00000200:27.0:1616513092.660685:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4b5835000] (21)-- 00000800:00000200:7.2:1616513092.665225:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513092.665278:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.665287:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513092.665296:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.182@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.665304:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.182@tcp1 of length 224 into portal 12 MB=0x5ffd54fda4b40 00000400:00000200:18.0:1616513092.665310:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.182@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 3584 00000400:00000200:18.0:1616513092.665314:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.665319:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513092.665326:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.665328:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.665330:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.665333:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.665339:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666516376384 00010000:00000200:30.0:1616513092.665346:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd700000 x1688666516376384/t0(0) o400->4e072d5f-0564-b949-e6a8-57ff6b84e9b8@10.31.138.182@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.665358:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666516376384, offset 192 00000400:00000200:30.0:1616513092.665363:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.182@tcp1 00000400:00000200:30.0:1616513092.665369:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.182@tcp1 routed destination 00000400:00000200:30.0:1616513092.665382:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.182@tcp1(10.31.138.182@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.665387:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513092.665392:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.665395:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513092.665396:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513092.665400:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.665405:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513092.665458:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513092.665460:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.665467:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513092.665470:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513092.665476:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513092.665480:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee220 00000800:00000200:14.0:1616513092.665484:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.665487:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513092.665514:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.665524:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513092.665533:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.182@tcp1 : PUT - for me 00000400:00000200:24.0:1616513092.665541:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.182@tcp1 of length 224 into portal 26 MB=0x5ffd54fda4ac0 00000400:00000200:24.0:1616513092.665550:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.182@tcp1 of length 224/224 into md 0xf97ce5b9 [1] + 0 00000400:00000200:24.0:1616513092.665554:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05320 00000400:00000200:24.0:1616513092.665557:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.665560:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513092.665564:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05320 00000800:00000200:24.0:1616513092.665567:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.665568:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.665570:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513092.665572:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.665629:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666516376256 00010000:00000200:0.0:1616513092.665640:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688666516376256/t0(0) o400->1d8622e9-f4e6-e8fa-3610-a44fce898590@10.31.138.182@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.665654:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666516376256, offset 192 00000400:00000200:0.0:1616513092.665660:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.182@tcp1 00000400:00000200:0.0:1616513092.665668:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.182@tcp1 routed destination 00000400:00000200:0.0:1616513092.665683:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.182@tcp1(10.31.138.182@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.665689:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.665695:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.665698:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.665700:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.665705:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513092.665711:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513092.665719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.665773:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.665780:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513092.665783:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.665796:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.665802:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae770 00000800:00000200:16.0:1616513092.665805:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.665808:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.677392:0:50815:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513092.677449:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.677459:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513092.677468:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.79@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.677477:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.79@tcp1 of length 224 into portal 26 MB=0x5ffd4d95ecb40 00000400:00000200:18.0:1616513092.677483:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.79@tcp1 of length 224/224 into md 0xf97ce5d9 [1] + 0 00000400:00000200:18.0:1616513092.677488:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05a08 00000400:00000200:18.0:1616513092.677491:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.677493:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513092.677497:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a08 00000800:00000200:18.0:1616513092.677500:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.677503:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513092.677504:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.677509:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.677563:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664528571200 00010000:00000200:0.0:1616513092.677574:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688664528571200/t0(0) o400->5f5070a6-29af-f33c-5885-f0d8953d635b@10.31.136.79@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.677588:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664528571200, offset 192 00000400:00000200:0.0:1616513092.677594:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.79@tcp1 00000400:00000200:0.0:1616513092.677602:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.79@tcp1 routed destination 00000400:00000200:0.0:1616513092.677617:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.79@tcp1(10.31.136.79@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.677632:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513092.677643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.677648:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.677651:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.677653:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.677658:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.677663:0:50815:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.677664:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513092.677700:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.677708:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.677717:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.79@tcp1 : PUT - for me 00000800:00000200:14.0:1616513092.677724:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513092.677725:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.79@tcp1 of length 224 into portal 12 MB=0x5ffd4d95ecc40 00000400:00000200:28.0:1616513092.677730:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.79@tcp1 of length 224/224 into md 0x865be291 [1] + 36728 00000800:00000200:14.0:1616513092.677730:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513092.677733:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.677735:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.677737:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513092.677739:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513092.677743:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513092.677745:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faeaa0 00000800:00000200:28.0:1616513092.677746:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.677748:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513092.677748:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513092.677750:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513092.677751:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513092.677812:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664528571456 00010000:00000200:12.0:1616513092.677824:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedb600 x1688664528571456/t0(0) o400->3118e50d-a110-18d9-7e04-3598e9695727@10.31.136.79@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513092.677839:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664528571456, offset 192 00000400:00000200:12.0:1616513092.677845:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.79@tcp1 00000400:00000200:12.0:1616513092.677853:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.79@tcp1 routed destination 00000400:00000200:12.0:1616513092.677868:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.79@tcp1(10.31.136.79@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.677874:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513092.677880:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.677883:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.677885:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.677890:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.677895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.677946:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.677953:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513092.677956:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.677962:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.677965:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa90 00000800:00000200:16.0:1616513092.677969:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.677972:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.679339:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.679395:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.679405:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.679414:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.192@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.679423:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.192@tcp1 of length 224 into portal 26 MB=0x5ffd41dbf54c0 00000400:00000200:28.0:1616513092.679429:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.192@tcp1 of length 224/224 into md 0xf97ce5e9 [1] + 0 00000400:00000200:28.0:1616513092.679445:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05650 00000400:00000200:28.0:1616513092.679448:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.679451:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513092.679452:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513092.679456:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05650 00000800:00000200:28.0:1616513092.679459:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.679462:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.679464:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.679520:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661380781248 00010000:00000200:0.0:1616513092.679531:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688661380781248/t0(0) o400->e55ddaf7-297d-868d-1986-b15781a83f21@10.31.132.192@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.679545:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661380781248, offset 192 00000400:00000200:0.0:1616513092.679551:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.192@tcp1 00000400:00000200:0.0:1616513092.679558:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.192@tcp1 routed destination 00000400:00000200:0.0:1616513092.679573:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.192@tcp1(10.31.132.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.679580:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.679585:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.679588:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.679590:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.679595:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.679602:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.679602:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513092.679658:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513092.679665:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513092.679667:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513092.679674:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.679680:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafa08 00000800:00000200:2.0:1616513092.679683:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.679686:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513092.679743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513092.679802:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.679811:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513092.679820:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.192@tcp1 : PUT - for me 00000400:00000200:16.0:1616513092.679829:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.192@tcp1 of length 224 into portal 12 MB=0x5ffd41dbf5680 00000400:00000200:16.0:1616513092.679835:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.192@tcp1 of length 224/224 into md 0x865be291 [1] + 36952 00000400:00000200:16.0:1616513092.679840:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513092.679843:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513092.679852:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.679855:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513092.679855:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513092.679859:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513092.679915:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661380781696 00010000:00000200:12.0:1616513092.679927:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b196300 x1688661380781696/t0(0) o400->828e3ea4-53a6-cf76-ee1f-2df1883fe785@10.31.132.192@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513092.679941:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661380781696, offset 192 00000400:00000200:12.0:1616513092.679947:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.192@tcp1 00000400:00000200:12.0:1616513092.679955:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.192@tcp1 routed destination 00000400:00000200:12.0:1616513092.679970:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.192@tcp1(10.31.132.192@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.679976:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513092.679982:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.679984:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.679987:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.679991:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.679997:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.680007:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513092.680013:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513092.680015:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513092.680017:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513092.680022:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513092.680026:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f078 00000800:00000200:24.0:1616513092.680029:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:5.2:1616513092.686961:0:50816:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513092.687019:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513092.687028:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513092.687036:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513092.687044:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 26 MB=0x5f931563c8d20 00000400:00000200:20.0:1616513092.687051:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0xf97ce5f9 [1] + 0 00000400:00000200:20.0:1616513092.687055:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04770 00000400:00000200:20.0:1616513092.687058:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.687061:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513092.687065:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04770 00000800:00000200:20.0:1616513092.687068:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513092.687071:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:16.0:1616513092.687072:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:20.0:1616513092.687079:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:20.0:1616513092.687081:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:16.0:1616513092.687083:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513092.687086:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000400:00000200:16.0:1616513092.687091:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513092.687093:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513092.687100:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8d60 00000400:00000200:16.0:1616513092.687100:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8e20 00000400:00000200:20.0:1616513092.687105:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 37176 00000400:00000200:20.0:1616513092.687109:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.687112:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 37400 00000100:00000200:20.0:1616513092.687117:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513092.687117:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513092.687122:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)++ 00000100:00000200:16.0:1616513092.687122:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513092.687124:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (24)-- 00000800:00000200:20.0:1616513092.687125:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:20.0:1616513092.687127:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000100:00000200:0.0:1616513092.687129:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179075872 00000800:00000200:20.0:1616513092.687131:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:16.0:1616513092.687131:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:16.0:1616513092.687136:0:6461:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (24)++ 00000400:00000200:20.0:1616513092.687137:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:16.0:1616513092.687140:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (25)-- 00010000:00000200:0.0:1616513092.687141:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1681365179075872/t0(0) o400->caa3ac69-e524-009d-3b77-bb4c680a29d9@10.242.46.230@o2ib1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513092.687142:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (24)-- 00000400:00000200:20.0:1616513092.687143:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8d80 00000800:00000200:16.0:1616513092.687144:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)++ 00000400:00000200:20.0:1616513092.687147:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 37624 00000800:00000200:24.0:1616513092.687148:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (24)++ 00000800:00000200:16.0:1616513092.687148:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513092.687152:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.687154:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513092.687154:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:24.0:1616513092.687160:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000400:00000200:16.0:1616513092.687160:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8d40 00000800:00000200:20.0:1616513092.687165:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (25)++ 00000100:00000200:0.0:1616513092.687165:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681365179075872, offset 192 00000800:00000200:20.0:1616513092.687167:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (26)-- 00000400:00000200:16.0:1616513092.687167:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 37848 00000800:00000200:20.0:1616513092.687168:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (25)-- 00000400:00000200:24.0:1616513092.687169:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000100:00000200:28.0:1616513092.687170:0:188449:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179075936 00000800:00000200:20.0:1616513092.687170:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (24)++ 00000400:00000200:16.0:1616513092.687171:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513092.687174:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000100:00000200:8.0:1616513092.687174:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076128 00000400:00000200:0.0:1616513092.687175:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:24.0:1616513092.687176:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8de0 00000100:00000200:16.0:1616513092.687177:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513092.687181:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:0.0:1616513092.687182:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00010000:00000200:28.0:1616513092.687185:0:188449:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dda00 x1681365179075936/t0(0) o400->boslfs02-MDT0001-lwp-OST0005_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:8.0:1616513092.687186:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b190900 x1681365179076128/t0(0) o400->boslfs02-MDT0001-lwp-OST0039_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513092.687189:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 38072 00000400:00000200:24.0:1616513092.687194:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.687196:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513092.687196:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000100:00000200:20.0:1616513092.687198:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179075968 00000800:00000200:16.0:1616513092.687200:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (25)++ 00000800:00000200:16.0:1616513092.687202:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (26)-- 00000800:00000200:0.0:1616513092.687202:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:16.0:1616513092.687204:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (25)-- 00000800:00000200:16.0:1616513092.687206:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (24)++ 00000100:00000200:28.0:1616513092.687207:0:188449:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179075936, offset 192 00000800:00000200:24.0:1616513092.687207:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (25)++ 00000800:00000200:16.0:1616513092.687208:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (25)-- 00000800:00000200:0.0:1616513092.687208:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00010000:00000200:20.0:1616513092.687209:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0dbf00 x1681365179075968/t0(0) o400->boslfs02-MDT0001-lwp-OST000d_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513092.687210:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (25)-- 00000100:00000200:8.0:1616513092.687210:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076128, offset 192 00000800:00000200:24.0:1616513092.687211:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (25)-- 00000800:00000200:0.0:1616513092.687211:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (25)++ 00000800:00000200:16.0:1616513092.687212:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)++ 00000400:00000200:28.0:1616513092.687214:0:188449:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:24.0:1616513092.687214:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (24)-- 00000800:00000200:0.0:1616513092.687215:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (25)++ 00000800:00000200:16.0:1616513092.687216:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:24.0:1616513092.687217:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (25)++ 00000400:00000200:8.0:1616513092.687219:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:28.0:1616513092.687220:0:188449:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:0.0:1616513092.687220:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (26)-- 00000800:00000200:24.0:1616513092.687221:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000100:00000200:12.0:1616513092.687221:0:170316:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179075904 00000100:00000200:20.0:1616513092.687222:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179075968, offset 192 00000400:00000200:20.0:1616513092.687227:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000100:00000200:0.0:1616513092.687227:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:12.0:1616513092.687233:0:170316:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6721680 x1681365179075904/t0(0) o400->boslfs02-MDT0001-lwp-OST0021_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513092.687234:0:188449:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513092.687236:0:188401:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076064 00000800:00000200:28.0:1616513092.687241:0:188449:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000400:00000200:8.0:1616513092.687242:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:16.0:1616513092.687243:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000100:00000200:12.0:1616513092.687248:0:170316:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179075904, offset 192 00010000:00000200:4.0:1616513092.687248:0:188401:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceec00 x1681365179076064/t0(0) o400->boslfs02-MDT0001-lwp-OST0011_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513092.687255:0:170316:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:8.0:1616513092.687256:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513092.687262:0:188401:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076064, offset 192 00000800:00000200:8.0:1616513092.687266:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000400:00000200:20.0:1616513092.687267:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:24.0:1616513092.687268:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:4.0:1616513092.687269:0:188401:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:20.0:1616513092.687280:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513092.687286:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513092.687287:0:188449:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513092.687290:0:188449:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (25)++ 00000400:00000200:12.0:1616513092.687290:0:170316:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:28.0:1616513092.687292:0:188449:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (26)++ 00000800:00000200:28.0:1616513092.687297:0:188449:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (27)-- 00000400:00000200:12.0:1616513092.687303:0:170316:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.687310:0:170316:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:8.0:1616513092.687310:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513092.687311:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (26)++ 00000400:00000200:4.0:1616513092.687312:0:188401:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:8.0:1616513092.687313:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (27)++ 00000800:00000200:28.0:1616513092.687315:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:8.0:1616513092.687315:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (28)++ 00000800:00000200:8.0:1616513092.687320:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (29)-- 00000400:00000200:4.0:1616513092.687326:0:188401:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513092.687333:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8e00 00000800:00000200:4.0:1616513092.687333:0:188401:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:20.0:1616513092.687334:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513092.687335:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8da0 00000800:00000200:12.0:1616513092.687335:0:170316:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513092.687337:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:20.0:1616513092.687337:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (28)++ 00000400:00000200:16.0:1616513092.687338:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 38296 00000800:00000200:12.0:1616513092.687338:0:170316:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (28)++ 00000800:00000200:4.0:1616513092.687338:0:188401:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513092.687339:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (29)++ 00000800:00000200:4.0:1616513092.687341:0:188401:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (31)++ 00000800:00000200:20.0:1616513092.687343:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (31)-- 00000800:00000200:12.0:1616513092.687343:0:170316:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (31)++ 00000400:00000200:28.0:1616513092.687344:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8e60 00000400:00000200:16.0:1616513092.687344:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513092.687346:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 38520 00000800:00000200:12.0:1616513092.687348:0:170316:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (32)-- 00000800:00000200:4.0:1616513092.687348:0:188401:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (32)++ 00000400:00000200:24.0:1616513092.687349:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513092.687350:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513092.687353:0:188401:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (32)-- 00000800:00000200:16.0:1616513092.687358:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (31)++ 00000400:00000200:20.0:1616513092.687359:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8dc0 00000800:00000200:16.0:1616513092.687360:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (32)-- 00000400:00000200:28.0:1616513092.687361:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 38744 00000800:00000200:16.0:1616513092.687362:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (31)-- 00000800:00000200:16.0:1616513092.687364:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (30)++ 00000100:00000200:24.0:1616513092.687365:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513092.687366:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513092.687367:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000400:00000200:16.0:1616513092.687374:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513092.687379:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8e80 00000800:00000200:24.0:1616513092.687380:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (31)++ 00000800:00000200:24.0:1616513092.687382:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (32)-- 00000400:00000200:20.0:1616513092.687382:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 38968 00000100:00000200:4.0:1616513092.687383:0:188401:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076096 00000800:00000200:24.0:1616513092.687384:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (31)-- 00000800:00000200:24.0:1616513092.687386:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (30)++ 00000100:00000200:0.0:1616513092.687386:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076000 00000100:00000200:28.0:1616513092.687387:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513092.687387:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513092.687388:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (31)-- 00000400:00000200:24.0:1616513092.687390:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:4.0:1616513092.687390:0:188401:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6726780 x1681365179076096/t0(0) o400->boslfs02-MDT0001-lwp-OST0019_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:0.0:1616513092.687393:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8d80 x1681365179076000/t0(0) o400->boslfs02-MDT0001-lwp-OST0031_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513092.687395:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:28.0:1616513092.687407:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (30)++ 00000400:00000200:16.0:1616513092.687408:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 39192 00000800:00000200:28.0:1616513092.687409:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (31)-- 00000800:00000200:28.0:1616513092.687411:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (30)-- 00000100:00000200:20.0:1616513092.687411:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513092.687411:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:28.0:1616513092.687413:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)++ 00000800:00000200:28.0:1616513092.687417:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513092.687418:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (30)++ 00000400:00000200:24.0:1616513092.687419:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faee58 00000800:00000200:20.0:1616513092.687421:0:6463:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (31)++ 00000800:00000200:24.0:1616513092.687422:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (31)-- 00000100:00000200:4.0:1616513092.687422:0:188401:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076096, offset 192 00000100:00000200:0.0:1616513092.687422:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076000, offset 192 00000100:00000200:16.0:1616513092.687423:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:28.0:1616513092.687424:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:20.0:1616513092.687424:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (32)-- 00000100:00000200:12.0:1616513092.687424:0:188393:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076192 00000800:00000200:24.0:1616513092.687425:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (31)++ 00000800:00000200:20.0:1616513092.687426:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (31)-- 00000800:00000200:24.0:1616513092.687428:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513092.687429:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (30)++ 00000800:00000200:16.0:1616513092.687429:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (30)++ 00010000:00000200:12.0:1616513092.687430:0:188393:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeede780 x1681365179076192/t0(0) o400->boslfs02-MDT0001-lwp-OST0009_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513092.687431:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8e40 00000800:00000200:16.0:1616513092.687431:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (32)-- 00000400:00000200:4.0:1616513092.687431:0:188401:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:20.0:1616513092.687432:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:16.0:1616513092.687433:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (31)-- 00000400:00000200:24.0:1616513092.687446:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000100:00000200:8.0:1616513092.687446:0:170316:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076032 00000400:00000200:4.0:1616513092.687450:0:188401:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:0.0:1616513092.687450:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00010000:00000200:8.0:1616513092.687451:0:170316:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0de300 x1681365179076032/t0(0) o400->boslfs02-MDT0001-lwp-OST0035_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513092.687454:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 39416 00000800:00000200:16.0:1616513092.687454:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (30)++ 00000800:00000200:16.0:1616513092.687456:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (31)-- 00000400:00000200:28.0:1616513092.687458:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.687458:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:12.0:1616513092.687458:0:188393:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076192, offset 192 00000100:00000200:28.0:1616513092.687460:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513092.687463:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:4.0:1616513092.687463:0:188401:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513092.687469:0:188401:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000400:00000200:12.0:1616513092.687470:0:188393:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:0.0:1616513092.687471:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000100:00000200:8.0:1616513092.687472:0:170316:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076032, offset 192 00000400:00000200:20.0:1616513092.687473:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513092.687473:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc4c8 00000400:00000200:8.0:1616513092.687478:0:170316:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:0.0:1616513092.687485:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513092.687489:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (30)++ 00000800:00000200:0.0:1616513092.687490:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513092.687491:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (31)-- 00000400:00000200:24.0:1616513092.687492:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8ee0 00000800:00000200:4.0:1616513092.687492:0:188401:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513092.687493:0:188393:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:28.0:1616513092.687494:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (30)-- 00000800:00000200:28.0:1616513092.687496:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)++ 00000400:00000200:24.0:1616513092.687496:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 39640 00000800:00000200:4.0:1616513092.687497:0:188401:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (29)++ 00000800:00000200:28.0:1616513092.687498:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (30)-- 00000800:00000200:4.0:1616513092.687499:0:188401:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (31)++ 00000400:00000200:28.0:1616513092.687500:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513092.687500:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.687502:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513092.687502:0:188401:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (31)-- 00000400:00000200:28.0:1616513092.687505:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:12.0:1616513092.687507:0:188393:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513092.687509:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:16.0:1616513092.687511:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (30)-- 00000800:00000200:12.0:1616513092.687512:0:188393:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:16.0:1616513092.687513:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)++ 00000800:00000200:16.0:1616513092.687515:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (30)-- 00000400:00000200:8.0:1616513092.687515:0:170316:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:16.0:1616513092.687517:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.687522:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.687525:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a330 00000400:00000200:8.0:1616513092.687527:0:170316:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513092.687529:0:188401:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076224 00000800:00000200:24.0:1616513092.687532:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (29)++ 00000800:00000200:8.0:1616513092.687532:0:170316:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513092.687533:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)-- 00000800:00000200:24.0:1616513092.687534:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (30)-- 00010000:00000200:4.0:1616513092.687534:0:188401:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6727500 x1681365179076224/t0(0) o400->boslfs02-MDT0001-lwp-OST003d_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513092.687534:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513092.687535:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)-- 00000800:00000200:12.0:1616513092.687536:0:188393:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.687536:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (28)++ 00000800:00000200:28.0:1616513092.687537:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (27)++ 00000400:00000200:20.0:1616513092.687537:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8ea0 00000800:00000200:8.0:1616513092.687538:0:170316:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.687538:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (28)++ 00000800:00000200:12.0:1616513092.687539:0:188393:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (29)++ 00000800:00000200:8.0:1616513092.687540:0:170316:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (30)++ 00000800:00000200:12.0:1616513092.687541:0:188393:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (32)++ 00000800:00000200:0.0:1616513092.687541:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (31)-- 00000800:00000200:24.0:1616513092.687542:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (32)-- 00000400:00000200:20.0:1616513092.687542:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 39864 00000800:00000200:24.0:1616513092.687544:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (31)++ 00000400:00000200:20.0:1616513092.687545:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513092.687545:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076160 00000800:00000200:12.0:1616513092.687545:0:188393:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.230@o2ib1: no credits 00000800:00000200:12.0:1616513092.687546:0:188393:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (32)-- 00000800:00000200:8.0:1616513092.687546:0:170316:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (32)++ 00000100:00000200:4.0:1616513092.687546:0:188401:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076224, offset 192 00000100:00000200:20.0:1616513092.687547:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513092.687550:0:170316:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.230@o2ib1: no credits 00000100:00000200:0.0:1616513092.687550:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076320 00000800:00000200:8.0:1616513092.687551:0:170316:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (32)-- 00000800:00000200:28.0:1616513092.687553:0:6462:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.230@o2ib1: no credits 00000800:00000200:28.0:1616513092.687555:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (31)-- 00000800:00000200:20.0:1616513092.687555:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (31)++ 00010000:00000200:16.0:1616513092.687555:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed9f80 x1681365179076160/t0(0) o400->boslfs02-MDT0001-lwp-OST0015_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:0.0:1616513092.687555:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cea880 x1681365179076320/t0(0) o400->boslfs02-MDT0001-lwp-OST0025_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:28.0:1616513092.687557:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513092.687557:0:6464:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.230@o2ib1: no credits 00000400:00000200:4.0:1616513092.687557:0:188401:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:24.0:1616513092.687559:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (31)-- 00000400:00000200:24.0:1616513092.687560:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513092.687561:0:6463:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.230@o2ib1: no credits 00000400:00000200:28.0:1616513092.687562:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:20.0:1616513092.687562:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (30)-- 00000400:00000200:4.0:1616513092.687562:0:188401:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:28.0:1616513092.687564:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f430 00000800:00000200:20.0:1616513092.687564:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)-- 00000100:00000200:8.0:1616513092.687564:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076256 00000400:00000200:24.0:1616513092.687565:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:20.0:1616513092.687566:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (28)++ 00000100:00000200:0.0:1616513092.687566:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076320, offset 192 00000400:00000200:24.0:1616513092.687568:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00010000:00000200:8.0:1616513092.687569:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d9b00 x1681365179076256/t0(0) o400->boslfs02-MDT0001-lwp-OST0001_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513092.687570:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.230@o2ib1 00000100:00000200:16.0:1616513092.687571:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076160, offset 192 00000800:00000200:20.0:1616513092.687574:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (29)++ 00000400:00000200:0.0:1616513092.687574:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:20.0:1616513092.687575:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (30)-- 00000400:00000200:4.0:1616513092.687575:0:188401:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513092.687577:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)-- 00000400:00000200:16.0:1616513092.687578:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:20.0:1616513092.687579:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (28)++ 00000100:00000200:8.0:1616513092.687579:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076256, offset 192 00000800:00000200:28.0:1616513092.687580:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)-- 00000800:00000200:20.0:1616513092.687581:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (29)-- 00000800:00000200:4.0:1616513092.687581:0:188401:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:24.0:1616513092.687582:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (28)-- 00000800:00000200:28.0:1616513092.687584:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (26)++ 00000400:00000200:8.0:1616513092.687584:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:0.0:1616513092.687584:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:20.0:1616513092.687589:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (27)-- 00000800:00000200:28.0:1616513092.687591:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513092.687591:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (26)++ 00000800:00000200:20.0:1616513092.687595:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:12.0:1616513092.687595:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (27)++ 00000400:00000200:0.0:1616513092.687597:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513092.687600:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:0.0:1616513092.687602:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000400:00000200:16.0:1616513092.687604:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:16.0:1616513092.687618:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513092.687624:0:188401:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513092.687625:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000400:00000200:8.0:1616513092.687625:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:4.0:1616513092.687627:0:188401:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (28)++ 00000800:00000200:4.0:1616513092.687628:0:188401:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (29)++ 00000800:00000200:4.0:1616513092.687631:0:188401:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (30)-- 00000400:00000200:8.0:1616513092.687637:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513092.687642:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:0.0:1616513092.687643:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513092.687644:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513092.687645:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513092.687645:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:0.0:1616513092.687645:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (29)++ 00000400:00000200:12.0:1616513092.687646:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:8.0:1616513092.687646:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513092.687647:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (30)++ 00000800:00000200:0.0:1616513092.687647:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (30)++ 00000800:00000200:8.0:1616513092.687648:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (30)++ 00000800:00000200:16.0:1616513092.687650:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (33)++ 00000800:00000200:0.0:1616513092.687650:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (33)-- 00000800:00000200:16.0:1616513092.687653:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (33)-- 00000400:00000200:12.0:1616513092.687653:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8f00 00000800:00000200:8.0:1616513092.687654:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (33)++ 00000400:00000200:20.0:1616513092.687655:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8f20 00000800:00000200:8.0:1616513092.687657:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (33)-- 00000400:00000200:28.0:1616513092.687658:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 224 into portal 12 MB=0x5f931563c8ec0 00000400:00000200:12.0:1616513092.687659:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 40088 00000800:00000200:16.0:1616513092.687666:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (32)++ 00000400:00000200:12.0:1616513092.687666:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513092.687668:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 40312 00000800:00000200:16.0:1616513092.687668:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (33)-- 00000400:00000200:16.0:1616513092.687670:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513092.687671:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.687673:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.230@o2ib1 of length 224/224 into md 0x865be291 [1] + 40536 00000400:00000200:16.0:1616513092.687675:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.687677:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513092.687677:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:20.0:1616513092.687683:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513092.687684:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (32)++ 00000800:00000200:12.0:1616513092.687686:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (33)-- 00000800:00000200:12.0:1616513092.687688:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (32)-- 00000800:00000200:20.0:1616513092.687689:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (31)++ 00000400:00000200:16.0:1616513092.687689:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbdc0 00000800:00000200:12.0:1616513092.687690:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (31)++ 00000100:00000200:28.0:1616513092.687691:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513092.687692:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (33)-- 00000800:00000200:16.0:1616513092.687692:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (33)-- 00000800:00000200:12.0:1616513092.687692:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (33)-- 00000800:00000200:20.0:1616513092.687693:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (31)-- 00000400:00000200:12.0:1616513092.687694:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513092.687695:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)++ 00000800:00000200:28.0:1616513092.687696:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (30)++ 00000800:00000200:28.0:1616513092.687699:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (31)-- 00000800:00000200:16.0:1616513092.687699:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (31)-- 00000800:00000200:28.0:1616513092.687700:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (30)-- 00000400:00000200:12.0:1616513092.687700:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.687701:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513092.687702:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (28)++ 00000100:00000200:0.0:1616513092.687702:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076352 00000400:00000200:12.0:1616513092.687703:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafc28 00000100:00000200:8.0:1616513092.687703:0:188419:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076384 00000800:00000200:28.0:1616513092.687705:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (29)-- 00000400:00000200:16.0:1616513092.687705:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.687706:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513092.687706:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)++ 00000800:00000200:12.0:1616513092.687707:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)-- 00010000:00000200:0.0:1616513092.687707:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d202d00 x1681365179076352/t0(0) o400->boslfs02-MDT0001-lwp-OST0029_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513092.687708:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f298 00010000:00000200:8.0:1616513092.687708:0:188419:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0df980 x1681365179076384/t0(0) o400->boslfs02-MDT0001-lwp-OST002d_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513092.687709:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.230@o2ib1 00000800:00000200:12.0:1616513092.687709:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (28)++ 00000400:00000200:28.0:1616513092.687711:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:20.0:1616513092.687711:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (29)++ 00000800:00000200:16.0:1616513092.687711:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)-- 00000800:00000200:12.0:1616513092.687712:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (29)-- 00000400:00000200:28.0:1616513092.687713:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2cc0 00000800:00000200:20.0:1616513092.687713:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (29)-- 00000800:00000200:16.0:1616513092.687713:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (29)++ 00000400:00000200:12.0:1616513092.687713:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513092.687714:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (28)-- 00000800:00000200:16.0:1616513092.687715:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (28)-- 00000800:00000200:28.0:1616513092.687716:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (26)-- 00000400:00000200:16.0:1616513092.687716:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513092.687717:0:188419:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076384, offset 192 00000100:00000200:24.0:1616513092.687718:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681365179076288 00000800:00000200:20.0:1616513092.687718:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (25)++ 00000400:00000200:12.0:1616513092.687718:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.687721:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:8.0:1616513092.687721:0:188419:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:20.0:1616513092.687722:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (26)-- 00000400:00000200:12.0:1616513092.687723:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb3a8 00000400:00000200:20.0:1616513092.687724:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.687725:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faee58 00000800:00000200:28.0:1616513092.687726:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (25)++ 00000400:00000200:8.0:1616513092.687726:0:188419:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00010000:00000200:24.0:1616513092.687727:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeeda880 x1681365179076288/t0(0) o400->boslfs02-MDT0001-lwp-OST001d_UUID@10.242.46.230@o2ib1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513092.687727:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076352, offset 192 00000800:00000200:28.0:1616513092.687728:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (26)-- 00000400:00000200:20.0:1616513092.687729:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.687730:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513092.687732:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:20.0:1616513092.687734:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788ee0 00000400:00000200:28.0:1616513092.687735:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.687738:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000400:00000200:8.0:1616513092.687738:0:188419:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513092.687740:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681365179076288, offset 192 00000800:00000200:12.0:1616513092.687742:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (25)-- 00000800:00000200:16.0:1616513092.687743:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (24)-- 00000800:00000200:8.0:1616513092.687743:0:188419:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:12.0:1616513092.687745:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)++ 00000400:00000200:24.0:1616513092.687746:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:0.0:1616513092.687746:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:12.0:1616513092.687750:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.230@o2ib1 00000800:00000200:12.0:1616513092.687753:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (24)++ 00000800:00000200:16.0:1616513092.687754:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (25)-- 00000800:00000200:12.0:1616513092.687754:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (25)-- 00000800:00000200:12.0:1616513092.687756:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:0.0:1616513092.687759:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513092.687763:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513092.687764:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513092.687765:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:8.0:1616513092.687766:0:188419:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513092.687768:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:8.0:1616513092.687768:0:188419:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513092.687770:0:188419:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513092.687772:0:188419:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513092.687778:0:50816:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:24.0:1616513092.687782:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513092.687788:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513092.687789:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:0.0:1616513092.687789:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513092.687791:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:0.0:1616513092.687791:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (23)++ 00000400:00000200:16.0:1616513092.687793:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513092.687793:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (24)++ 00000800:00000200:24.0:1616513092.687794:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.687795:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (24)-- 00000800:00000200:12.0:1616513092.687796:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (24)-- 00000800:00000200:24.0:1616513092.687797:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (23)++ 00000400:00000200:16.0:1616513092.687798:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:24.0:1616513092.687799:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (23)++ 00000400:00000200:16.0:1616513092.687800:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3f58 00000800:00000200:5.2:1616513092.687800:0:50816:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:24.0:1616513092.687803:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (25)-- 00000800:00000200:16.0:1616513092.687804:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (25)-- 00000800:00000200:16.0:1616513092.687806:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:16.0:1616513092.687809:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (24)-- 00000800:00000200:20.0:1616513092.687810:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (24)++ 00000800:00000200:20.0:1616513092.687814:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (25)-- 00000800:00000200:28.0:1616513092.687815:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (25)-- 00000400:00000200:20.0:1616513092.687816:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.687816:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513092.687821:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.687821:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.687823:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafa08 00000400:00000200:20.0:1616513092.687825:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3210 00000800:00000200:16.0:1616513092.687825:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513092.687827:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:11.2:1616513092.706733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.706790:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.706800:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.706809:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.87@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.706818:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.87@tcp1 of length 224 into portal 26 MB=0x5ffd2f8eb0900 00000400:00000200:28.0:1616513092.706825:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.87@tcp1 of length 224/224 into md 0xf97ce619 [1] + 0 00000400:00000200:28.0:1616513092.706829:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05ed0 00000400:00000200:28.0:1616513092.706832:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.706835:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513092.706839:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05ed0 00000800:00000200:28.0:1616513092.706842:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.706843:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.706845:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.706848:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.706886:0:50818:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513092.706904:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656467921152 00010000:00000200:0.0:1616513092.706915:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688656467921152/t0(0) o400->03fc6aa2-cd09-57d6-d88c-bbe9d1063b60@10.31.129.87@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.706930:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688656467921152, offset 192 00000400:00000200:0.0:1616513092.706936:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.87@tcp1 00000800:00000200:14.0:1616513092.706942:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513092.706944:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.87@tcp1 routed destination 00000800:00000200:14.0:1616513092.706952:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513092.706959:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.87@tcp1(10.31.129.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513092.706962:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.706965:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513092.706968:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.53@tcp1 : PUT - for me 00000800:00000200:0.0:1616513092.706971:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.706974:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513092.706976:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513092.706978:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.53@tcp1 of length 224 into portal 26 MB=0x6012fd23c5640 00000800:00000200:0.0:1616513092.706981:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513092.706987:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.53@tcp1 of length 224/224 into md 0xf97ce629 [1] + 0 00000100:00000200:0.0:1616513092.706988:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513092.706992:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04a18 00000400:00000200:14.0:1616513092.706994:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513092.706997:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000100:00000200:14.0:1616513092.706997:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513092.707003:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04a18 00000800:00000200:22.0:1616513092.707005:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513092.707006:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000400:00000200:22.0:1616513092.707008:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513092.707009:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513092.707011:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:0.0:1616513092.707013:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690154762524224 00000400:00000200:22.0:1616513092.707014:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:14.0:1616513092.707014:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:0.0:1616513092.707019:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1690154762524224/t0(0) o400->eef900af-ea06-6dbd-43e7-6cdde440b08a@10.31.140.53@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513092.707020:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faeaa0 00000800:00000200:16.0:1616513092.707020:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513092.707023:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513092.707029:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513092.707031:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690154762524224, offset 192 00000400:00000200:0.0:1616513092.707036:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.53@tcp1 00000400:00000200:16.0:1616513092.707038:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.87@tcp1 : PUT - for me 00000400:00000200:0.0:1616513092.707042:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.53@tcp1 routed destination 00000400:00000200:0.0:1616513092.707055:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.53@tcp1(10.31.140.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.707063:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513092.707066:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.87@tcp1 of length 224 into portal 12 MB=0x5ffd2f8eb0c80 00000800:00000200:0.0:1616513092.707068:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.707070:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513092.707072:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.87@tcp1 of length 224/224 into md 0x865be291 [1] + 40760 00000800:00000200:0.0:1616513092.707072:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513092.707075:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513092.707077:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513092.707078:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513092.707079:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:16.0:1616513092.707080:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:11.2:1616513092.707083:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.707086:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.707090:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513092.707092:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513092.707094:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.707099:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513092.707099:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513092.707101:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.707106:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.707111:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae3b8 00000800:00000200:16.0:1616513092.707113:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513092.707150:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656467922048 00010000:00000200:24.0:1616513092.707163:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6726300 x1688656467922048/t0(0) o400->ac9c8eeb-09a8-eae4-25c9-231a8eaa77a0@10.31.129.87@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513092.707177:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688656467922048, offset 192 00000400:00000200:24.0:1616513092.707183:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.87@tcp1 00000400:00000200:24.0:1616513092.707190:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.87@tcp1 routed destination 00000400:00000200:24.0:1616513092.707205:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.87@tcp1(10.31.129.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513092.707212:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513092.707217:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513092.707220:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513092.707223:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513092.707238:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.707242:0:50818:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.707297:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.707304:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513092.707307:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513092.707308:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513092.707313:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513092.707317:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2198 00000800:00000200:18.0:1616513092.707321:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513092.707506:0:50818:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513092.707517:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.707523:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513092.707527:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513092.707531:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.53@tcp1 : PUT - for me 00000400:00000200:22.0:1616513092.707539:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.53@tcp1 of length 224 into portal 12 MB=0x6012fd23c5e00 00000400:00000200:22.0:1616513092.707545:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.53@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 3808 00000400:00000200:22.0:1616513092.707551:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513092.707553:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513092.707560:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.707562:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.707564:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.707577:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690154762526208 00010000:00000200:30.0:1616513092.707586:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a29f80 x1690154762526208/t0(0) o400->0990d422-a514-a1bd-eaf5-6759382e949e@10.31.140.53@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.707599:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690154762526208, offset 192 00000400:00000200:30.0:1616513092.707604:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.53@tcp1 00000400:00000200:30.0:1616513092.707611:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.53@tcp1 routed destination 00000400:00000200:30.0:1616513092.707624:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.53@tcp1(10.31.140.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.707630:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513092.707636:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.707638:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513092.707640:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513092.707644:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.707649:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.707660:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.707665:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513092.707668:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513092.707669:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513092.707674:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513092.707677:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeff0 00000800:00000200:20.0:1616513092.707681:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.724153:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513092.724209:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.724218:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513092.724227:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.176@tcp1 : PUT - for me 00000400:00000200:14.0:1616513092.724236:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.176@tcp1 of length 224 into portal 26 MB=0x5ffde5b493100 00000400:00000200:14.0:1616513092.724243:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.176@tcp1 of length 224/224 into md 0xf97ce639 [1] + 0 00000400:00000200:14.0:1616513092.724247:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04110 00000400:00000200:14.0:1616513092.724250:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513092.724253:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513092.724257:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04110 00000800:00000200:14.0:1616513092.724260:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.724262:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.724263:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513092.724269:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.724272:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513092.724272:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513092.724281:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.176@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.724290:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.176@tcp1 of length 224 into portal 12 MB=0x5ffde5b493180 00000400:00000200:18.0:1616513092.724296:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.176@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 4032 00000400:00000200:18.0:1616513092.724301:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.724304:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513092.724311:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.724314:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.724316:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.724320:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705362899200 00000100:00000200:30.0:1616513092.724323:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705362899328 00010000:00000200:30.0:1616513092.724330:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd706c00 x1688705362899328/t0(0) o400->78ecd552-f4af-643d-2e33-0f0083ba0ebe@10.31.132.176@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:0.0:1616513092.724331:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688705362899200/t0(0) o400->49d05c31-813a-45fa-8589-b74e99a8afc3@10.31.132.176@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513092.724340:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705362899328, offset 192 00000400:00000200:30.0:1616513092.724345:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.176@tcp1 00000100:00000200:0.0:1616513092.724346:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705362899200, offset 192 00000400:00000200:30.0:1616513092.724351:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.176@tcp1 routed destination 00000400:00000200:0.0:1616513092.724352:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.176@tcp1 00000400:00000200:0.0:1616513092.724360:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.176@tcp1 routed destination 00000400:00000200:30.0:1616513092.724363:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.176@tcp1(10.31.132.176@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.724368:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513092.724373:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.724375:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513092.724375:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.176@tcp1(10.31.132.176@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.724376:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513092.724380:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513092.724382:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513092.724385:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.724387:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.724390:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.724392:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.724397:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513092.724398:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:0.0:1616513092.724404:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513092.724405:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513092.724407:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513092.724409:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513092.724414:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513092.724418:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef430 00000800:00000200:22.0:1616513092.724422:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513092.724443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.724465:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.724471:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513092.724474:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.724481:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.724486:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafdc0 00000800:00000200:28.0:1616513092.724490:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.724493:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.745716:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.745772:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.745791:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513092.745800:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.44@tcp1 : PUT - for me 00000400:00000200:12.0:1616513092.745808:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.44@tcp1 of length 224 into portal 26 MB=0x5ffd3d5aebd40 00000400:00000200:12.0:1616513092.745815:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.44@tcp1 of length 224/224 into md 0xf97ce669 [1] + 0 00000400:00000200:12.0:1616513092.745819:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb044c8 00000400:00000200:12.0:1616513092.745821:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513092.745824:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513092.745830:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb044c8 00000800:00000200:20.0:1616513092.745831:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.745833:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.745836:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.745837:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.745892:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660171734336 00010000:00000200:0.0:1616513092.745902:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688660171734336/t0(0) o400->b1822800-247d-71c9-62fe-2e8d1e113a57@10.31.129.44@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.745915:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660171734336, offset 192 00000400:00000200:0.0:1616513092.745922:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.44@tcp1 00000400:00000200:0.0:1616513092.745929:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.44@tcp1 routed destination 00000400:00000200:0.0:1616513092.745943:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.44@tcp1(10.31.129.44@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.745949:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.745954:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.745957:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.745960:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.745965:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.745972:0:50822:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.745972:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513092.746020:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513092.746027:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.746035:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513092.746038:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513092.746044:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513092.746050:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faeaa0 00000800:00000200:18.0:1616513092.746054:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.746057:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513092.746074:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.746083:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.746091:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.44@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.746100:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.44@tcp1 of length 224 into portal 12 MB=0x5ffd3d5aec140 00000400:00000200:28.0:1616513092.746107:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.44@tcp1 of length 224/224 into md 0x865be291 [1] + 40984 00000400:00000200:28.0:1616513092.746112:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.746114:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513092.746121:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.746124:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.746126:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.746129:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513092.746185:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660171735360 00010000:00000200:24.0:1616513092.746198:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedc380 x1688660171735360/t0(0) o400->e1c3b438-2dff-db3a-932f-b06437b0474c@10.31.129.44@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513092.746212:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660171735360, offset 192 00000400:00000200:24.0:1616513092.746219:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.44@tcp1 00000400:00000200:24.0:1616513092.746226:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.44@tcp1 routed destination 00000400:00000200:24.0:1616513092.746241:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.44@tcp1(10.31.129.44@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513092.746248:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513092.746253:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513092.746256:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513092.746258:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.746263:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.746269:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.746279:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.746284:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513092.746286:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513092.746288:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513092.746292:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513092.746296:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3dc0 00000800:00000200:20.0:1616513092.746300:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.752673:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513092.752742:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.752751:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513092.752760:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.2@tcp1 : PUT - for me 00000400:00000200:14.0:1616513092.752770:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.2@tcp1 of length 224 into portal 26 MB=0x5ffed68e1dd00 00000400:00000200:14.0:1616513092.752776:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.2@tcp1 of length 224/224 into md 0xf97ce679 [1] + 0 00000400:00000200:14.0:1616513092.752781:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb055c8 00000400:00000200:14.0:1616513092.752783:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513092.752786:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513092.752792:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb055c8 00000800:00000200:14.0:1616513092.752795:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.752796:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.752798:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513092.752800:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513092.752849:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513092.752856:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688770015517952 00010000:00000200:0.0:1616513092.752866:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688770015517952/t0(0) o400->58bdc1b8-d67b-89a4-59f7-549d5ecd2ade@10.31.142.2@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.752881:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688770015517952, offset 192 00000400:00000200:0.0:1616513092.752887:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.2@tcp1 00000400:00000200:0.0:1616513092.752895:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.2@tcp1 routed destination 00000400:00000200:0.0:1616513092.752910:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.2@tcp1(10.31.142.2@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.752917:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513092.752922:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.752923:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.752925:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.752928:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.752932:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513092.752934:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513092.752937:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.752939:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513092.752943:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.2@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.752952:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.2@tcp1 of length 224 into portal 12 MB=0x5ffed68e1dd80 00000400:00000200:28.0:1616513092.752958:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.2@tcp1 of length 224/224 into md 0x865be291 [1] + 41208 00000400:00000200:28.0:1616513092.752963:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.752966:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513092.752972:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.752975:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.752977:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.752980:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513092.752998:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.753005:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513092.753008:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513092.753015:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513092.753020:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faee58 00000800:00000200:18.0:1616513092.753024:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.753027:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513092.753038:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688770015518080 00010000:00000200:24.0:1616513092.753050:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeed8900 x1688770015518080/t0(0) o400->662df7d7-00fe-bec3-bc58-45af421c1b1f@10.31.142.2@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513092.753064:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688770015518080, offset 192 00000400:00000200:24.0:1616513092.753070:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.2@tcp1 00000400:00000200:24.0:1616513092.753078:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.2@tcp1 routed destination 00000400:00000200:24.0:1616513092.753093:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.2@tcp1(10.31.142.2@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513092.753099:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513092.753105:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513092.753108:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513092.753110:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.753115:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.753120:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.753172:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.753179:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513092.753182:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513092.753188:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513092.753192:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3cb0 00000800:00000200:12.0:1616513092.753196:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.753199:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.760896:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513092.760952:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.760961:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513092.760970:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.250@tcp1 : PUT - for me 00000400:00000200:22.0:1616513092.760980:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.250@tcp1 of length 224 into portal 26 MB=0x604192fcd17c0 00000400:00000200:22.0:1616513092.760986:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.250@tcp1 of length 224/224 into md 0xf97ce689 [1] + 0 00000400:00000200:22.0:1616513092.760991:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05a90 00000400:00000200:22.0:1616513092.760994:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513092.760996:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513092.761000:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05a90 00000800:00000200:22.0:1616513092.761003:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.761006:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513092.761008:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.761010:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.761076:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356082927552 00010000:00000200:0.0:1616513092.761087:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1693356082927552/t0(0) o400->dad5071b-ae80-44a7-db0f-e6ce28ea37fe@10.31.128.250@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.761102:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693356082927552, offset 192 00000400:00000200:0.0:1616513092.761108:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.250@tcp1 00000400:00000200:0.0:1616513092.761116:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.250@tcp1 routed destination 00000400:00000200:0.0:1616513092.761131:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.250@tcp1(10.31.128.250@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.761138:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.761144:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.761146:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.761148:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.761153:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.761158:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.761160:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513092.761210:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.761217:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513092.761219:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513092.761226:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513092.761231:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faee58 00000800:00000200:18.0:1616513092.761234:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.761237:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.761243:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513092.761246:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513092.761249:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.250@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.761264:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.250@tcp1 of length 224 into portal 12 MB=0x604192fcd1b40 00000400:00000200:18.0:1616513092.761269:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.250@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 4256 00000400:00000200:18.0:1616513092.761274:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.761276:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513092.761282:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.761284:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.761286:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.761301:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356082928448 00010000:00000200:30.0:1616513092.761312:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd701f80 x1693356082928448/t0(0) o400->42701952-0a48-0a6a-c82d-e459b2d7a9bb@10.31.128.250@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.761326:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693356082928448, offset 192 00000400:00000200:30.0:1616513092.761333:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.250@tcp1 00000400:00000200:30.0:1616513092.761340:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.250@tcp1 routed destination 00000400:00000200:30.0:1616513092.761355:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.250@tcp1(10.31.128.250@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.761361:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513092.761367:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.761369:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513092.761372:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513092.761376:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.761423:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.761482:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.761489:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513092.761492:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.761498:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.761502:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee000 00000800:00000200:28.0:1616513092.761506:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.761509:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.763141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.763198:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.763207:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513092.763216:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.87@tcp1 : PUT - for me 00000400:00000200:12.0:1616513092.763225:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.87@tcp1 of length 224 into portal 26 MB=0x5ffde9aca8400 00000400:00000200:12.0:1616513092.763232:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.87@tcp1 of length 224/224 into md 0xf97ce699 [1] + 0 00000400:00000200:12.0:1616513092.763237:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05430 00000400:00000200:12.0:1616513092.763239:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513092.763242:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513092.763246:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05430 00000800:00000200:12.0:1616513092.763249:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.763251:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.763252:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513092.763259:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513092.763262:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513092.763263:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513092.763271:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.87@tcp1 : PUT - for me 00000400:00000200:20.0:1616513092.763280:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.87@tcp1 of length 224 into portal 12 MB=0x5ffde9aca8480 00000400:00000200:20.0:1616513092.763287:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.87@tcp1 of length 224/224 into md 0x865be291 [1] + 41432 00000400:00000200:20.0:1616513092.763292:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.763294:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513092.763301:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.763304:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513092.763306:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.763307:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706428339200 00010000:00000200:0.0:1616513092.763318:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688706428339200/t0(0) o400->4f7170a7-43eb-b676-dfee-28ece36f9ce6@10.31.142.87@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.763332:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706428339200, offset 192 00000800:00000200:11.2:1616513092.763334:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:0.0:1616513092.763339:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.87@tcp1 00000400:00000200:0.0:1616513092.763346:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.87@tcp1 routed destination 00000400:00000200:0.0:1616513092.763360:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.87@tcp1(10.31.142.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513092.763366:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706428339328 00000800:00000200:0.0:1616513092.763366:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.763371:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.763374:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.763376:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:24.0:1616513092.763378:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0de780 x1688706428339328/t0(0) o400->60eead20-1e78-084d-62b9-32e8477e9d04@10.31.142.87@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513092.763380:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.763386:0:50824:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.763387:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513092.763391:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513092.763393:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706428339328, offset 192 00000400:00000200:24.0:1616513092.763399:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.87@tcp1 00000800:00000200:16.0:1616513092.763400:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513092.763402:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513092.763406:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.87@tcp1 routed destination 00000800:00000200:12.0:1616513092.763407:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513092.763410:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513092.763422:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.87@tcp1(10.31.142.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513092.763429:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513092.763431:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.159@tcp1 : PUT - for me 00000400:00000200:12.0:1616513092.763431:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.159@tcp1 : PUT - for me 00000800:00000200:24.0:1616513092.763435:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513092.763439:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513092.763452:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.159@tcp1 of length 224 into portal 26 MB=0x5ffd56a55d780 00000400:00000200:16.0:1616513092.763453:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.159@tcp1 of length 224 into portal 12 MB=0x5ffd56a55d7c0 00000800:00000200:24.0:1616513092.763454:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:12.0:1616513092.763458:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.159@tcp1 of length 224/224 into md 0xf97ce6b9 [1] + 0 00000800:00000200:24.0:1616513092.763459:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513092.763460:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.159@tcp1 of length 224/224 into md 0x865be291 [1] + 41656 00000400:00000200:12.0:1616513092.763461:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb043b8 00000400:00000200:12.0:1616513092.763463:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513092.763465:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:11.2:1616513092.763465:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:12.0:1616513092.763466:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513092.763467:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:16.0:1616513092.763468:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513092.763472:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb043b8 00000800:00000200:14.0:1616513092.763473:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513092.763475:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000400:00000200:14.0:1616513092.763476:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513092.763476:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:28.0:1616513092.763477:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:12.0:1616513092.763478:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (26)-- 00000800:00000200:16.0:1616513092.763479:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (26)-- 00000400:00000200:14.0:1616513092.763483:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513092.763483:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:28.0:1616513092.763484:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:16.0:1616513092.763484:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513092.763486:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:24.0:1616513092.763486:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666960680896 00000800:00000200:12.0:1616513092.763487:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513092.763488:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafdc0 00000100:00000200:0.0:1616513092.763490:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666960680832 00010000:00000200:24.0:1616513092.763492:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6724380 x1688666960680896/t0(0) o400->4354d90e-4785-53fa-2f5d-64a5c745e2cd@10.31.132.159@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513092.763492:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513092.763493:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513092.763495:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513092.763495:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688666960680832/t0(0) o400->79473c95-0d6d-e5fb-ab1c-3f615da32605@10.31.132.159@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513092.763497:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f36d8 00000800:00000200:28.0:1616513092.763501:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513092.763504:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666960680896, offset 192 00000100:00000200:0.0:1616513092.763506:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666960680832, offset 192 00000400:00000200:24.0:1616513092.763510:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.159@tcp1 00000400:00000200:0.0:1616513092.763514:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.159@tcp1 00000400:00000200:24.0:1616513092.763515:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.159@tcp1 routed destination 00000400:00000200:0.0:1616513092.763525:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.159@tcp1 routed destination 00000400:00000200:24.0:1616513092.763531:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.159@tcp1(10.31.132.159@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513092.763537:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.159@tcp1(10.31.132.159@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.763543:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513092.763547:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.763548:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.763550:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513092.763552:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.763552:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.763554:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.763555:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513092.763556:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513092.763559:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513092.763559:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513092.763560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513092.763564:0:50824:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513092.763570:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.763574:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513092.763578:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513092.763578:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513092.763584:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.763588:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafdc0 00000800:00000200:16.0:1616513092.763592:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513092.763618:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513092.763625:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513092.763628:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513092.763634:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.763638:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2d48 00000800:00000200:2.0:1616513092.763642:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.763645:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513092.772097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513092.772153:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.772163:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513092.772172:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.239@tcp1 : PUT - for me 00000400:00000200:20.0:1616513092.772181:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.239@tcp1 of length 224 into portal 26 MB=0x5ffd3d5cf1480 00000400:00000200:20.0:1616513092.772187:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.239@tcp1 of length 224/224 into md 0xf97ce6c9 [1] + 0 00000400:00000200:20.0:1616513092.772192:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04f68 00000400:00000200:20.0:1616513092.772195:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.772197:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513092.772202:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04f68 00000800:00000200:20.0:1616513092.772205:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.772207:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513092.772207:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513092.772211:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.772268:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513092.772273:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660173853824 00010000:00000200:0.0:1616513092.772284:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688660173853824/t0(0) o400->3be1f475-9940-eb96-be3f-59b1a425cbb8@10.31.128.239@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.772297:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660173853824, offset 192 00000400:00000200:0.0:1616513092.772303:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.239@tcp1 00000400:00000200:0.0:1616513092.772310:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.239@tcp1 routed destination 00000400:00000200:0.0:1616513092.772326:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.239@tcp1(10.31.128.239@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.772332:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.772337:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513092.772339:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.772339:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.772341:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.772346:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513092.772349:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513092.772352:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513092.772353:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513092.772357:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.239@tcp1 : PUT - for me 00000400:00000200:12.0:1616513092.772366:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.239@tcp1 of length 224 into portal 12 MB=0x5ffd3d5cf17c0 00000400:00000200:12.0:1616513092.772372:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.239@tcp1 of length 224/224 into md 0x865be291 [1] + 41880 00000400:00000200:12.0:1616513092.772377:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513092.772380:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513092.772386:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.772389:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513092.772391:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.772394:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513092.772413:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.772420:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513092.772423:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513092.772429:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513092.772447:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf980 00000800:00000200:14.0:1616513092.772451:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.772454:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.772460:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000100:00000200:24.0:1616513092.772461:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660173854656 00000800:00000200:14.0:1616513092.772462:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.772464:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513092.772466:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.772468:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513092.772472:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d203f00 x1688660173854656/t0(0) o400->1345f987-ae63-724c-186f-409496f0a090@10.31.128.239@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513092.772485:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660173854656, offset 192 00000400:00000200:24.0:1616513092.772491:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.239@tcp1 00000400:00000200:24.0:1616513092.772498:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.239@tcp1 routed destination 00000400:00000200:24.0:1616513092.772511:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.239@tcp1(10.31.128.239@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513092.772517:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513092.772523:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513092.772525:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513092.772527:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.772531:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.772537:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.772548:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.772553:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513092.772555:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513092.772557:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513092.772562:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.772566:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2110 00000800:00000200:28.0:1616513092.772570:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.776888:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.776945:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.776954:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513092.776963:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.46@tcp1 : PUT - for me 00000400:00000200:12.0:1616513092.776972:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.46@tcp1 of length 224 into portal 26 MB=0x5ffd505902040 00000400:00000200:12.0:1616513092.776979:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.46@tcp1 of length 224/224 into md 0xf97ce6d9 [1] + 0 00000400:00000200:12.0:1616513092.776984:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04990 00000400:00000200:12.0:1616513092.776986:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513092.776989:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513092.776993:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04990 00000800:00000200:16.0:1616513092.776997:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.776997:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.776999:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513092.777001:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.777053:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665270001728 00010000:00000200:0.0:1616513092.777063:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688665270001728/t0(0) o400->703fb7c7-f4f3-5c39-ef32-70426e314b83@10.31.136.46@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.777075:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665270001728, offset 192 00000400:00000200:0.0:1616513092.777081:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.46@tcp1 00000800:00000200:11.2:1616513092.777088:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:0.0:1616513092.777088:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.46@tcp1 routed destination 00000400:00000200:0.0:1616513092.777106:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.46@tcp1(10.31.136.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.777111:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.777116:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.777119:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.777120:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.777125:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513092.777130:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513092.777132:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513092.777157:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.777166:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513092.777175:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.46@tcp1 : PUT - for me 00000400:00000200:20.0:1616513092.777184:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.46@tcp1 of length 224 into portal 12 MB=0x5ffd505902200 00000800:00000200:18.0:1616513092.777186:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513092.777190:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.46@tcp1 of length 224/224 into md 0x865be291 [1] + 42104 00000800:00000200:18.0:1616513092.777192:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513092.777195:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513092.777195:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.777198:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513092.777201:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513092.777204:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.777207:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513092.777207:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfa90 00000800:00000200:20.0:1616513092.777211:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513092.777212:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513092.777214:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513092.777214:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513092.777222:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513092.777232:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.777239:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513092.777240:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.777246:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513092.777247:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.216@tcp1 : PUT - for me 00000800:00000200:28.0:1616513092.777250:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513092.777252:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.216@tcp1 : PUT - for me 00000400:00000200:16.0:1616513092.777256:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.216@tcp1 of length 224 into portal 26 MB=0x5ffd4851024c0 00000400:00000200:12.0:1616513092.777258:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.216@tcp1 of length 224 into portal 12 MB=0x5ffd485102600 00000400:00000200:16.0:1616513092.777262:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.216@tcp1 of length 224/224 into md 0xf97ce6f9 [1] + 0 00000400:00000200:12.0:1616513092.777263:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.216@tcp1 of length 224/224 into md 0x865be291 [1] + 42328 00000400:00000200:16.0:1616513092.777267:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04220 00000400:00000200:16.0:1616513092.777269:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:12.0:1616513092.777269:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513092.777271:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:24.0:1616513092.777272:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665270002176 00000100:00000200:16.0:1616513092.777272:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513092.777276:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04220 00000800:00000200:12.0:1616513092.777277:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.777279:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513092.777279:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513092.777281:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:24.0:1616513092.777283:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0d9680 x1688665270002176/t0(0) o400->a2e56b25-81fd-53c3-b950-e1f288a755ad@10.31.136.46@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513092.777283:0:6461:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.777287:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513092.777288:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663114130624 00000800:00000200:16.0:1616513092.777289:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.777291:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513092.777291:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663114130944 00010000:00000200:0.0:1616513092.777293:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688663114130624/t0(0) o400->46d4d318-8a34-8c2d-2b45-791befdd19c0@10.31.132.216@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513092.777297:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665270002176, offset 192 00010000:00000200:8.0:1616513092.777300:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d200900 x1688663114130944/t0(0) o400->6e51a3c1-ac9d-db12-870b-99d3bfeac434@10.31.132.216@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513092.777301:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513092.777302:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.46@tcp1 00000100:00000200:0.0:1616513092.777304:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663114130624, offset 192 00000800:00000200:20.0:1616513092.777305:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513092.777307:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513092.777309:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.46@tcp1 routed destination 00000800:00000200:20.0:1616513092.777309:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513092.777309:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.216@tcp1 00000100:00000200:8.0:1616513092.777312:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663114130944, offset 192 00000400:00000200:8.0:1616513092.777317:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.216@tcp1 00000400:00000200:24.0:1616513092.777323:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.46@tcp1(10.31.136.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513092.777329:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513092.777330:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.216@tcp1 routed destination 00000400:00000200:8.0:1616513092.777337:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.216@tcp1 routed destination 00000400:00000200:0.0:1616513092.777342:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.216@tcp1(10.31.132.216@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.777347:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513092.777349:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.216@tcp1(10.31.132.216@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513092.777355:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.777355:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513092.777356:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513092.777358:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513092.777359:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.777359:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.777360:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513092.777361:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513092.777362:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.777363:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513092.777366:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513092.777366:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:0.0:1616513092.777367:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513092.777368:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513092.777370:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:11.2:1616513092.777371:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513092.777380:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513092.777383:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:12.0:1616513092.777385:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513092.777387:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513092.777390:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:12.0:1616513092.777390:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.777392:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513092.777393:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2220 00000800:00000200:28.0:1616513092.777394:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513092.777397:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513092.777398:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513092.777401:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:16.0:1616513092.777403:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513092.777420:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.777427:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513092.777430:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513092.777450:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513092.777455:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be908 00000800:00000200:22.0:1616513092.777458:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.777461:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513092.791645:0:50827:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513092.791717:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.791726:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513092.791735:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.26@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.791744:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.26@tcp1 of length 224 into portal 12 MB=0x5f92588a04cc0 00000400:00000200:18.0:1616513092.791750:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.26@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 4480 00000400:00000200:18.0:1616513092.791755:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.791758:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513092.791764:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.791767:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.791769:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.791772:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513092.791782:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681314484866240 00010000:00000200:30.0:1616513092.791792:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd701b00 x1681314484866240/t0(0) o400->f6b87e6d-3b37-0471-1f73-9d89d5363b3d@10.31.132.26@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513092.791805:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681314484866240, offset 192 00000400:00000200:30.0:1616513092.791811:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.26@tcp1 00000400:00000200:30.0:1616513092.791817:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.26@tcp1 routed destination 00000400:00000200:30.0:1616513092.791831:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.26@tcp1(10.31.132.26@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513092.791837:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513092.791842:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513092.791844:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513092.791846:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513092.791850:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.791856:0:50827:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.791869:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.791876:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513092.791879:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513092.791885:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513092.791889:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee440 00000800:00000200:11.2:1616513092.791892:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513092.791893:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.791896:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513092.791950:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.791959:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513092.791967:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.26@tcp1 : PUT - for me 00000400:00000200:20.0:1616513092.791975:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.26@tcp1 of length 224 into portal 26 MB=0x5f92588a04c40 00000400:00000200:20.0:1616513092.791981:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.26@tcp1 of length 224/224 into md 0xf97ce729 [1] + 0 00000400:00000200:20.0:1616513092.791986:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb05b18 00000400:00000200:20.0:1616513092.791988:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.791991:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513092.791995:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb05b18 00000800:00000200:20.0:1616513092.791998:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.792000:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513092.792002:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.792004:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.792055:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681314484866112 00010000:00000200:0.0:1616513092.792065:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1681314484866112/t0(0) o400->634b5d7a-33d3-5de6-df5a-615e66b56995@10.31.132.26@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.792077:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681314484866112, offset 192 00000400:00000200:0.0:1616513092.792082:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.26@tcp1 00000400:00000200:0.0:1616513092.792089:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.26@tcp1 routed destination 00000400:00000200:0.0:1616513092.792102:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.26@tcp1(10.31.132.26@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.792108:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.792113:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.792115:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.792117:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.792121:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.792127:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513092.792127:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513092.792178:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.792184:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513092.792187:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513092.792193:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513092.792198:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be198 00000800:00000200:12.0:1616513092.792202:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.792204:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.807015:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513092.807076:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.807085:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513092.807095:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.151@tcp1 : PUT - for me 00000400:00000200:14.0:1616513092.807104:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.151@tcp1 of length 224 into portal 26 MB=0x5ffd7682519c0 00000400:00000200:14.0:1616513092.807111:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.151@tcp1 of length 224/224 into md 0xf97ce739 [1] + 0 00000400:00000200:14.0:1616513092.807115:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04d48 00000400:00000200:14.0:1616513092.807118:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513092.807121:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513092.807125:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04d48 00000800:00000200:14.0:1616513092.807128:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.807130:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.807131:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513092.807134:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.807147:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675513866688 00010000:00000200:0.0:1616513092.807157:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688675513866688/t0(0) o400->0984150c-c8d3-0bf3-35f8-a6362b194718@10.31.129.151@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.807169:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688675513866688, offset 192 00000400:00000200:0.0:1616513092.807175:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.151@tcp1 00000400:00000200:0.0:1616513092.807182:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.151@tcp1 routed destination 00000400:00000200:0.0:1616513092.807199:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.151@tcp1(10.31.129.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.807205:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.807210:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.807213:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513092.807215:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.807215:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.807219:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.807224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.807224:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513092.807271:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.807280:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513092.807289:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.151@tcp1 : PUT - for me 00000800:00000200:22.0:1616513092.807295:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513092.807298:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.151@tcp1 of length 224 into portal 12 MB=0x5ffd768251d80 00000800:00000200:22.0:1616513092.807302:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513092.807304:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.151@tcp1 of length 224/224 into md 0x865be291 [1] + 42552 00000400:00000200:22.0:1616513092.807305:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513092.807309:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513092.807311:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:20.0:1616513092.807312:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513092.807318:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513092.807320:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf298 00000800:00000200:20.0:1616513092.807321:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513092.807323:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.807325:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513092.807325:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.807328:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513092.807333:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675513867648 00010000:00000200:8.0:1616513092.807342:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0da400 x1688675513867648/t0(0) o400->07530155-2453-308b-2672-4679439d635f@10.31.129.151@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513092.807353:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688675513867648, offset 192 00000400:00000200:8.0:1616513092.807358:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.151@tcp1 00000400:00000200:8.0:1616513092.807364:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.151@tcp1 routed destination 00000400:00000200:8.0:1616513092.807377:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.151@tcp1(10.31.129.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513092.807382:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513092.807386:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513092.807389:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513092.807390:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513092.807394:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.807399:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.807452:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.807459:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513092.807462:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513092.807469:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513092.807473:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ee0 00000800:00000200:12.0:1616513092.807476:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.807479:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.830165:0:50832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513092.830228:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.830237:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513092.830246:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.61@tcp1 : PUT - for me 00000400:00000200:18.0:1616513092.830256:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.61@tcp1 of length 224 into portal 26 MB=0x5ffd4ca58ff40 00000400:00000200:18.0:1616513092.830263:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.61@tcp1 of length 224/224 into md 0xf97ce759 [1] + 0 00000400:00000200:18.0:1616513092.830268:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d825eb04e58 00000400:00000200:18.0:1616513092.830270:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513092.830273:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513092.830277:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d825eb04e58 00000800:00000200:18.0:1616513092.830280:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.830282:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.830283:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513092.830286:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.830342:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664276533056 00010000:00000200:0.0:1616513092.830353:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688664276533056/t0(0) o400->9ebb6a97-a1be-f5e8-ce1f-68ccc27c186c@10.31.138.61@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.830367:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664276533056, offset 192 00000400:00000200:0.0:1616513092.830373:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.61@tcp1 00000400:00000200:0.0:1616513092.830381:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.61@tcp1 routed destination 00000400:00000200:0.0:1616513092.830396:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.61@tcp1(10.31.138.61@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.830403:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.830409:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.830412:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.830414:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.830419:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.830423:0:50832:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513092.830425:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513092.830477:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.830484:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513092.830487:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513092.830493:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513092.830499:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf650 00000800:00000200:22.0:1616513092.830502:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.830505:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513092.830589:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513092.830645:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.830654:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513092.830663:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.61@tcp1 : PUT - for me 00000400:00000200:20.0:1616513092.830672:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.61@tcp1 of length 224 into portal 12 MB=0x5ffd4ca590180 00000400:00000200:20.0:1616513092.830678:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.61@tcp1 of length 224/224 into md 0x865be291 [1] + 42776 00000400:00000200:20.0:1616513092.830686:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.830688:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513092.830696:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.830698:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513092.830698:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513092.830702:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513092.830709:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664276533632 00010000:00000200:8.0:1616513092.830718:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ec800 x1688664276533632/t0(0) o400->cc61e978-e1da-6772-abb3-536b81b44241@10.31.138.61@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513092.830728:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664276533632, offset 192 00000400:00000200:8.0:1616513092.830733:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.61@tcp1 00000400:00000200:8.0:1616513092.830738:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.61@tcp1 routed destination 00000400:00000200:8.0:1616513092.830749:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.61@tcp1(10.31.138.61@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513092.830754:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513092.830758:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513092.830760:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513092.830762:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513092.830765:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.830771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.830826:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.830832:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513092.830835:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513092.830842:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513092.830846:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3980 00000800:00000200:12.0:1616513092.830850:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.830853:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.851190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.851246:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.851255:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.851264:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.103@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.851273:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.103@tcp1 of length 224 into portal 26 MB=0x5ffdbc8dbb240 00000400:00000200:28.0:1616513092.851280:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.103@tcp1 of length 224/224 into md 0xf97ce789 [1] + 0 00000400:00000200:28.0:1616513092.851285:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faedd0 00000400:00000200:28.0:1616513092.851287:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.851290:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513092.851294:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faedd0 00000800:00000200:28.0:1616513092.851298:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.851299:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.851300:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.851303:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.851358:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694316315200 00010000:00000200:0.0:1616513092.851369:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688694316315200/t0(0) o400->7c792aa7-7771-3bdc-ec9c-78f52cda55a8@10.31.136.103@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.851384:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694316315200, offset 192 00000400:00000200:0.0:1616513092.851390:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.103@tcp1 00000400:00000200:0.0:1616513092.851398:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.103@tcp1 routed destination 00000400:00000200:0.0:1616513092.851413:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.103@tcp1(10.31.136.103@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.851419:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.851426:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.851428:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.851430:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.851448:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513092.851455:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513092.851492:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513092.851504:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513092.851506:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.851512:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513092.851515:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513092.851519:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513092.851522:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513092.851527:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf7e8 00000800:00000200:12.0:1616513092.851528:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513092.851531:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.851534:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513092.851537:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.103@tcp1 : PUT - for me 00000400:00000200:12.0:1616513092.851547:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.103@tcp1 of length 224 into portal 12 MB=0x5ffdbc8dbb340 00000400:00000200:12.0:1616513092.851553:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.103@tcp1 of length 224/224 into md 0x865be291 [1] + 43000 00000400:00000200:12.0:1616513092.851558:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513092.851561:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513092.851567:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.851571:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513092.851572:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.851575:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513092.851586:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694316315456 00010000:00000200:8.0:1616513092.851596:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d206300 x1688694316315456/t0(0) o400->111af96e-f3b7-887f-f4bd-66afdea06a33@10.31.136.103@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513092.851610:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694316315456, offset 192 00000400:00000200:8.0:1616513092.851616:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.103@tcp1 00000400:00000200:8.0:1616513092.851623:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.103@tcp1 routed destination 00000400:00000200:8.0:1616513092.851638:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.103@tcp1(10.31.136.103@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513092.851644:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513092.851650:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513092.851652:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513092.851655:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513092.851660:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.851665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.851676:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.851681:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513092.851684:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513092.851686:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513092.851690:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513092.851703:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf35c8 00000800:00000200:20.0:1616513092.851707:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.864514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513092.864570:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513092.864580:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513092.864589:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.29@tcp1 : PUT - for me 00000400:00000200:16.0:1616513092.864598:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.29@tcp1 of length 224 into portal 26 MB=0x5ffdda44412c0 00000400:00000200:16.0:1616513092.864605:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.29@tcp1 of length 224/224 into md 0xf97ce799 [1] + 0 00000400:00000200:16.0:1616513092.864610:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf3a8 00000400:00000200:16.0:1616513092.864613:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513092.864615:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513092.864622:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf3a8 00000800:00000200:12.0:1616513092.864624:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.864625:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513092.864632:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.864634:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.864688:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702292333248 00010000:00000200:0.0:1616513092.864698:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688702292333248/t0(0) o400->90954a2e-46f2-dc30-92fa-90f0779efd6b@10.31.138.29@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.864711:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702292333248, offset 192 00000400:00000200:0.0:1616513092.864719:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.29@tcp1 00000400:00000200:0.0:1616513092.864726:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.29@tcp1 routed destination 00000400:00000200:0.0:1616513092.864740:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.29@tcp1(10.31.138.29@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.864746:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.864752:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.864755:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.864757:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.864762:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513092.864768:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513092.864769:0:50835:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.864825:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.864832:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513092.864835:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513092.864841:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513092.864847:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfc28 00000800:00000200:22.0:1616513092.864851:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.864854:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513092.865919:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.865976:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.865985:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.865994:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.29@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.866002:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.29@tcp1 of length 224 into portal 12 MB=0x5ffdda4441400 00000400:00000200:28.0:1616513092.866008:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.29@tcp1 of length 224/224 into md 0x865be291 [1] + 43224 00000400:00000200:28.0:1616513092.866014:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.866016:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513092.866023:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.866026:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.866028:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513092.866031:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513092.866040:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702292333568 00010000:00000200:8.0:1616513092.866049:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedb180 x1688702292333568/t0(0) o400->a8a9dc19-eb58-20dd-80cf-a79c299a963b@10.31.138.29@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513092.866062:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702292333568, offset 192 00000400:00000200:8.0:1616513092.866068:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.29@tcp1 00000400:00000200:8.0:1616513092.866075:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.29@tcp1 routed destination 00000400:00000200:8.0:1616513092.866089:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.29@tcp1(10.31.138.29@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513092.866095:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513092.866101:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513092.866103:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513092.866105:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513092.866109:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.866115:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.866167:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.866173:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513092.866176:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513092.866182:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513092.866186:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2330 00000800:00000200:12.0:1616513092.866190:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.866193:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:31.0:1616513092.871553:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513092.871561:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513092.875153:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.875209:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.875218:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.875227:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.49@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.875237:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.49@tcp1 of length 224 into portal 26 MB=0x5ffe6ac885700 00000400:00000200:28.0:1616513092.875244:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.49@tcp1 of length 224/224 into md 0xf97ce7a9 [1] + 0 00000400:00000200:28.0:1616513092.875249:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae198 00000400:00000200:28.0:1616513092.875252:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.875254:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513092.875261:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae198 00000800:00000200:28.0:1616513092.875264:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.875265:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.875267:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.875272:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.875275:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513092.875276:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513092.875285:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.49@tcp1 : PUT - for me 00000400:00000200:20.0:1616513092.875294:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.49@tcp1 of length 224 into portal 12 MB=0x5ffe6ac885980 00000400:00000200:20.0:1616513092.875300:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.49@tcp1 of length 224/224 into md 0x865be291 [1] + 43448 00000400:00000200:20.0:1616513092.875306:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.875308:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513092.875315:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.875318:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513092.875320:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513092.875328:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688741085731200 00000100:00000200:0.0:1616513092.875329:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688741085730560 00010000:00000200:8.0:1616513092.875336:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0e8480 x1688741085731200/t0(0) o400->134f5fe1-a43a-6d40-bea3-d2a77a177626@10.31.140.49@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:0.0:1616513092.875339:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688741085730560/t0(0) o400->fdf46f50-6851-ef36-bc9e-25ac10f8833f@10.31.140.49@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513092.875348:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688741085731200, offset 192 00000400:00000200:8.0:1616513092.875353:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.49@tcp1 00000100:00000200:0.0:1616513092.875356:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688741085730560, offset 192 00000400:00000200:8.0:1616513092.875359:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.49@tcp1 routed destination 00000400:00000200:0.0:1616513092.875362:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.49@tcp1 00000400:00000200:0.0:1616513092.875369:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.49@tcp1 routed destination 00000400:00000200:8.0:1616513092.875373:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.49@tcp1(10.31.140.49@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513092.875379:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513092.875449:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.49@tcp1(10.31.140.49@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.875455:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513092.875456:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513092.875458:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513092.875460:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513092.875464:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513092.875470:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:7.2:1616513092.875471:0:50836:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.875473:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.875475:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.875479:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.875484:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513092.875487:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513092.875526:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.875534:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513092.875536:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513092.875542:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513092.875544:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513092.875546:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2440 00000800:00000200:14.0:1616513092.875550:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513092.875551:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513092.875552:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513092.875554:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513092.875556:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513092.875559:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513092.875561:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513092.875561:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.875563:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513092.875564:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513092.875566:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf100 00000800:00000200:14.0:1616513092.875566:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513092.875570:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.907233:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513092.907290:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513092.907299:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513092.907300:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513092.907309:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.206@tcp1 : PUT - for me 00000400:00000200:2.0:1616513092.907319:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.206@tcp1 of length 224 into portal 12 MB=0x5ffdbdc56a580 00000400:00000200:2.0:1616513092.907325:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.206@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 4704 00000400:00000200:2.0:1616513092.907331:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513092.907334:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513092.907343:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.907345:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513092.907346:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513092.907350:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513092.907367:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:6.0:1616513092.907367:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694643139968 00000800:00000200:20.0:1616513092.907377:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00010000:00000200:6.0:1616513092.907380:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4380 x1688694643139968/t0(0) o400->3da51561-26d9-4502-ddb2-ca9e2724b409@10.31.141.206@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513092.907386:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.206@tcp1 : PUT - for me 00000400:00000200:20.0:1616513092.907395:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.206@tcp1 of length 224 into portal 26 MB=0x5ffdbdc56a4c0 00000100:00000200:6.0:1616513092.907395:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694643139968, offset 192 00000400:00000200:6.0:1616513092.907407:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.206@tcp1 00000400:00000200:20.0:1616513092.907413:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.206@tcp1 of length 224/224 into md 0xf97ce7c9 [1] + 0 00000400:00000200:6.0:1616513092.907415:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.206@tcp1 routed destination 00000400:00000200:20.0:1616513092.907418:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf298 00000400:00000200:20.0:1616513092.907420:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.907423:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513092.907428:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf298 00000800:00000200:12.0:1616513092.907429:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:6.0:1616513092.907430:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.206@tcp1(10.31.141.206@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513092.907444:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513092.907447:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.907450:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:6.0:1616513092.907450:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513092.907452:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:6.0:1616513092.907452:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513092.907455:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513092.907459:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.907464:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.907480:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.907486:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513092.907489:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513092.907492:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513092.907493:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694643139776 00000400:00000200:18.0:1616513092.907496:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513092.907500:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01650 00000800:00000200:18.0:1616513092.907504:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513092.907504:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688694643139776/t0(0) o400->e08f60cf-07e5-8e3e-e706-922a195e8d66@10.31.141.206@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.907519:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694643139776, offset 192 00000400:00000200:0.0:1616513092.907525:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.206@tcp1 00000400:00000200:0.0:1616513092.907532:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.206@tcp1 routed destination 00000400:00000200:0.0:1616513092.907547:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.206@tcp1(10.31.141.206@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.907554:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513092.907566:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.907569:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.907571:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.907576:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.907581:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513092.907583:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513092.907645:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.907651:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513092.907654:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.907661:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.907666:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfb18 00000800:00000200:28.0:1616513092.907670:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.907673:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.970562:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.970620:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.970630:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513092.970640:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.76@tcp1 : PUT - for me 00000400:00000200:12.0:1616513092.970650:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.76@tcp1 of length 224 into portal 26 MB=0x601ad7c08efc0 00000400:00000200:12.0:1616513092.970657:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.76@tcp1 of length 224/224 into md 0xf97ce7d9 [1] + 0 00000400:00000200:12.0:1616513092.970662:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fafba0 00000400:00000200:12.0:1616513092.970666:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513092.970669:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513092.970674:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafba0 00000800:00000200:20.0:1616513092.970675:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.970678:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513092.970681:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513092.970683:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.970744:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690694482194368 00010000:00000200:0.0:1616513092.970757:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1690694482194368/t0(0) o400->4df55e66-cb7c-297b-5956-0320bb2c340a@10.31.140.76@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.970773:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690694482194368, offset 192 00000400:00000200:0.0:1616513092.970780:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.76@tcp1 00000400:00000200:0.0:1616513092.970788:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.76@tcp1 routed destination 00000400:00000200:0.0:1616513092.970804:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.76@tcp1(10.31.140.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.970811:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513092.970818:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.970821:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513092.970823:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.970830:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513092.970837:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513092.970838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513092.970896:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513092.970903:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513092.970906:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513092.970913:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513092.970919:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beff0 00000800:00000200:14.0:1616513092.970923:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513092.970926:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513092.971082:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.971137:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.971147:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513092.971155:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.76@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.971164:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.76@tcp1 of length 224 into portal 12 MB=0x601ad7c08fc00 00000400:00000200:28.0:1616513092.971171:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.76@tcp1 of length 224/224 into md 0x865be291 [1] + 43672 00000400:00000200:28.0:1616513092.971177:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.971179:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513092.971186:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.971190:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513092.971191:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513092.971194:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513092.971207:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690694482197504 00010000:00000200:8.0:1616513092.971219:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeedc800 x1690694482197504/t0(0) o400->9a6fca0a-cb5e-6cd6-3773-68cf989244e9@10.31.140.76@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513092.971232:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690694482197504, offset 192 00000400:00000200:8.0:1616513092.971238:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.76@tcp1 00000400:00000200:8.0:1616513092.971245:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.76@tcp1 routed destination 00000400:00000200:8.0:1616513092.971259:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.76@tcp1(10.31.140.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513092.971264:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513092.971270:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513092.971272:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513092.971274:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513092.971279:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513092.971284:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513092.971294:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.971300:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513092.971302:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513092.971304:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513092.971309:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513092.971313:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3cb0 00000800:00000200:20.0:1616513092.971317:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513092.983249:0:50847:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513092.983305:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.983315:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513092.983323:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.109@tcp1 : PUT - for me 00000400:00000200:22.0:1616513092.983333:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.109@tcp1 of length 224 into portal 26 MB=0x5f91ac86a25c0 00000400:00000200:22.0:1616513092.983339:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.109@tcp1 of length 224/224 into md 0xf97ce7e9 [1] + 0 00000400:00000200:22.0:1616513092.983344:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf870 00000400:00000200:22.0:1616513092.983346:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513092.983349:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513092.983353:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf870 00000800:00000200:22.0:1616513092.983356:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.983359:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513092.983361:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.983363:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.983368:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513092.983375:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.109@tcp1 : PUT - for me 00000400:00000200:22.0:1616513092.983381:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.109@tcp1 of length 224 into portal 12 MB=0x5f91ac86a2640 00000400:00000200:22.0:1616513092.983386:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.109@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 4928 00000400:00000200:22.0:1616513092.983392:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513092.983394:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513092.983400:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.983402:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513092.983403:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.983406:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513092.983418:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681268310418880 00000100:00000200:6.0:1616513092.983421:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681268310419008 00010000:00000200:0.0:1616513092.983429:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1681268310418880/t0(0) o400->c608d7be-6df9-48bb-d54d-198a90f83d8b@10.31.143.109@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:6.0:1616513092.983431:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2c800 x1681268310419008/t0(0) o400->84b38476-d982-253e-7217-750adb18d6f8@10.31.143.109@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513092.983447:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681268310419008, offset 192 00000400:00000200:6.0:1616513092.983453:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.109@tcp1 00000100:00000200:0.0:1616513092.983457:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681268310418880, offset 192 00000400:00000200:6.0:1616513092.983460:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.109@tcp1 routed destination 00000400:00000200:0.0:1616513092.983464:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.109@tcp1 00000400:00000200:6.0:1616513092.983472:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.109@tcp1(10.31.143.109@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513092.983472:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.109@tcp1 routed destination 00000800:00000200:6.0:1616513092.983478:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:6.0:1616513092.983483:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513092.983486:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513092.983487:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513092.983487:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.109@tcp1(10.31.143.109@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513092.983492:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513092.983494:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513092.983497:0:50847:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513092.983499:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.983502:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.983504:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.983509:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513092.983518:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513092.983550:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513092.983553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513092.983557:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513092.983560:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513092.983562:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513092.983566:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513092.983571:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00ff0 00000800:00000200:2.0:1616513092.983574:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513092.983609:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513092.983616:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513092.983618:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513092.983625:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513092.983631:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bff58 00000800:00000200:28.0:1616513092.983634:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.983637:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513092.988196:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513092.988253:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513092.988262:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513092.988271:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:12.0:1616513092.988282:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d0322c6380 00000400:00000200:12.0:1616513092.988288:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0x865be291 [1] + 43896 00000400:00000200:12.0:1616513092.988294:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513092.988296:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513092.988303:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.988306:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513092.988306:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513092.988311:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513092.988328:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042590114688 00010000:00000200:8.0:1616513092.988408:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d203600 x1693042590114688/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:718/0 lens 440/368 e 0 to 0 dl 1616513098 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513092.988420:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042590114688, offset 192 00000400:00000200:8.0:1616513092.988425:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:8.0:1616513092.988432:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:8.0:1616513092.988451:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513092.988457:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513092.988462:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513092.988464:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513092.988466:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513092.988470:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.988478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513092.988532:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513092.988539:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513092.988542:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513092.988548:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513092.988552:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf26e8 00000800:00000200:22.0:1616513092.988556:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513092.988559:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513092.996865:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513092.996921:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.996931:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513092.996933:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513092.996940:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.7@tcp1 : PUT - for me 00000400:00000200:28.0:1616513092.996949:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.7@tcp1 of length 224 into portal 26 MB=0x60006c253eec0 00000400:00000200:28.0:1616513092.996958:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.7@tcp1 of length 224/224 into md 0xf97ce7f9 [1] + 0 00000400:00000200:28.0:1616513092.996963:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf100 00000400:00000200:28.0:1616513092.996965:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513092.996968:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513092.996973:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf100 00000800:00000200:28.0:1616513092.996976:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513092.996978:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513092.996980:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513092.997039:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688878890348224 00010000:00000200:0.0:1616513092.997050:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688878890348224/t0(0) o400->afa66ea3-c5aa-348c-c6d7-878e1c8eb9d1@10.31.133.7@tcp1:49/0 lens 224/224 e 0 to 0 dl 1616513184 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513092.997064:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688878890348224, offset 192 00000400:00000200:0.0:1616513092.997070:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.7@tcp1 00000400:00000200:0.0:1616513092.997078:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.7@tcp1 routed destination 00000400:00000200:0.0:1616513092.997093:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.7@tcp1(10.31.133.7@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513092.997099:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:11.2:1616513092.997100:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513092.997105:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513092.997112:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513092.997114:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513092.997119:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513092.997125:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513092.997154:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513092.997163:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513092.997172:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.7@tcp1 : PUT - for me 00000400:00000200:20.0:1616513092.997181:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.7@tcp1 of length 224 into portal 12 MB=0x60006c253efc0 00000400:00000200:20.0:1616513092.997187:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.7@tcp1 of length 224/224 into md 0x865be291 [1] + 44336 00000400:00000200:20.0:1616513092.997193:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513092.997195:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513092.997202:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513092.997205:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513092.997206:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000100:00000200:8.0:1616513092.997212:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688878890348480 00000800:00000200:20.0:1616513092.997213:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513092.997215:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513092.997216:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513092.997217:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:8.0:1616513092.997218:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ea880 x1688878890348480/t0(0) o400->8cf3d54c-8b99-5411-1e2a-b18e32d32b94@10.31.133.7@tcp1:718/0 lens 224/224 e 0 to 0 dl 1616513098 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513092.997224:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513092.997230:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfcb0 00000100:00000200:8.0:1616513092.997232:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688878890348480, offset 192 00000800:00000200:12.0:1616513092.997233:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513092.997236:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.7@tcp1 00000400:00000200:8.0:1616513092.997242:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.7@tcp1 routed destination 00000400:00000200:8.0:1616513092.997253:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.7@tcp1(10.31.133.7@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513092.997258:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513092.997261:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513092.997264:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513092.997266:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513092.997268:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513092.997275:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513092.997329:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513092.997336:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513092.997338:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513092.997345:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513092.997349:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3760 00000800:00000200:18.0:1616513092.997353:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513092.997356:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.017966:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513093.018020:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.018029:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513093.018032:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513093.018038:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.37@tcp1 : PUT - for me 00000400:00000200:22.0:1616513093.018048:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.37@tcp1 of length 224 into portal 12 MB=0x5ffddd7460980 00000400:00000200:22.0:1616513093.018053:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.37@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 5152 00000400:00000200:22.0:1616513093.018058:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.018061:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513093.018067:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.018070:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.018071:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513093.018085:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703148099968 00010000:00000200:6.0:1616513093.018095:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a28000 x1688703148099968/t0(0) o400->e31725e8-15c5-eba1-7059-8b16968c3cd9@10.31.140.37@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513093.018106:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703148099968, offset 192 00000400:00000200:6.0:1616513093.018111:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.37@tcp1 00000400:00000200:6.0:1616513093.018118:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.37@tcp1 routed destination 00000800:00000200:11.2:1616513093.018130:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:6.0:1616513093.018130:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.37@tcp1(10.31.140.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513093.018135:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:6.0:1616513093.018140:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513093.018142:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513093.018144:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.018148:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:6.0:1616513093.018148:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513093.018157:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513093.018166:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.37@tcp1 : PUT - for me 00000400:00000200:28.0:1616513093.018175:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.37@tcp1 of length 224 into portal 26 MB=0x5ffddd7460680 00000400:00000200:28.0:1616513093.018182:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.37@tcp1 of length 224/224 into md 0xf97ce809 [1] + 0 00000400:00000200:28.0:1616513093.018186:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faeee0 00000400:00000200:28.0:1616513093.018189:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.018192:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513093.018196:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faeee0 00000800:00000200:16.0:1616513093.018199:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513093.018200:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513093.018202:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:28.0:1616513093.018205:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513093.018207:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513093.018208:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.018209:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513093.018215:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513093.018219:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00440 00000800:00000200:16.0:1616513093.018223:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.018258:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703148099200 00010000:00000200:0.0:1616513093.018268:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688703148099200/t0(0) o400->93947cdc-0ff3-b891-4283-ca9dc7da003c@10.31.140.37@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.018282:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703148099200, offset 192 00000400:00000200:0.0:1616513093.018288:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.37@tcp1 00000400:00000200:0.0:1616513093.018296:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.37@tcp1 routed destination 00000400:00000200:0.0:1616513093.018309:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.37@tcp1(10.31.140.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.018315:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.018321:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.018323:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.018325:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.018330:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.018335:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.018336:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513093.018389:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.018396:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.018399:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513093.018406:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.018411:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beb28 00000800:00000200:18.0:1616513093.018415:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.018418:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.027126:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513093.027180:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.027189:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513093.027197:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.67@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.027205:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.67@tcp1 of length 224 into portal 26 MB=0x603ff6c15c8c0 00000400:00000200:14.0:1616513093.027211:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.67@tcp1 of length 224/224 into md 0xf97ce829 [1] + 0 00000400:00000200:14.0:1616513093.027218:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae880 00000400:00000200:14.0:1616513093.027220:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.027223:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513093.027227:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae880 00000800:00000200:14.0:1616513093.027230:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.027232:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.027234:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.027236:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.027241:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513093.027247:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.67@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.027252:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.67@tcp1 of length 224 into portal 12 MB=0x603ff6c15c940 00000400:00000200:14.0:1616513093.027257:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.67@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 5376 00000400:00000200:14.0:1616513093.027261:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.027263:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513093.027267:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.027269:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.027270:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.027273:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513093.027284:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245425174848 00000100:00000200:0.0:1616513093.027291:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245425174720 00010000:00000200:6.0:1616513093.027292:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e00480 x1693245425174848/t0(0) o400->fa796449-551b-6b7f-dc0c-1d922fe7d0bd@10.31.132.67@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:0.0:1616513093.027302:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1693245425174720/t0(0) o400->22740cb1-b3f2-4812-be7f-294a1852af03@10.31.132.67@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:6.0:1616513093.027304:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245425174848, offset 192 00000400:00000200:6.0:1616513093.027309:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.67@tcp1 00000400:00000200:6.0:1616513093.027314:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.67@tcp1 routed destination 00000100:00000200:0.0:1616513093.027316:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245425174720, offset 192 00000400:00000200:0.0:1616513093.027325:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.67@tcp1 00000400:00000200:6.0:1616513093.027326:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.67@tcp1(10.31.132.67@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513093.027331:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513093.027333:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.67@tcp1 routed destination 00000800:00000200:6.0:1616513093.027335:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513093.027338:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:6.0:1616513093.027340:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513093.027343:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513093.027349:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.67@tcp1(10.31.132.67@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.027355:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.027361:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.027364:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.027366:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.027371:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.027376:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.027377:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.027408:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513093.027430:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.027445:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.027457:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513093.027460:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513093.027464:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513093.027467:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513093.027471:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513093.027473:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513093.027473:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be6e8 00000800:00000200:2.0:1616513093.027476:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513093.027480:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513093.027484:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00440 00000800:00000200:12.0:1616513093.027488:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513093.027491:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513093.036779:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513093.036836:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.036845:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513093.036854:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.48@tcp1 : PUT - for me 00000400:00000200:16.0:1616513093.036864:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.48@tcp1 of length 224 into portal 26 MB=0x603d045d14d40 00000400:00000200:16.0:1616513093.036871:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.48@tcp1 of length 224/224 into md 0xf97ce839 [1] + 0 00000400:00000200:16.0:1616513093.036876:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae440 00000400:00000200:16.0:1616513093.036878:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.036881:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513093.036885:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae440 00000800:00000200:16.0:1616513093.036888:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.036889:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.036891:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513093.036893:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.036948:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042919689536 00010000:00000200:0.0:1616513093.036959:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1693042919689536/t0(0) o400->820cc64e-8e2e-429e-c1bc-9047d6d37047@10.31.140.48@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.036974:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693042919689536, offset 192 00000400:00000200:0.0:1616513093.036980:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.48@tcp1 00000400:00000200:0.0:1616513093.036988:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.48@tcp1 routed destination 00000400:00000200:0.0:1616513093.037003:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.48@tcp1(10.31.140.48@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.037009:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.037015:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.037017:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.037019:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.037024:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.037029:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.037031:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513093.037060:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513093.037082:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.037089:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513093.037091:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513093.037098:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513093.037104:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bec38 00000800:00000200:12.0:1616513093.037107:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513093.037110:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513093.037113:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.037122:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513093.037130:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.48@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.037138:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.48@tcp1 of length 224 into portal 12 MB=0x603d045d14f80 00000400:00000200:14.0:1616513093.037144:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.48@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 5600 00000400:00000200:14.0:1616513093.037148:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.037151:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513093.037156:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.037159:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.037161:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.037164:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513093.037171:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042919690112 00010000:00000200:6.0:1616513093.037180:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e00900 x1693042919690112/t0(0) o400->5898e81a-3c0d-afb8-a73d-89b5bcae2c96@10.31.140.48@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513093.037191:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693042919690112, offset 192 00000400:00000200:6.0:1616513093.037195:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.48@tcp1 00000400:00000200:6.0:1616513093.037201:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.48@tcp1 routed destination 00000400:00000200:6.0:1616513093.037212:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.48@tcp1(10.31.140.48@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513093.037217:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:6.0:1616513093.037222:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513093.037224:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513093.037225:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513093.037229:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.037234:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.037245:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.037252:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.037254:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513093.037261:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.037265:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00ff0 00000800:00000200:22.0:1616513093.037269:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.037272:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.042146:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513093.042198:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.042207:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513093.042215:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.110@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.042224:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.110@tcp1 of length 224 into portal 26 MB=0x6000849d68680 00000400:00000200:14.0:1616513093.042232:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.110@tcp1 of length 224/224 into md 0xf97ce849 [1] + 0 00000400:00000200:14.0:1616513093.042236:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae000 00000400:00000200:14.0:1616513093.042239:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.042241:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513093.042245:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae000 00000800:00000200:14.0:1616513093.042248:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.042251:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513093.042251:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.042258:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.042261:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.042262:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513093.042271:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.110@tcp1 : PUT - for me 00000400:00000200:18.0:1616513093.042280:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.110@tcp1 of length 224 into portal 12 MB=0x6000849d68780 00000400:00000200:18.0:1616513093.042286:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.110@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 5824 00000400:00000200:18.0:1616513093.042292:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.042294:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513093.042302:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.042306:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.042307:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.042307:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688885458798208 00000100:00000200:10.0:1616513093.042317:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688885458798464 00010000:00000200:0.0:1616513093.042317:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1688885458798208/t0(0) o400->d258bcec-f3cb-451e-a457-5685bd5b89fd@10.31.138.110@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:10.0:1616513093.042327:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd703180 x1688885458798464/t0(0) o400->caf0d9ea-7b49-b677-e6ce-69c43e0af531@10.31.138.110@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513093.042330:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688885458798208, offset 192 00000100:00000200:10.0:1616513093.042337:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688885458798464, offset 192 00000400:00000200:0.0:1616513093.042337:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.110@tcp1 00000400:00000200:10.0:1616513093.042342:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.110@tcp1 00000400:00000200:0.0:1616513093.042344:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.110@tcp1 routed destination 00000400:00000200:10.0:1616513093.042348:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.110@tcp1 routed destination 00000400:00000200:0.0:1616513093.042358:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.110@tcp1(10.31.138.110@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513093.042360:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.110@tcp1(10.31.138.110@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.042364:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513093.042365:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513093.042370:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.042371:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513093.042372:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513093.042374:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.042374:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.042376:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513093.042377:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513093.042380:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513093.042382:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.042387:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.042427:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513093.042449:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.042456:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513093.042458:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513093.042460:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513093.042465:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513093.042469:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83980 00000800:00000200:2.0:1616513093.042473:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513093.042482:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.042489:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.042492:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.042498:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.042504:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf5c8 00000800:00000200:20.0:1616513093.042507:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.042510:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513093.056790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513093.056828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513093.056852:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.056861:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513093.056870:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.240@tcp1 : PUT - for me 00000400:00000200:18.0:1616513093.056880:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.240@tcp1 of length 224 into portal 26 MB=0x60233ff94a480 00000800:00000200:12.0:1616513093.056883:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513093.056886:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.240@tcp1 of length 224/224 into md 0xf97ce859 [1] + 0 00000400:00000200:18.0:1616513093.056891:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae6e8 00000800:00000200:12.0:1616513093.056893:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:18.0:1616513093.056894:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.056897:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513093.056901:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae6e8 00000400:00000200:12.0:1616513093.056902:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.240@tcp1 : PUT - for me 00000800:00000200:18.0:1616513093.056905:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.056908:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513093.056909:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:12.0:1616513093.056911:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.240@tcp1 of length 224 into portal 12 MB=0x60233ff94a6c0 00000800:00000200:18.0:1616513093.056913:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513093.056917:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.240@tcp1 of length 224/224 into md 0x865be291 [1] + 44560 00000400:00000200:12.0:1616513093.056923:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513093.056927:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513093.056933:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.056936:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513093.056938:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513093.056941:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513093.056949:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691272214783680 00010000:00000200:8.0:1616513093.056958:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d205a00 x1691272214783680/t0(0) o400->3ea63e59-9d07-2bd6-4dc3-93acb861270d@10.31.142.240@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513093.056969:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691272214783680, offset 192 00000100:00000200:0.0:1616513093.056969:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691272214783104 00000400:00000200:8.0:1616513093.056974:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.240@tcp1 00010000:00000200:0.0:1616513093.056979:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1691272214783104/t0(0) o400->489eacf0-f0bd-497e-4436-2c6147ed5ed4@10.31.142.240@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513093.056980:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.240@tcp1 routed destination 00000400:00000200:8.0:1616513093.056992:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.240@tcp1(10.31.142.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513093.056993:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691272214783104, offset 192 00000800:00000200:8.0:1616513093.056997:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513093.056999:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.240@tcp1 00000800:00000200:8.0:1616513093.057002:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.057004:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513093.057006:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513093.057006:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.240@tcp1 routed destination 00000800:00000200:8.0:1616513093.057010:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.057014:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513093.057020:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.240@tcp1(10.31.142.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.057026:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.057031:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.057034:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.057036:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.057040:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.057045:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.057046:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513093.057067:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.057081:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.057084:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.057091:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.057094:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3650 00000800:00000200:14.0:1616513093.057097:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513093.057100:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.057103:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513093.057104:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513093.057106:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513093.057110:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513093.057113:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.057118:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be4c8 00000800:00000200:14.0:1616513093.057121:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.066387:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513093.066458:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.066467:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513093.066478:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.186@tcp1 : PUT - for me 00000400:00000200:22.0:1616513093.066488:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.186@tcp1 of length 224 into portal 26 MB=0x5ffde95f82400 00000400:00000200:22.0:1616513093.066495:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.186@tcp1 of length 224/224 into md 0xf97ce879 [1] + 0 00000400:00000200:22.0:1616513093.066499:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae908 00000400:00000200:22.0:1616513093.066502:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.066505:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513093.066509:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae908 00000800:00000200:22.0:1616513093.066512:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.066514:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.066515:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513093.066518:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.066538:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706347443200 00010000:00000200:0.0:1616513093.066547:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688706347443200/t0(0) o400->9731353a-a51f-e821-d202-e2ba0d42a085@10.31.128.186@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.066560:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706347443200, offset 192 00000400:00000200:0.0:1616513093.066565:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.186@tcp1 00000800:00000200:7.2:1616513093.066568:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513093.066572:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.186@tcp1 routed destination 00000800:00000200:2.0:1616513093.066580:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513093.066585:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.186@tcp1(10.31.128.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513093.066589:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.066591:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.066596:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513093.066598:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.186@tcp1 : PUT - for me 00000800:00000200:0.0:1616513093.066599:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.066600:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.066605:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513093.066607:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.186@tcp1 of length 224 into portal 12 MB=0x5ffde95f828c0 00000100:00000200:0.0:1616513093.066612:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:19.0:1616513093.066613:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928480, portal 4 00000400:00000200:2.0:1616513093.066613:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.186@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 6048 00000100:00000200:19.0:1616513093.066620:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928480, offset 0 00000400:00000200:2.0:1616513093.066620:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.066623:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:19.0:1616513093.066627:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:2.0:1616513093.066632:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:19.0:1616513093.066633:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:14.0:1616513093.066634:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.066635:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.066638:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:19.0:1616513093.066648:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513093.066648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:18.0:1616513093.066652:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706347444416 00000800:00000200:19.0:1616513093.066655:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:19.0:1616513093.066662:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00010000:00000200:18.0:1616513093.066663:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d2400 x1688706347444416/t0(0) o400->a742e05d-9433-3718-f2a4-cdee49404d95@10.31.128.186@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:19.0:1616513093.066665:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:19.0:1616513093.066668:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:19.0:1616513093.066674:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:18.0:1616513093.066676:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706347444416, offset 192 00000400:00000200:18.0:1616513093.066682:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.186@tcp1 00000400:00000200:18.0:1616513093.066689:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.186@tcp1 routed destination 00000400:00000200:18.0:1616513093.066703:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.186@tcp1(10.31.128.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513093.066703:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513093.066709:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513093.066709:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513093.066712:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513093.066715:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513093.066717:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513093.066719:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513093.066719:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:5.2:1616513093.066719:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:12.0:1616513093.066725:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beaa0 00000800:00000200:18.0:1616513093.066729:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513093.066729:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513093.066732:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513093.066734:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513093.066735:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513093.066740:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513093.066742:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513093.066744:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513093.066747:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:18.0:1616513093.066748:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.066750:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513093.066753:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513093.066753:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc566300 x1693067725928480/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513093.066755:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.066758:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc550 00000800:00000200:18.0:1616513093.066761:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513093.066765:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706aa0 00000800:00000200:12.0:1616513093.066768:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513093.066770:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513093.066826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:12.0:1616513093.066836:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513093.066842:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.230@o2ib1 00000800:00000200:12.0:1616513093.066844:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513093.066845:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513093.066846:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:12.0:1616513093.066849:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513093.066920:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513093.066931:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513093.066940:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:16.0:1616513093.066943:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:20.0:1616513093.066949:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513093.066957:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627820 00000400:00000200:20.0:1616513093.066964:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866db881 [1] + 192 00000400:00000200:20.0:1616513093.066969:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513093.066974:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc566300 x1693067725928480/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513093.066989:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513093.066992:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513093.066994:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:19.0:1616513093.067000:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc566300 x1693067725928480/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513093.067012:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc566300 x1693067725928480/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513093.067022:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000800:00000200:11.2:1616513093.129523:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513093.129582:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.129592:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513093.129601:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.133@tcp1 : PUT - for me 00000400:00000200:28.0:1616513093.129611:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.133@tcp1 of length 224 into portal 26 MB=0x5ffdf3995cdc0 00000400:00000200:28.0:1616513093.129618:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.133@tcp1 of length 224/224 into md 0xf97ce889 [1] + 0 00000400:00000200:28.0:1616513093.129622:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae4c8 00000400:00000200:28.0:1616513093.129625:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.129628:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513093.129633:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae4c8 00000800:00000200:28.0:1616513093.129636:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.129641:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513093.129643:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513093.129645:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513093.129656:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513093.129665:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.133@tcp1 : PUT - for me 00000400:00000200:12.0:1616513093.129674:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.133@tcp1 of length 224 into portal 12 MB=0x5ffdf3995ce40 00000400:00000200:12.0:1616513093.129680:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.133@tcp1 of length 224/224 into md 0x865be291 [1] + 44784 00000400:00000200:12.0:1616513093.129686:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513093.129688:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513093.129695:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513093.129695:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709092462016 00000800:00000200:12.0:1616513093.129700:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513093.129702:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.129704:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513093.129705:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688709092462016/t0(0) o400->6b4a3909-2c96-704e-f9d4-e5c60da08347@10.31.142.133@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513093.129711:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709092462144 00000100:00000200:0.0:1616513093.129717:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688709092462016, offset 192 00010000:00000200:8.0:1616513093.129721:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d201680 x1688709092462144/t0(0) o400->f2bddaa5-3ae6-14f2-0ea5-c6d825e25ac3@10.31.142.133@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513093.129723:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.133@tcp1 00000400:00000200:0.0:1616513093.129729:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.133@tcp1 routed destination 00000100:00000200:8.0:1616513093.129733:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688709092462144, offset 192 00000400:00000200:8.0:1616513093.129738:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.133@tcp1 00000400:00000200:0.0:1616513093.129743:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.133@tcp1(10.31.142.133@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.129748:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:8.0:1616513093.129750:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.133@tcp1 routed destination 00000800:00000200:0.0:1616513093.129757:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.129759:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.129761:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513093.129763:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.133@tcp1(10.31.142.133@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.129766:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513093.129768:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513093.129772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.129772:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513093.129774:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.129776:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513093.129777:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513093.129781:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.129790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513093.129825:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.129832:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513093.129835:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513093.129835:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.129842:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.129847:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be880 00000800:00000200:14.0:1616513093.129849:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513093.129851:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513093.129856:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513093.129859:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513093.129866:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.129870:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf22a8 00000800:00000200:14.0:1616513093.129873:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.129876:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513093.130480:0:6487:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928496, portal 4 00000100:00000200:24.0:1616513093.130485:0:6487:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928496, offset 0 00000400:00000200:24.0:1616513093.130492:0:6487:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:24.0:1616513093.130498:0:6487:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:24.0:1616513093.130511:0:6487:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513093.130517:0:6487:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:24.0:1616513093.130523:0:6487:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513093.130526:0:6487:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:24.0:1616513093.130528:0:6487:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513093.130533:0:6487:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513093.130579:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513093.130592:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513093.130596:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513093.130598:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513093.130600:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:12.0:1616513093.130604:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:12.0:1616513093.130609:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cbeede300 x1693067725928496/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513093.130621:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f26e8 00000800:00000200:12.0:1616513093.130624:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513093.130758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513093.130768:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513093.130774:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513093.130778:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:28.0:1616513093.130782:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513093.130787:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627830 00000400:00000200:28.0:1616513093.130793:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866db8b1 [1] + 192 00000400:00000200:28.0:1616513093.130798:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.130802:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cbeede300 x1693067725928496/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:7.2:1616513093.130815:0:50863:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513093.130816:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513093.130818:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513093.130820:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:24.0:1616513093.130824:0:6487:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cbeede300 x1693067725928496/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:24.0:1616513093.130837:0:6487:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cbeede300 x1693067725928496/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513093.130846:0:6487:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2aa0 00000800:00000200:22.0:1616513093.130871:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.130881:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513093.130890:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.19@tcp1 : PUT - for me 00000400:00000200:22.0:1616513093.130899:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.19@tcp1 of length 224 into portal 12 MB=0x604f9ccf71700 00000400:00000200:22.0:1616513093.130906:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.19@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 6272 00000400:00000200:22.0:1616513093.130911:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.130913:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513093.130922:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.130925:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513093.130927:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.130930:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.130963:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513093.130977:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.130985:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513093.130986:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:26.0:1616513093.130988:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694320792377088 00000400:00000200:16.0:1616513093.130995:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.19@tcp1 : PUT - for me 00010000:00000200:26.0:1616513093.131001:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a28900 x1694320792377088/t0(0) o400->1b948c47-a6bc-3007-ff95-2d9ed32b822c@10.31.141.19@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513093.131004:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.19@tcp1 of length 224 into portal 26 MB=0x604f9ccf71440 00000400:00000200:16.0:1616513093.131011:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.19@tcp1 of length 224/224 into md 0xf97ce899 [1] + 0 00000400:00000200:16.0:1616513093.131015:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf210 00000100:00000200:26.0:1616513093.131018:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694320792377088, offset 192 00000400:00000200:16.0:1616513093.131018:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.131021:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513093.131025:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf210 00000800:00000200:16.0:1616513093.131028:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513093.131031:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.19@tcp1 00000800:00000200:16.0:1616513093.131031:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.131033:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513093.131040:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.19@tcp1 routed destination 00000400:00000200:26.0:1616513093.131054:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.19@tcp1(10.31.141.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.131061:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513093.131067:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.131069:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513093.131072:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513093.131077:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.131082:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.131091:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694320792376384 00000800:00000200:20.0:1616513093.131092:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.131096:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.131097:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513093.131100:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:0.0:1616513093.131102:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1694320792376384/t0(0) o400->5eda2b9f-5d34-4d14-488e-a8c299b20339@10.31.141.19@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513093.131103:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.131106:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af320 00000800:00000200:20.0:1616513093.131108:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.131117:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694320792376384, offset 192 00000400:00000200:0.0:1616513093.131123:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.19@tcp1 00000400:00000200:0.0:1616513093.131131:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.19@tcp1 routed destination 00000400:00000200:0.0:1616513093.131146:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.19@tcp1(10.31.141.19@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.131153:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.131159:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.131161:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.131164:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.131169:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513093.131175:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513093.131176:0:50863:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.131230:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.131237:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513093.131240:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513093.131246:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.131252:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be2a8 00000800:00000200:14.0:1616513093.131255:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.131258:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.251481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513093.251538:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513093.251549:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513093.251559:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.166@tcp1 : PUT - for me 00000400:00000200:12.0:1616513093.251570:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.166@tcp1 of length 224 into portal 26 MB=0x6001dfefae1c0 00000400:00000200:12.0:1616513093.251577:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.166@tcp1 of length 224/224 into md 0xf97ce8c9 [1] + 0 00000400:00000200:12.0:1616513093.251582:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf8f8 00000400:00000200:12.0:1616513093.251586:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513093.251589:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:16.0:1616513093.251594:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513093.251594:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf8f8 00000800:00000200:12.0:1616513093.251598:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513093.251601:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513093.251603:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513093.251606:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513093.251606:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513093.251615:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.166@tcp1 : PUT - for me 00000400:00000200:16.0:1616513093.251625:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.166@tcp1 of length 224 into portal 12 MB=0x6001dfefae240 00000400:00000200:16.0:1616513093.251632:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.166@tcp1 of length 224/224 into md 0x865be291 [1] + 45008 00000400:00000200:16.0:1616513093.251638:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.251640:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513093.251649:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.251652:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.251654:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.251665:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688978692170176 00000100:00000200:8.0:1616513093.251672:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688978692170304 00010000:00000200:0.0:1616513093.251679:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688978692170176/t0(0) o400->228c0ff7-3f97-7e3b-072a-c36d58b29bdd@10.31.141.166@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:8.0:1616513093.251685:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6722880 x1688978692170304/t0(0) o400->625a366c-8d76-8030-8636-1d4744cc1d90@10.31.141.166@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513093.251695:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688978692170176, offset 192 00000400:00000200:0.0:1616513093.251704:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.166@tcp1 00000100:00000200:8.0:1616513093.251709:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688978692170304, offset 192 00000400:00000200:0.0:1616513093.251712:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.166@tcp1 routed destination 00000400:00000200:8.0:1616513093.251715:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.166@tcp1 00000400:00000200:0.0:1616513093.251728:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.166@tcp1(10.31.141.166@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.251736:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:8.0:1616513093.251737:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.166@tcp1 routed destination 00000800:00000200:0.0:1616513093.251746:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.251749:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.251751:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513093.251752:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.166@tcp1(10.31.141.166@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.251757:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513093.251759:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513093.251762:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513093.251765:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513093.251765:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513093.251768:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513093.251770:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513093.251775:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.251782:0:50875:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513093.251818:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.251825:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.251828:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513093.251828:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513093.251834:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.251840:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02becc0 00000800:00000200:22.0:1616513093.251842:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513093.251844:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513093.251849:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.251852:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513093.251859:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.251863:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2110 00000800:00000200:22.0:1616513093.251867:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.251870:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.254609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513093.254670:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.254680:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513093.254689:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.173@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.254699:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.173@tcp1 of length 224 into portal 26 MB=0x600727bd15b00 00000400:00000200:14.0:1616513093.254707:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.173@tcp1 of length 224/224 into md 0xf97ce8d9 [1] + 0 00000400:00000200:14.0:1616513093.254712:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faeb28 00000400:00000200:14.0:1616513093.254714:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.254717:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513093.254721:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faeb28 00000800:00000200:14.0:1616513093.254725:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.254725:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.254728:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.254732:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.254735:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.254736:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513093.254745:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.173@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.254754:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.173@tcp1 of length 224 into portal 12 MB=0x600727bd15b80 00000400:00000200:2.0:1616513093.254760:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.173@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 6496 00000400:00000200:2.0:1616513093.254766:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.254769:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513093.254777:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.254780:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.254782:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.254786:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689341563853568 00010000:00000200:0.0:1616513093.254797:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1689341563853568/t0(0) o400->92d32fc3-9549-9f26-c28a-b395e2540e7d@10.31.140.173@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.254811:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689341563853568, offset 192 00000400:00000200:0.0:1616513093.254817:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.173@tcp1 00000400:00000200:0.0:1616513093.254825:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.173@tcp1 routed destination 00000400:00000200:0.0:1616513093.254840:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.173@tcp1(10.31.140.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513093.254844:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689341563853696 00000800:00000200:0.0:1616513093.254846:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.254852:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.254855:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:26.0:1616513093.254856:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6c00 x1689341563853696/t0(0) o400->280c8cc6-0841-3fda-cfdb-b5abcd5239d1@10.31.140.173@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513093.254857:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.254862:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.254867:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.254868:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513093.254871:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689341563853696, offset 192 00000400:00000200:26.0:1616513093.254877:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.173@tcp1 00000400:00000200:26.0:1616513093.254884:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.173@tcp1 routed destination 00000400:00000200:26.0:1616513093.254899:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.173@tcp1(10.31.140.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.254905:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.254911:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.254914:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.254916:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513093.254920:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513093.254921:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.254925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513093.254927:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.254930:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513093.254936:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513093.254937:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513093.254940:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513093.254942:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf188 00000800:00000200:16.0:1616513093.254945:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513093.254947:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.254950:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513093.254951:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513093.254956:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.254960:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae7f8 00000800:00000200:18.0:1616513093.254964:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.266255:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513093.266312:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.266321:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513093.266330:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.205@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.266340:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.205@tcp1 of length 224 into portal 26 MB=0x5ffe0ff53f600 00000400:00000200:2.0:1616513093.266346:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.205@tcp1 of length 224/224 into md 0xf97ce909 [1] + 0 00000400:00000200:2.0:1616513093.266351:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae990 00000400:00000200:2.0:1616513093.266354:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.266357:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513093.266360:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae990 00000800:00000200:2.0:1616513093.266363:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.266366:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.266366:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.266371:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.266400:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513093.266425:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716705003008 00010000:00000200:0.0:1616513093.266453:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688716705003008/t0(0) o400->996a559d-d4c2-d711-3692-68cf2ea27e21@10.31.142.205@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513093.266454:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.266463:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:0.0:1616513093.266468:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688716705003008, offset 192 00000400:00000200:14.0:1616513093.266472:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.205@tcp1 : PUT - for me 00000400:00000200:0.0:1616513093.266474:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.205@tcp1 00000400:00000200:14.0:1616513093.266481:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.205@tcp1 of length 224 into portal 12 MB=0x5ffe0ff53f740 00000400:00000200:0.0:1616513093.266482:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.205@tcp1 routed destination 00000400:00000200:14.0:1616513093.266488:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.205@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 6720 00000400:00000200:14.0:1616513093.266493:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.266496:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513093.266497:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.205@tcp1(10.31.142.205@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513093.266503:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.266506:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.266507:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513093.266507:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513093.266512:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513093.266513:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.266516:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.266518:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.266523:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.266529:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.266529:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513093.266567:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716705003328 00010000:00000200:26.0:1616513093.266579:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06c00 x1688716705003328/t0(0) o400->9d993849-c088-eed4-f90c-7b9e79556204@10.31.142.205@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513093.266583:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.266589:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513093.266592:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:26.0:1616513093.266593:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688716705003328, offset 192 00000400:00000200:12.0:1616513093.266599:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513093.266600:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.205@tcp1 00000400:00000200:12.0:1616513093.266604:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be5d8 00000400:00000200:26.0:1616513093.266607:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.205@tcp1 routed destination 00000800:00000200:12.0:1616513093.266608:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513093.266611:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513093.266621:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.205@tcp1(10.31.142.205@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.266628:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.266633:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.266636:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.266640:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513093.266645:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.266649:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.266659:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.266665:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.266667:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513093.266670:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513093.266676:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.266680:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af8f8 00000800:00000200:22.0:1616513093.266683:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.279273:0:50878:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513093.279337:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.279346:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513093.279355:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.76@tcp1 : PUT - for me 00000400:00000200:18.0:1616513093.279364:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.76@tcp1 of length 224 into portal 26 MB=0x5ffdd880f03c0 00000400:00000200:18.0:1616513093.279373:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.76@tcp1 of length 224/224 into md 0xf97ce939 [1] + 0 00000400:00000200:18.0:1616513093.279378:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf5c8 00000400:00000200:18.0:1616513093.279380:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.279383:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513093.279387:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf5c8 00000800:00000200:18.0:1616513093.279391:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.279391:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.279394:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.279398:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.279401:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.279402:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513093.279411:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.76@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.279420:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.76@tcp1 of length 224 into portal 12 MB=0x5ffdd880f0440 00000400:00000200:14.0:1616513093.279426:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.76@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 6944 00000400:00000200:14.0:1616513093.279432:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.279449:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513093.279457:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.279460:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.279462:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.279464:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701819093952 00010000:00000200:0.0:1616513093.279476:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688701819093952/t0(0) o400->da915f78-0538-b6e0-a1ed-5095b6ef9bc9@10.31.142.76@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.279490:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701819093952, offset 192 00000400:00000200:0.0:1616513093.279497:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.76@tcp1 00000400:00000200:0.0:1616513093.279504:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.76@tcp1 routed destination 00000400:00000200:0.0:1616513093.279519:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.76@tcp1(10.31.142.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.279526:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.279531:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:26.0:1616513093.279532:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701819094080 00000800:00000200:0.0:1616513093.279534:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.279536:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.279541:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:26.0:1616513093.279545:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05100 x1688701819094080/t0(0) o400->9e7e17ab-d4f8-ab96-4a0f-3fa16c42bc32@10.31.142.76@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513093.279548:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.279548:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513093.279559:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701819094080, offset 192 00000400:00000200:26.0:1616513093.279565:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.76@tcp1 00000400:00000200:26.0:1616513093.279573:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.76@tcp1 routed destination 00000400:00000200:26.0:1616513093.279587:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.76@tcp1(10.31.142.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.279594:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.279600:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.279603:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.279605:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513093.279607:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513093.279609:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513093.279614:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513093.279614:0:50878:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513093.279616:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513093.279623:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513093.279629:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513093.279631:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be770 00000800:00000200:28.0:1616513093.279634:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513093.279636:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513093.279637:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513093.279639:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513093.279640:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513093.279643:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513093.279645:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513093.279646:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513093.279648:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513093.279649:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513093.279650:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeee0 00000800:00000200:28.0:1616513093.279651:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513093.279654:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.318252:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513093.318308:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.318317:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513093.318328:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.48@tcp1 : PUT - for me 00000400:00000200:20.0:1616513093.318338:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.48@tcp1 of length 224 into portal 26 MB=0x602bfe947a800 00000400:00000200:20.0:1616513093.318345:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.48@tcp1 of length 224/224 into md 0xf97ce949 [1] + 0 00000400:00000200:20.0:1616513093.318350:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faec38 00000400:00000200:20.0:1616513093.318353:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513093.318355:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513093.318360:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faec38 00000800:00000200:28.0:1616513093.318363:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.318364:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.318367:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.318369:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513093.318400:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513093.318428:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691873136060416 00000800:00000200:16.0:1616513093.318455:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:0.0:1616513093.318455:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1691873136060416/t0(0) o400->68f19b65-b371-ed66-8724-86778a1503a2@10.31.142.48@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513093.318465:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513093.318470:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691873136060416, offset 192 00000400:00000200:16.0:1616513093.318474:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.48@tcp1 : PUT - for me 00000400:00000200:0.0:1616513093.318477:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.48@tcp1 00000400:00000200:0.0:1616513093.318485:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.48@tcp1 routed destination 00000400:00000200:0.0:1616513093.318501:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.48@tcp1(10.31.142.48@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513093.318508:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.48@tcp1 of length 224 into portal 12 MB=0x602bfe947a900 00000800:00000200:0.0:1616513093.318508:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513093.318510:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513093.318515:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513093.318516:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.48@tcp1 of length 224/224 into md 0x865be291 [1] + 45232 00000800:00000200:0.0:1616513093.318518:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513093.318521:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513093.318522:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:16.0:1616513093.318525:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513093.318527:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:11.2:1616513093.318532:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.318533:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513093.318535:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513093.318536:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513093.318538:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513093.318541:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.318545:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513093.318546:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.318547:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513093.318553:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:8.0:1616513093.318555:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691873136060672 00000400:00000200:16.0:1616513093.318558:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf320 00000800:00000200:16.0:1616513093.318561:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513093.318568:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6727080 x1691873136060672/t0(0) o400->3b36fe1a-66f7-6db0-3cd6-9043901d528d@10.31.142.48@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513093.318582:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691873136060672, offset 192 00000400:00000200:8.0:1616513093.318589:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.48@tcp1 00000400:00000200:8.0:1616513093.318596:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.48@tcp1 routed destination 00000400:00000200:8.0:1616513093.318611:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.48@tcp1(10.31.142.48@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513093.318618:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513093.318624:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.318627:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513093.318629:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513093.318635:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.318641:0:50882:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.318697:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.318705:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513093.318708:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513093.318714:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513093.318718:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2880 00000800:00000200:2.0:1616513093.318724:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.318727:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.330596:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513093.330616:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513093.330650:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513093.330665:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513093.330674:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.47@tcp1 : PUT - for me 00000800:00000200:18.0:1616513093.330676:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513093.330684:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.47@tcp1 of length 224 into portal 26 MB=0x5ffdbf5ccbf80 00000800:00000200:18.0:1616513093.330686:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513093.330690:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.47@tcp1 of length 224/224 into md 0xf97ce969 [1] + 0 00000400:00000200:18.0:1616513093.330695:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.47@tcp1 : PUT - for me 00000400:00000200:12.0:1616513093.330695:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf7e8 00000400:00000200:12.0:1616513093.330698:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513093.330701:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513093.330704:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.47@tcp1 of length 224 into portal 12 MB=0x5ffdbf5ccc100 00000400:00000200:12.0:1616513093.330705:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf7e8 00000800:00000200:12.0:1616513093.330708:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.330709:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513093.330711:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.47@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 7168 00000800:00000200:12.0:1616513093.330711:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513093.330713:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513093.330716:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.330718:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513093.330726:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.330729:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.330731:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.330733:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513093.330753:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695070310656 00010000:00000200:26.0:1616513093.330766:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd704380 x1688695070310656/t0(0) o400->d0ee3a17-2816-7cd1-8bda-b06dfabe0fc8@10.31.143.47@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513093.330772:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695070310272 00000100:00000200:26.0:1616513093.330781:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695070310656, offset 192 00010000:00000200:0.0:1616513093.330783:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688695070310272/t0(0) o400->f09d73cb-5570-06ab-6c95-ff4118a348e8@10.31.143.47@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513093.330787:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.47@tcp1 00000400:00000200:26.0:1616513093.330794:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.47@tcp1 routed destination 00000100:00000200:0.0:1616513093.330798:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695070310272, offset 192 00000400:00000200:0.0:1616513093.330804:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.47@tcp1 00000400:00000200:26.0:1616513093.330809:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.47@tcp1(10.31.143.47@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.330816:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513093.330819:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.47@tcp1 routed destination 00000800:00000200:26.0:1616513093.330822:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.330825:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513093.330827:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513093.330832:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513093.330834:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.47@tcp1(10.31.143.47@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513093.330837:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.330841:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.330847:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.330850:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.330852:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.330857:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.330862:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.330864:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513093.330890:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.330898:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.330900:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.330907:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.330911:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afd38 00000800:00000200:20.0:1616513093.330915:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.330918:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513093.330919:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.330926:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513093.330929:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513093.330936:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513093.330941:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bea18 00000800:00000200:2.0:1616513093.330944:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.330947:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.333984:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513093.334039:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.334048:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513093.334057:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.183@tcp1 : PUT - for me 00000400:00000200:18.0:1616513093.334067:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.183@tcp1 of length 224 into portal 26 MB=0x5ffd4a5cc3ec0 00000400:00000200:18.0:1616513093.334074:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.183@tcp1 of length 224/224 into md 0xf97ce989 [1] + 0 00000400:00000200:18.0:1616513093.334079:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fafcb0 00000400:00000200:18.0:1616513093.334082:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.334084:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513093.334089:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafcb0 00000800:00000200:18.0:1616513093.334091:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.334094:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513093.334094:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.334101:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513093.334105:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513093.334106:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513093.334114:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.183@tcp1 : PUT - for me 00000400:00000200:22.0:1616513093.334123:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.183@tcp1 of length 224 into portal 12 MB=0x5ffd4a5cc3f80 00000400:00000200:22.0:1616513093.334129:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.183@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 7392 00000400:00000200:22.0:1616513093.334135:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.334137:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513093.334144:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.334147:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.334149:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.334151:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663663328960 00010000:00000200:0.0:1616513093.334162:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688663663328960/t0(0) o400->6f644ab1-0316-27f2-fedf-674768fd8307@10.31.138.183@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.334175:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663663328960, offset 192 00000400:00000200:0.0:1616513093.334181:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.183@tcp1 00000400:00000200:0.0:1616513093.334188:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.183@tcp1 routed destination 00000400:00000200:0.0:1616513093.334202:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.183@tcp1(10.31.138.183@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513093.334208:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663663329152 00000800:00000200:0.0:1616513093.334208:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.334213:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.334216:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.334218:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:26.0:1616513093.334221:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a28d80 x1688663663329152/t0(0) o400->81452e03-916a-b7e9-893c-1e033b9b7d8c@10.31.138.183@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513093.334222:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.334229:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.334230:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513093.334235:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663663329152, offset 192 00000400:00000200:26.0:1616513093.334250:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.183@tcp1 00000400:00000200:26.0:1616513093.334258:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.183@tcp1 routed destination 00000400:00000200:26.0:1616513093.334272:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.183@tcp1(10.31.138.183@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.334279:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513093.334284:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513093.334285:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.334287:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.334290:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513093.334291:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513093.334293:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513093.334294:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.334300:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513093.334302:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513093.334307:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf870 00000800:00000200:28.0:1616513093.334310:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.334313:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513093.334360:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.334368:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.334371:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513093.334373:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513093.334379:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.334383:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af760 00000800:00000200:14.0:1616513093.334387:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:31.0:1616513093.386664:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928512, portal 4 00000100:00000200:18.0:1616513093.386665:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928528, portal 4 00000100:00000200:31.0:1616513093.386670:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928512, offset 0 00000100:00000200:18.0:1616513093.386672:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928528, offset 0 00000400:00000200:31.0:1616513093.386678:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:18.0:1616513093.386679:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:31.0:1616513093.386685:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:18.0:1616513093.386686:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:31.0:1616513093.386700:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513093.386708:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:31.0:1616513093.386715:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513093.386715:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513093.386719:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:31.0:1616513093.386721:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:18.0:1616513093.386723:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:31.0:1616513093.386728:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:18.0:1616513093.386730:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513093.386733:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:18.0:1616513093.386736:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:5.2:1616513093.386736:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:18.0:1616513093.386741:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (24)-- 00000800:00000200:20.0:1616513093.386793:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)++ 00000800:00000200:20.0:1616513093.386800:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (24)-- 00000400:00000200:20.0:1616513093.386803:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.386810:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513093.386817:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd604380 x1693067725928512/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513093.386829:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2660 00000800:00000200:20.0:1616513093.386834:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:20.0:1616513093.386836:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513093.386840:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513093.386841:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.386846:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:16.0:1616513093.386847:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:20.0:1616513093.386851:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc7e02880 x1693067725928528/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513093.386861:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc220 00000800:00000200:20.0:1616513093.386863:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513093.386964:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:12.0:1616513093.387018:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513093.387028:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.230@o2ib1 00000800:00000200:16.0:1616513093.387029:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:12.0:1616513093.387037:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:16.0:1616513093.387038:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513093.387039:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513093.387045:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627840 00000400:00000200:16.0:1616513093.387047:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513093.387053:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866db971 [1] + 192 00000400:00000200:16.0:1616513093.387055:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627850 00000400:00000200:12.0:1616513093.387058:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513093.387062:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866db981 [1] + 192 00000400:00000200:16.0:1616513093.387067:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513093.387070:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd604380 x1693067725928512/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513093.387085:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000100:00000200:16.0:1616513093.387087:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc7e02880 x1693067725928528/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513093.387088:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:12.0:1616513093.387090:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513093.387101:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:31.0:1616513093.387104:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd604380 x1693067725928512/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513093.387104:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513093.387106:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:31.0:1616513093.387118:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd604380 x1693067725928512/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513093.387129:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000100:00000200:18.0:1616513093.387134:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc7e02880 x1693067725928528/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:18.0:1616513093.387147:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc7e02880 x1693067725928528/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513093.387158:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc550 00000800:00000200:11.2:1616513093.436993:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513093.437050:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.437060:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513093.437070:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.230@tcp1 : PUT - for me 00000400:00000200:28.0:1616513093.437080:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.230@tcp1 of length 224 into portal 26 MB=0x5ffdf143e25c0 00000400:00000200:28.0:1616513093.437087:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.230@tcp1 of length 224/224 into md 0xf97ce9a9 [1] + 0 00000400:00000200:28.0:1616513093.437092:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae660 00000400:00000200:28.0:1616513093.437095:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.437098:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513093.437103:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae660 00000800:00000200:20.0:1616513093.437105:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.437107:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.437109:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.437111:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.437171:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708465960384 00010000:00000200:0.0:1616513093.437185:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688708465960384/t0(0) o400->87517747-70fe-2a9d-2ff6-2d2bfbb0bf49@10.31.141.230@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.437202:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688708465960384, offset 192 00000400:00000200:0.0:1616513093.437210:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.230@tcp1 00000400:00000200:0.0:1616513093.437218:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.230@tcp1 routed destination 00000400:00000200:0.0:1616513093.437234:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.230@tcp1(10.31.141.230@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513093.437235:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.437241:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.437247:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.437250:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.437252:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513093.437259:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513093.437267:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513093.437289:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.437298:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513093.437307:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.230@tcp1 : PUT - for me 00000400:00000200:12.0:1616513093.437316:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.230@tcp1 of length 224 into portal 12 MB=0x5ffdf143e26c0 00000400:00000200:12.0:1616513093.437322:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.230@tcp1 of length 224/224 into md 0x865be291 [1] + 45456 00000400:00000200:12.0:1616513093.437328:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513093.437330:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513093.437338:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513093.437341:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513093.437344:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:12.0:1616513093.437351:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513093.437352:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513093.437354:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513093.437355:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513093.437359:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708465960640 00000400:00000200:16.0:1616513093.437362:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513093.437367:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beee0 00000800:00000200:16.0:1616513093.437371:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513093.437372:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d200480 x1688708465960640/t0(0) o400->0259a8c9-5adb-bf6d-d724-7946afd221c3@10.31.141.230@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513093.437386:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688708465960640, offset 192 00000400:00000200:8.0:1616513093.437392:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.230@tcp1 00000400:00000200:8.0:1616513093.437399:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.230@tcp1 routed destination 00000400:00000200:8.0:1616513093.437414:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.230@tcp1(10.31.141.230@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513093.437421:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513093.437427:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.437430:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513093.437432:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513093.437443:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.437452:0:50895:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.437510:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.437518:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.437521:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513093.437528:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.437532:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2990 00000800:00000200:22.0:1616513093.437536:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.437539:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.483004:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513093.483063:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.483074:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513093.483084:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.39@tcp1 : PUT - for me 00000400:00000200:18.0:1616513093.483095:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.39@tcp1 of length 224 into portal 26 MB=0x604a297822800 00000400:00000200:18.0:1616513093.483102:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.39@tcp1 of length 224/224 into md 0xf97ce9c9 [1] + 0 00000400:00000200:18.0:1616513093.483107:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faed48 00000400:00000200:18.0:1616513093.483110:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.483113:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513093.483118:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faed48 00000800:00000200:14.0:1616513093.483118:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.483122:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513093.483125:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513093.483127:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.483130:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.483130:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513093.483139:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.39@tcp1 : PUT - for me 00000100:00000200:0.0:1616513093.483145:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946233366528 00000400:00000200:14.0:1616513093.483148:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.39@tcp1 of length 224 into portal 12 MB=0x604a297822ac0 00000400:00000200:14.0:1616513093.483155:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.39@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 7616 00010000:00000200:0.0:1616513093.483159:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1693946233366528/t0(0) o400->f267e3ee-058d-2b3c-8177-ac8ff7d8f1ff@10.31.141.39@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513093.483160:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.483163:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513093.483176:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.483176:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946233366528, offset 192 00000800:00000200:14.0:1616513093.483179:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.483181:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513093.483182:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.39@tcp1 00000400:00000200:0.0:1616513093.483191:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.39@tcp1 routed destination 00000400:00000200:0.0:1616513093.483206:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.39@tcp1(10.31.141.39@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.483213:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.483220:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.483222:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.483225:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.483231:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.483238:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.483239:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513093.483243:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946233367232 00010000:00000200:26.0:1616513093.483257:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01680 x1693946233367232/t0(0) o400->3c4b78fb-f921-cb2f-c062-39a9cca0de60@10.31.141.39@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513093.483272:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946233367232, offset 192 00000400:00000200:26.0:1616513093.483279:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.39@tcp1 00000400:00000200:26.0:1616513093.483286:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.39@tcp1 routed destination 00000800:00000200:28.0:1616513093.483293:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.483300:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513093.483301:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.39@tcp1(10.31.141.39@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513093.483303:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513093.483308:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513093.483310:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513093.483314:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513093.483316:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beee0 00000800:00000200:26.0:1616513093.483317:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513093.483319:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513093.483319:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513093.483322:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513093.483324:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.483329:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.483385:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.483392:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513093.483395:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513093.483396:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513093.483402:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513093.483406:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae000 00000800:00000200:2.0:1616513093.483410:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.548371:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513093.548428:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.548452:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513093.548462:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.66@tcp1 : PUT - for me 00000400:00000200:20.0:1616513093.548473:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.66@tcp1 of length 224 into portal 26 MB=0x601f2e741ec00 00000400:00000200:20.0:1616513093.548480:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.66@tcp1 of length 224/224 into md 0xf97ce9d9 [1] + 0 00000800:00000200:16.0:1616513093.548483:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513093.548485:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faebb0 00000400:00000200:20.0:1616513093.548488:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513093.548491:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:16.0:1616513093.548494:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513093.548497:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faebb0 00000800:00000200:20.0:1616513093.548501:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:16.0:1616513093.548503:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.66@tcp1 : PUT - for me 00000800:00000200:20.0:1616513093.548504:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513093.548506:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513093.548509:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513093.548513:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.66@tcp1 of length 224 into portal 12 MB=0x601f2e741ec80 00000400:00000200:16.0:1616513093.548519:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.66@tcp1 of length 224/224 into md 0x865be291 [1] + 45680 00000400:00000200:16.0:1616513093.548526:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.548529:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513093.548537:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.548540:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.548542:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513093.548560:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690992633834624 00010000:00000200:8.0:1616513093.548573:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6723a80 x1690992633834624/t0(0) o400->b9e6dc2e-9163-8e54-2062-214a205e3a1f@10.31.141.66@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513093.548579:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690992633834496 00000100:00000200:8.0:1616513093.548587:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690992633834624, offset 192 00000400:00000200:8.0:1616513093.548592:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.66@tcp1 00010000:00000200:0.0:1616513093.548592:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1690992633834496/t0(0) o400->917a0c4c-a4b4-2707-ab03-5495bece49f9@10.31.141.66@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513093.548599:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.66@tcp1 routed destination 00000100:00000200:0.0:1616513093.548607:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690992633834496, offset 192 00000400:00000200:8.0:1616513093.548613:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.66@tcp1(10.31.141.66@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513093.548613:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.66@tcp1 00000800:00000200:8.0:1616513093.548620:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513093.548622:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.66@tcp1 routed destination 00000800:00000200:8.0:1616513093.548633:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.548635:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513093.548637:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513093.548640:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.66@tcp1(10.31.141.66@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513093.548642:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.548647:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.548647:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.548654:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.548657:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.548659:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.548665:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.548672:0:50906:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.548672:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513093.548703:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.548710:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513093.548713:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513093.548714:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513093.548720:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513093.548724:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3430 00000800:00000200:12.0:1616513093.548728:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513093.548732:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.548739:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.548742:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513093.548749:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.548755:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bea18 00000800:00000200:22.0:1616513093.548758:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.548761:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.552923:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513093.552978:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.552990:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513093.552999:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.241@tcp1 : PUT - for me 00000400:00000200:18.0:1616513093.553008:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.241@tcp1 of length 224 into portal 26 MB=0x5ffe39600f6c0 00000400:00000200:18.0:1616513093.553015:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.241@tcp1 of length 224/224 into md 0xf97ce9f9 [1] + 0 00000400:00000200:18.0:1616513093.553019:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae5d8 00000400:00000200:18.0:1616513093.553022:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.553025:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513093.553029:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae5d8 00000800:00000200:18.0:1616513093.553032:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.553035:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.553035:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513093.553042:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.553045:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.553046:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513093.553055:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.241@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.553063:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.241@tcp1 of length 224 into portal 12 MB=0x5ffe39600f980 00000400:00000200:2.0:1616513093.553070:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.241@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 7840 00000400:00000200:2.0:1616513093.553075:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.553078:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513093.553086:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.553089:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.553091:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.553093:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727822857920 00010000:00000200:0.0:1616513093.553104:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688727822857920/t0(0) o400->13b8b99c-1e8b-8f29-fb22-54925458153e@10.31.141.241@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.553118:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688727822857920, offset 192 00000400:00000200:0.0:1616513093.553125:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.241@tcp1 00000400:00000200:0.0:1616513093.553132:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.241@tcp1 routed destination 00000400:00000200:0.0:1616513093.553147:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.241@tcp1(10.31.141.241@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513093.553150:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727822858624 00000800:00000200:0.0:1616513093.553154:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.553160:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.553162:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:26.0:1616513093.553163:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5100 x1688727822858624/t0(0) o400->f644fd8e-6b2b-20b8-a056-d4e4fd3da97e@10.31.141.241@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513093.553164:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.553169:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.553176:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513093.553177:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688727822858624, offset 192 00000400:00000200:26.0:1616513093.553184:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.241@tcp1 00000400:00000200:26.0:1616513093.553191:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.241@tcp1 routed destination 00000400:00000200:26.0:1616513093.553206:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.241@tcp1(10.31.141.241@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.553212:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513093.553217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513093.553218:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.553223:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.553225:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513093.553230:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.553240:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513093.553276:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.553283:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513093.553286:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513093.553292:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513093.553295:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513093.553298:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be770 00000800:00000200:28.0:1616513093.553301:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513093.553302:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513093.553304:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513093.553305:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513093.553306:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513093.553311:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.553315:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae5d8 00000800:00000200:14.0:1616513093.553319:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513093.578517:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928544, portal 4 00000100:00000200:22.0:1616513093.578523:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928544, offset 0 00000400:00000200:22.0:1616513093.578530:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:22.0:1616513093.578536:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:22.0:1616513093.578549:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513093.578556:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:22.0:1616513093.578562:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513093.578565:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:22.0:1616513093.578567:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:22.0:1616513093.578573:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513093.578621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.578641:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513093.578647:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513093.578650:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513093.578658:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513093.578666:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf81200 x1693067725928544/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513093.578679:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b870 00000800:00000200:1.0:1616513093.578685:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513093.578688:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513093.578794:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513093.578805:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.578811:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513093.578815:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513093.578820:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513093.578829:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627860 00000400:00000200:1.0:1616513093.578837:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700035 [1] + 192 00000400:00000200:1.0:1616513093.578842:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513093.578847:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf81200 x1693067725928544/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513093.578862:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.578865:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513093.578867:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:22.0:1616513093.578871:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf81200 x1693067725928544/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513093.578884:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf81200 x1693067725928544/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513093.578895:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b980 00000800:00000200:7.2:1616513093.581729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513093.581787:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.581797:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513093.581806:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.26@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.581815:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.26@tcp1 of length 224 into portal 26 MB=0x604a2d9fb7f00 00000400:00000200:2.0:1616513093.581822:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.26@tcp1 of length 224/224 into md 0xf97cea09 [1] + 0 00000400:00000200:2.0:1616513093.581826:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf078 00000400:00000200:2.0:1616513093.581829:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.581832:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513093.581838:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf078 00000800:00000200:22.0:1616513093.581841:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.581841:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.581843:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.581845:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.581902:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947348614912 00010000:00000200:0.0:1616513093.581913:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1693947348614912/t0(0) o400->4e3993a5-b0e8-5bd2-c967-52724bc6e2ca@10.31.141.26@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.581928:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693947348614912, offset 192 00000400:00000200:0.0:1616513093.581934:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.26@tcp1 00000400:00000200:0.0:1616513093.581941:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.26@tcp1 routed destination 00000400:00000200:0.0:1616513093.581956:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.26@tcp1(10.31.141.26@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.581963:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.581968:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.581973:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.581975:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.581979:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.581986:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.582028:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.582084:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.582091:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.582094:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.582100:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.582106:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf188 00000800:00000200:20.0:1616513093.582109:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.582112:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513093.582126:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513093.582139:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513093.582150:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513093.582159:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.26@tcp1 : PUT - for me 00000400:00000200:12.0:1616513093.582167:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.26@tcp1 of length 224 into portal 12 MB=0x604a2d9fb8a40 00000400:00000200:12.0:1616513093.582174:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.26@tcp1 of length 224/224 into md 0x865be291 [1] + 45904 00000400:00000200:12.0:1616513093.582179:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:28.0:1616513093.582181:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513093.582181:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513093.582188:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513093.582191:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513093.582193:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513093.582205:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693947348617792 00010000:00000200:8.0:1616513093.582215:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d207500 x1693947348617792/t0(0) o400->25493809-03e1-ea02-c339-21022cfedf15@10.31.141.26@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513093.582228:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693947348617792, offset 192 00000400:00000200:8.0:1616513093.582233:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.26@tcp1 00000400:00000200:8.0:1616513093.582240:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.26@tcp1 routed destination 00000400:00000200:8.0:1616513093.582253:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.26@tcp1(10.31.141.26@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513093.582259:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513093.582264:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.582267:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513093.582269:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513093.582273:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.582278:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.582332:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.582339:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.582342:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513093.582348:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.582352:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a08 00000800:00000200:18.0:1616513093.582357:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.582359:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.587140:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513093.587200:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.587209:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513093.587219:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.129@tcp1 : PUT - for me 00000400:00000200:22.0:1616513093.587227:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.129@tcp1 of length 224 into portal 26 MB=0x5ffd8ff955a40 00000400:00000200:22.0:1616513093.587233:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.129@tcp1 of length 224/224 into md 0xf97cea29 [1] + 0 00000400:00000200:22.0:1616513093.587238:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf320 00000400:00000200:22.0:1616513093.587241:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.587244:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513093.587248:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf320 00000800:00000200:22.0:1616513093.587251:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.587253:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.587253:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513093.587258:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.587311:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688682349550144 00010000:00000200:0.0:1616513093.587322:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688682349550144/t0(0) o400->a6302840-e34d-63a3-77ea-0a75cbcbf62b@10.31.132.129@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.587336:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688682349550144, offset 192 00000400:00000200:0.0:1616513093.587342:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.129@tcp1 00000400:00000200:0.0:1616513093.587349:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.129@tcp1 routed destination 00000400:00000200:0.0:1616513093.587364:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.129@tcp1(10.31.132.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.587371:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.587377:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.587379:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.587382:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.587386:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.587393:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.587434:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.587488:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.587495:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.587497:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.587504:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.587510:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be2a8 00000800:00000200:20.0:1616513093.587513:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.587516:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513093.587976:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513093.588028:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513093.588037:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513093.588046:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.129@tcp1 : PUT - for me 00000400:00000200:12.0:1616513093.588055:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.129@tcp1 of length 224 into portal 12 MB=0x5ffd8ff955d00 00000400:00000200:12.0:1616513093.588061:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.129@tcp1 of length 224/224 into md 0x865be291 [1] + 46128 00000400:00000200:12.0:1616513093.588066:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513093.588069:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513093.588076:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.588079:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513093.588081:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.588083:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513093.588088:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688682349550848 00010000:00000200:8.0:1616513093.588096:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d200d80 x1688682349550848/t0(0) o400->fd20ecc0-0a5f-1c97-7614-81d8835c64a4@10.31.132.129@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513093.588107:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688682349550848, offset 192 00000400:00000200:8.0:1616513093.588112:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.129@tcp1 00000400:00000200:8.0:1616513093.588117:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.129@tcp1 routed destination 00000400:00000200:8.0:1616513093.588130:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.129@tcp1(10.31.132.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513093.588136:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513093.588140:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.588142:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513093.588144:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513093.588147:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.588154:0:50910:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.588208:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.588215:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.588218:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513093.588225:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.588229:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2e58 00000800:00000200:18.0:1616513093.588233:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.588236:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.590204:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513093.590259:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.590268:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513093.590276:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.198@tcp1 : PUT - for me 00000400:00000200:20.0:1616513093.590285:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.198@tcp1 of length 224 into portal 26 MB=0x600af40027d40 00000400:00000200:20.0:1616513093.590293:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.198@tcp1 of length 224/224 into md 0xf97cea49 [1] + 0 00000400:00000200:20.0:1616513093.590298:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faecc0 00000400:00000200:20.0:1616513093.590301:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513093.590304:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513093.590308:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faecc0 00000800:00000200:20.0:1616513093.590311:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.590312:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.590314:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513093.590319:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513093.590322:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.590323:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513093.590331:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.198@tcp1 : PUT - for me 00000400:00000200:28.0:1616513093.590342:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.198@tcp1 of length 224 into portal 12 MB=0x600af40027e40 00000400:00000200:28.0:1616513093.590348:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.198@tcp1 of length 224/224 into md 0x865be291 [1] + 46352 00000400:00000200:28.0:1616513093.590353:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.590356:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513093.590363:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.590366:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.590368:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.590384:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689602553445696 00000100:00000200:8.0:1616513093.590388:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689602553445952 00010000:00000200:8.0:1616513093.590392:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbeede300 x1689602553445952/t0(0) o400->8c02b3ac-a21e-df50-a948-d6cdcbb08c8c@10.31.140.198@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:0.0:1616513093.590395:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1689602553445696/t0(0) o400->8c615476-e25d-8bc8-f09a-d25c10d0cf70@10.31.140.198@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513093.590402:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689602553445952, offset 192 00000400:00000200:8.0:1616513093.590406:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.198@tcp1 00000100:00000200:0.0:1616513093.590409:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689602553445696, offset 192 00000400:00000200:8.0:1616513093.590411:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.198@tcp1 routed destination 00000400:00000200:0.0:1616513093.590416:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.198@tcp1 00000400:00000200:8.0:1616513093.590423:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.198@tcp1(10.31.140.198@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513093.590428:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513093.590431:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.198@tcp1 routed destination 00000800:00000200:8.0:1616513093.590452:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.590454:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513093.590455:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513093.590458:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513093.590460:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.198@tcp1(10.31.140.198@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513093.590463:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.590467:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.590472:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.590475:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.590477:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.590482:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.590489:0:50910:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.590489:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513093.590517:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.590524:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.590527:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513093.590528:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513093.590533:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513093.590537:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2908 00000800:00000200:16.0:1616513093.590541:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513093.590545:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.590553:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513093.590556:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513093.590562:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513093.590568:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beaa0 00000800:00000200:2.0:1616513093.590571:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.590574:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.590678:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513093.590733:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.590743:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513093.590752:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.87@tcp1 : PUT - for me 00000400:00000200:18.0:1616513093.590762:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.87@tcp1 of length 224 into portal 26 MB=0x6014fb74ad000 00000400:00000200:18.0:1616513093.590768:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.87@tcp1 of length 224/224 into md 0xf97cea69 [1] + 0 00000400:00000200:18.0:1616513093.590773:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf430 00000400:00000200:18.0:1616513093.590778:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.590780:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513093.590785:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf430 00000800:00000200:18.0:1616513093.590788:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.590788:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.590791:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.590793:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.590799:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690291749441536 00010000:00000200:0.0:1616513093.590804:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1690291749441536/t0(0) o400->a31647ba-403b-ed89-e4e0-ee49bd2e0b93@10.31.140.87@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.590816:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690291749441536, offset 192 00000400:00000200:0.0:1616513093.590821:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.87@tcp1 00000400:00000200:0.0:1616513093.590827:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.87@tcp1 routed destination 00000400:00000200:0.0:1616513093.590841:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.87@tcp1(10.31.140.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.590847:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.590851:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.590854:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.590855:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.590858:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.590863:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.590864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.590874:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.590878:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513093.590881:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513093.590884:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513093.590886:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513093.590891:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf5c8 00000800:00000200:28.0:1616513093.590894:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513093.590982:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513093.590992:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.590998:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513093.591001:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513093.591006:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.87@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.591014:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.87@tcp1 of length 224 into portal 12 MB=0x6014fb74add40 00000400:00000200:2.0:1616513093.591020:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.87@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 8064 00000400:00000200:2.0:1616513093.591026:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.591028:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513093.591035:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.591038:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.591039:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513093.591100:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690291749444928 00010000:00000200:26.0:1616513093.591112:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0000 x1690291749444928/t0(0) o400->7c5ba746-8e4b-06b0-d914-c482edc9ff2f@10.31.140.87@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513093.591126:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690291749444928, offset 192 00000400:00000200:26.0:1616513093.591133:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.87@tcp1 00000400:00000200:26.0:1616513093.591140:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.87@tcp1 routed destination 00000400:00000200:26.0:1616513093.591155:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.87@tcp1(10.31.140.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.591162:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.591167:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.591170:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.591172:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513093.591177:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.591182:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.591192:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.591198:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513093.591200:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513093.591201:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513093.591207:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.591211:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeaa0 00000800:00000200:14.0:1616513093.591215:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.600124:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513093.600179:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.600188:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513093.600197:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.147@tcp1 : PUT - for me 00000400:00000200:20.0:1616513093.600206:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.147@tcp1 of length 224 into portal 26 MB=0x60540e8503080 00000400:00000200:20.0:1616513093.600213:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.147@tcp1 of length 224/224 into md 0xf97cea79 [1] + 0 00000400:00000200:20.0:1616513093.600218:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf6d8 00000400:00000200:20.0:1616513093.600220:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513093.600223:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513093.600227:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf6d8 00000800:00000200:20.0:1616513093.600231:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.600233:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513093.600233:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.600240:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513093.600243:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.600243:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513093.600252:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.147@tcp1 : PUT - for me 00000400:00000200:16.0:1616513093.600261:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.147@tcp1 of length 224 into portal 12 MB=0x60540e8503100 00000400:00000200:16.0:1616513093.600267:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.147@tcp1 of length 224/224 into md 0x865be291 [1] + 46576 00000400:00000200:16.0:1616513093.600273:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.600275:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513093.600282:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.600285:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.600287:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.600291:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694626193879168 00000100:00000200:8.0:1616513093.600295:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694626193879296 00010000:00000200:8.0:1616513093.600303:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6720480 x1694626193879296/t0(0) o400->f5292d1d-0d2d-2cef-769a-d3683a040167@10.31.143.147@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:0.0:1616513093.600303:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1694626193879168/t0(0) o400->64eeef89-d2db-436b-b406-9f165c00d683@10.31.143.147@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513093.600314:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694626193879296, offset 192 00000400:00000200:8.0:1616513093.600318:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.147@tcp1 00000100:00000200:0.0:1616513093.600321:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694626193879168, offset 192 00000400:00000200:8.0:1616513093.600325:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.147@tcp1 routed destination 00000400:00000200:0.0:1616513093.600327:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.147@tcp1 00000400:00000200:8.0:1616513093.600339:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.147@tcp1(10.31.143.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513093.600344:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513093.600347:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.147@tcp1 routed destination 00000800:00000200:8.0:1616513093.600354:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.600356:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513093.600358:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513093.600361:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513093.600362:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.147@tcp1(10.31.143.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513093.600366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.600368:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.600374:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.600377:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.600379:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.600384:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.600391:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.600391:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513093.600420:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.600446:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513093.600453:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513093.600456:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513093.600457:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:28.0:1616513093.600462:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513093.600464:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513093.600466:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2aa0 00000400:00000200:22.0:1616513093.600466:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513093.600470:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513093.600475:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.600480:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be6e8 00000800:00000200:22.0:1616513093.600483:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.600486:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.625582:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513093.625640:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.625650:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513093.625659:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.131@tcp1 : PUT - for me 00000400:00000200:16.0:1616513093.625668:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.131@tcp1 of length 224 into portal 26 MB=0x603fc0b789280 00000400:00000200:16.0:1616513093.625675:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.131@tcp1 of length 224/224 into md 0xf97cea99 [1] + 0 00000400:00000200:16.0:1616513093.625680:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fafd38 00000400:00000200:16.0:1616513093.625683:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.625686:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513093.625691:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafd38 00000800:00000200:12.0:1616513093.625693:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.625694:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.625699:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.625701:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.625718:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230919357056 00010000:00000200:0.0:1616513093.625730:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1693230919357056/t0(0) o400->8675b3c0-bde2-90f0-e97d-33b54491ca83@10.31.138.131@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.625745:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230919357056, offset 192 00000400:00000200:0.0:1616513093.625754:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.131@tcp1 00000400:00000200:0.0:1616513093.625762:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.131@tcp1 routed destination 00000400:00000200:0.0:1616513093.625778:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.131@tcp1(10.31.138.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.625785:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.625791:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.625794:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.625796:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.625801:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.625806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.625808:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513093.625860:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.625867:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.625870:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.625876:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.625882:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfcb0 00000800:00000200:20.0:1616513093.625885:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.625888:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.626584:0:6480:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928560, portal 4 00000100:00000200:0.0:1616513093.626588:0:6480:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928560, offset 0 00000400:00000200:0.0:1616513093.626593:0:6480:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:0.0:1616513093.626598:0:6480:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:0.0:1616513093.626610:0:6480:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.626615:0:6480:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:0.0:1616513093.626621:0:6480:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.626623:0:6480:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:0.0:1616513093.626625:0:6480:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:0.0:1616513093.626628:0:6480:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513093.626675:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513093.626735:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513093.626752:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513093.626755:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513093.626761:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513093.626767:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4557cee300 x1693067725928560/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513093.626778:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfb18 00000800:00000200:18.0:1616513093.626781:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513093.626784:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513093.626790:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513093.626797:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:14.0:1616513093.626801:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513093.626804:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627870 00000400:00000200:18.0:1616513093.626810:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cf079 [1] + 192 00000400:00000200:18.0:1616513093.626815:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.626818:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4557cee300 x1693067725928560/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513093.626831:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513093.626833:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513093.626835:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:0.0:1616513093.626838:0:6480:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4557cee300 x1693067725928560/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:0.0:1616513093.626851:0:6480:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4557cee300 x1693067725928560/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513093.626860:0:6480:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beff0 00000800:00000200:7.2:1616513093.627121:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513093.627178:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.627188:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513093.627189:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513093.627197:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.131@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.627206:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.131@tcp1 of length 224 into portal 12 MB=0x603fc0b789300 00000400:00000200:2.0:1616513093.627212:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.131@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 8288 00000400:00000200:2.0:1616513093.627218:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.627221:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513093.627229:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.627232:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.627234:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513093.627294:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230919357184 00010000:00000200:26.0:1616513093.627307:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3f00 x1693230919357184/t0(0) o400->d78e59ac-09d8-4c07-3307-9b2947cd480e@10.31.138.131@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513093.627322:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230919357184, offset 192 00000400:00000200:26.0:1616513093.627328:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.131@tcp1 00000400:00000200:26.0:1616513093.627335:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.131@tcp1 routed destination 00000400:00000200:26.0:1616513093.627350:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.131@tcp1(10.31.138.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.627357:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.627363:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.627365:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.627368:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513093.627372:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.627378:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.627396:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.627403:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.627406:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513093.627408:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513093.627413:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.627416:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afcb0 00000800:00000200:22.0:1616513093.627421:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.631519:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513093.631553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513093.631573:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513093.631582:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513093.631591:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.115@tcp1 : PUT - for me 00000400:00000200:12.0:1616513093.631602:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.115@tcp1 of length 224 into portal 26 MB=0x5ffde2278cdc0 00000800:00000200:14.0:1616513093.631606:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513093.631608:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.115@tcp1 of length 224/224 into md 0xf97ceaa9 [1] + 0 00000400:00000200:12.0:1616513093.631613:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fafe48 00000800:00000200:14.0:1616513093.631615:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513093.631616:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513093.631619:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513093.631624:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.115@tcp1 : PUT - for me 00000400:00000200:12.0:1616513093.631624:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafe48 00000800:00000200:16.0:1616513093.631627:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513093.631627:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.631630:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513093.631632:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513093.631633:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.115@tcp1 of length 224 into portal 12 MB=0x5ffde2278ce40 00000400:00000200:14.0:1616513093.631640:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.115@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 8512 00000400:00000200:14.0:1616513093.631647:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.631649:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513093.631656:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.631659:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.631659:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.631666:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.631701:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704409718208 00010000:00000200:0.0:1616513093.631712:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688704409718208/t0(0) o400->1642b296-e008-560e-da2d-c08fe3f4e2cc@10.31.142.115@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513093.631722:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704409718336 00000100:00000200:0.0:1616513093.631727:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704409718208, offset 192 00000400:00000200:0.0:1616513093.631733:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.115@tcp1 00010000:00000200:26.0:1616513093.631734:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05a00 x1688704409718336/t0(0) o400->98e0634c-e0e8-2683-c9e4-aba596f13daa@10.31.142.115@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513093.631741:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.115@tcp1 routed destination 00000400:00000200:0.0:1616513093.631757:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.115@tcp1(10.31.142.115@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513093.631759:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704409718336, offset 192 00000800:00000200:0.0:1616513093.631764:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513093.631765:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.115@tcp1 00000800:00000200:0.0:1616513093.631770:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:26.0:1616513093.631773:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.115@tcp1 routed destination 00000800:00000200:0.0:1616513093.631773:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.631775:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.631780:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.631785:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.631786:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513093.631787:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.115@tcp1(10.31.142.115@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.631794:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.631799:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.631802:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.631804:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513093.631809:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.631814:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513093.631839:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.631845:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.631848:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.631857:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.631862:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beff0 00000800:00000200:20.0:1616513093.631865:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.631868:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513093.631870:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.631877:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.631880:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513093.631882:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513093.631887:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.631890:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afc28 00000800:00000200:18.0:1616513093.631894:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.641483:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513093.641537:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.641547:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513093.641555:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.641564:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x6039913424180 00000400:00000200:2.0:1616513093.641570:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0xf9654ef9 [1] + 8736 00000400:00000200:2.0:1616513093.641575:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.641578:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513093.641585:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.641587:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.641589:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.641592:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513093.641646:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805848252800 00010000:00000200:26.0:1616513093.641710:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6300 x1692805848252800/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:719/0 lens 336/336 e 0 to 0 dl 1616513099 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513093.641723:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805848252800, offset 192 00000400:00000200:26.0:1616513093.641729:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:26.0:1616513093.641736:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:26.0:1616513093.641750:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.641756:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513093.641761:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.641763:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513093.641765:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513093.641770:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.641778:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.641833:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.641840:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.641842:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513093.641849:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513093.641853:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afb18 00000800:00000200:16.0:1616513093.641857:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.641860:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513093.642587:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928576, portal 4 00000100:00000200:8.0:1616513093.642592:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928576, offset 0 00000400:00000200:8.0:1616513093.642599:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:8.0:1616513093.642605:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:8.0:1616513093.642617:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513093.642623:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:8.0:1616513093.642629:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.642632:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:8.0:1616513093.642634:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:8.0:1616513093.642639:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513093.642687:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513093.642708:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513093.642714:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513093.642717:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513093.642724:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513093.642731:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b38e780 x1693067725928576/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513093.642744:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2b28 00000800:00000200:27.0:1616513093.642749:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513093.642752:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513093.642856:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513093.642865:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513093.642871:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513093.642874:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513093.642879:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513093.642892:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627880 00000400:00000200:27.0:1616513093.642899:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716714d [1] + 192 00000400:00000200:27.0:1616513093.642904:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513093.642911:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b38e780 x1693067725928576/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513093.642925:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513093.642929:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513093.642930:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:8.0:1616513093.642932:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b38e780 x1693067725928576/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:8.0:1616513093.642944:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b38e780 x1693067725928576/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513093.642954:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf24c8 00000100:00000200:13.0:1616513093.659300:0:6491:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928592, portal 4 00000100:00000200:13.0:1616513093.659305:0:6491:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928592, offset 0 00000400:00000200:13.0:1616513093.659311:0:6491:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:13.0:1616513093.659317:0:6491:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:13.0:1616513093.659329:0:6491:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:13.0:1616513093.659336:0:6491:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513093.659342:0:6491:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513093.659347:0:6491:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:13.0:1616513093.659349:0:6491:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:13.0:1616513093.659355:0:6491:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513093.659402:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.659422:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513093.659427:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513093.659430:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513093.659468:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:5.0:1616513093.659472:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:1.0:1616513093.659475:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843a2eda00 x1693067725928592/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513093.659487:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178344660 00000800:00000200:1.0:1616513093.659492:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513093.659546:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513093.659555:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.659563:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513093.659565:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513093.659572:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513093.659580:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627890 00000400:00000200:1.0:1616513093.659587:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700045 [1] + 192 00000400:00000200:1.0:1616513093.659592:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513093.659596:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843a2eda00 x1693067725928592/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513093.659611:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.659614:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513093.659616:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:13.0:1616513093.659617:0:6491:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843a2eda00 x1693067725928592/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:13.0:1616513093.659629:0:6491:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843a2eda00 x1693067725928592/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:13.0:1616513093.659638:0:6491:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345078 00000800:00000200:7.2:1616513093.666647:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513093.666716:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.666726:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513093.666735:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.84@tcp1 : PUT - for me 00000400:00000200:22.0:1616513093.666745:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.84@tcp1 of length 224 into portal 26 MB=0x6036786a1dd40 00000400:00000200:22.0:1616513093.666751:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.84@tcp1 of length 224/224 into md 0xf97ceab9 [1] + 0 00000400:00000200:22.0:1616513093.666756:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faff58 00000400:00000200:22.0:1616513093.666759:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.666762:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:11.2:1616513093.666763:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513093.666767:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faff58 00000800:00000200:22.0:1616513093.666770:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.666771:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.666773:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513093.666776:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.666832:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692593035533632 00000800:00000200:20.0:1616513093.666837:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:0.0:1616513093.666842:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1692593035533632/t0(0) o400->7a1067d1-7f28-1b25-b1c5-1e05769c118e@10.31.142.84@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513093.666846:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513093.666855:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.84@tcp1 : PUT - for me 00000100:00000200:0.0:1616513093.666856:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692593035533632, offset 192 00000400:00000200:0.0:1616513093.666862:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.84@tcp1 00000400:00000200:20.0:1616513093.666863:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.84@tcp1 of length 224 into portal 12 MB=0x6036786a1df80 00000400:00000200:0.0:1616513093.666869:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.84@tcp1 routed destination 00000400:00000200:20.0:1616513093.666870:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.84@tcp1 of length 224/224 into md 0x865be291 [1] + 46800 00000400:00000200:20.0:1616513093.666875:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513093.666878:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513093.666884:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.84@tcp1(10.31.142.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513093.666885:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.666888:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513093.666890:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513093.666890:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513093.666893:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513093.666896:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.666898:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.666900:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513093.666904:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692593035534208 00000800:00000200:0.0:1616513093.666905:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.666910:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.666911:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:8.0:1616513093.666915:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0e9f80 x1692593035534208/t0(0) o400->bcd57cf9-80ae-f4eb-0e84-230c64ec1506@10.31.142.84@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513093.666928:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692593035534208, offset 192 00000400:00000200:8.0:1616513093.666934:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.84@tcp1 00000400:00000200:8.0:1616513093.666941:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.84@tcp1 routed destination 00000400:00000200:8.0:1616513093.666955:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.84@tcp1(10.31.142.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513093.666961:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513093.666968:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.666970:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513093.666971:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513093.666972:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513093.666977:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513093.666978:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513093.666981:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513093.666982:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513093.666987:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513093.666993:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfc28 00000800:00000200:2.0:1616513093.666997:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.667000:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513093.667047:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.667054:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.667057:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513093.667064:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513093.667068:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf24c8 00000800:00000200:16.0:1616513093.667071:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.667074:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:15.0:1616513093.674589:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928608, portal 4 00000100:00000200:15.0:1616513093.674595:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928608, offset 0 00000400:00000200:15.0:1616513093.674602:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:15.0:1616513093.674608:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:15.0:1616513093.674621:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:15.0:1616513093.674628:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:15.0:1616513093.674634:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513093.674636:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:15.0:1616513093.674639:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:15.0:1616513093.674643:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513093.674697:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.674714:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513093.674720:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513093.674722:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513093.674729:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513093.674736:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456ecc80 x1693067725928608/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513093.674748:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0bc28 00000800:00000200:1.0:1616513093.674752:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513093.674755:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513093.674835:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513093.674843:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.674848:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513093.674851:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513093.674856:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513093.674864:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c6278a0 00000400:00000200:1.0:1616513093.674870:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700065 [1] + 192 00000400:00000200:1.0:1616513093.674875:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513093.674880:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456ecc80 x1693067725928608/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513093.674893:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.674895:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513093.674897:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:15.0:1616513093.674901:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456ecc80 x1693067725928608/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:15.0:1616513093.674914:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456ecc80 x1693067725928608/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:15.0:1616513093.674924:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000800:00000200:11.2:1616513093.675332:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513093.675387:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.675396:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513093.675405:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:20.0:1616513093.675415:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 328 into portal 26 MB=0x603dfad685940 00000400:00000200:20.0:1616513093.675421:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 328/328 into md 0xf97ceac9 [1] + 0 00000400:00000200:20.0:1616513093.675426:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae2a8 00000400:00000200:20.0:1616513093.675429:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513093.675432:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513093.675449:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae2a8 00000800:00000200:20.0:1616513093.675453:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.675454:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.675455:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513093.675458:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.675511:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109082151232 00010000:00000200:0.0:1616513093.675541:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1693109082151232/t0(0) o101->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:50/0 lens 328/344 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.675555:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 312 bytes to portal 25, xid 1693109082151232, offset 192 00000400:00000200:0.0:1616513093.675562:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.675569:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.675585:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.675592:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 312 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.675598:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.675602:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.675604:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.675610:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.675616:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.675617:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513093.675672:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.675679:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.675681:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513093.675688:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.675693:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf650 00000800:00000200:18.0:1616513093.675697:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.675700:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.676025:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513093.676080:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.676089:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513093.676098:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.676107:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 304 into portal 26 MB=0x603dfad685980 00000400:00000200:2.0:1616513093.676114:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 304/304 into md 0xf97cead9 [1] + 0 00000400:00000200:2.0:1616513093.676118:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae110 00000400:00000200:2.0:1616513093.676121:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.676124:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513093.676128:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae110 00000800:00000200:2.0:1616513093.676131:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.676133:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.676134:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.676137:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.676142:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109082151296 20000000:01000000:0.0:1616513093.676149:0:246878:0:(mgs_nids.c:632:mgs_get_ir_logs()) Reading IR log boslfs02-cliir bufsize 16384. 20000000:01000000:0.0:1616513093.676152:0:246878:0:(mgs_nids.c:204:mgs_nidtbl_read()) Read IR logs boslfs02 return with 0, version 871 00000400:00000200:0.0:1616513093.676160:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.676166:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.676171:0:246878:0:(lib-move.c:1738:lnet_handle_send()) rspt_next_hop_nid = 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513093.676182:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.676188:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 0 bytes in 0 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.676193:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.676195:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.676197:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.676200:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.676204:0:246878:0:(niobuf.c:262:ptlrpc_start_bulk_transfer()) Transferring 0 pages 0 bytes via portal 33 id 12345-10.242.104.148@tcp1 mbits 0x603dfad685980-0x603dfad685980 00000800:00000200:11.2:1616513093.676207:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.676261:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.676268:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.676271:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513093.676277:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:16.0:1616513093.676281:0:6461:0:(events.c:450:server_bulk_callback()) event type 5, status 0, desc ffff8d5cd675da00 00000800:00000200:16.0:1616513093.676286:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.676289:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513093.676492:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513093.676504:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.676512:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513093.676514:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513093.676521:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : ACK - for me 00000400:00000200:28.0:1616513093.676532:0:6462:0:(lib-move.c:3970:lnet_parse_ack()) 10.242.46.233@o2ib1: ACK from 12345-10.242.104.148@tcp1 into md 0x866dbaf1 00000400:00000200:28.0:1616513093.676535:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.676538:0:6462:0:(events.c:450:server_bulk_callback()) event type 4, status 0, desc ffff8d5cd675da00 00000400:00000200:28.0:1616513093.676543:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be198 00000800:00000200:28.0:1616513093.676546:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.676548:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.676550:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513093.676552:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1693109082151296/t0(0) o256->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:50/0 lens 304/240 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.676563:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 208 bytes to portal 25, xid 1693109082151296, offset 192 00000400:00000200:0.0:1616513093.676568:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.676573:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.676586:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.676592:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 208 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.676596:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.676598:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.676600:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.676603:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513093.676607:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513093.676608:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.676618:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.676621:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.676623:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513093.676627:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513093.676628:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.676632:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be908 00000800:00000200:18.0:1616513093.676634:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.676942:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513093.676953:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.676961:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513093.676963:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513093.676970:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.676979:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 328 into portal 26 MB=0x603dfad685a40 00000400:00000200:14.0:1616513093.676985:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 328/328 into md 0xf97ceaf9 [1] + 0 00000400:00000200:14.0:1616513093.676990:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faef68 00000400:00000200:14.0:1616513093.676992:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.676995:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513093.677000:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faef68 00000800:00000200:14.0:1616513093.677003:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.677006:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.677008:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.677011:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109082151488 00010000:00000200:0.0:1616513093.677024:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1693109082151488/t0(0) o101->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:50/0 lens 328/344 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.677034:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 312 bytes to portal 25, xid 1693109082151488, offset 192 00000400:00000200:0.0:1616513093.677039:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.677044:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.677057:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.677062:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 312 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.677066:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.677068:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.677070:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.677073:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.677077:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.677078:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.677087:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.677091:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.677092:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513093.677096:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513093.677098:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513093.677103:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf980 00000800:00000200:16.0:1616513093.677106:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513093.677475:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513093.677484:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.677489:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513093.677493:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513093.677496:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:20.0:1616513093.677503:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 296 into portal 26 MB=0x603dfad685a80 00000400:00000200:20.0:1616513093.677515:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 296/296 into md 0xf97ceb19 [1] + 0 00000400:00000200:20.0:1616513093.677519:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faeff0 00000400:00000200:20.0:1616513093.677520:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513093.677523:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513093.677527:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faeff0 00000800:00000200:20.0:1616513093.677529:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.677531:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.677533:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.677536:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109082151552 00010000:00000200:0.0:1616513093.677567:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1693109082151552/t0(0) o501->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:50/0 lens 296/272 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.677576:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 240 bytes to portal 25, xid 1693109082151552, offset 192 00000400:00000200:0.0:1616513093.677580:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.677585:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.677596:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.677601:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 240 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.677605:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.677607:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.677608:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.677610:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513093.677614:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513093.677616:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.677626:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.677631:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.677634:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513093.677635:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513093.677640:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.677646:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be198 00000800:00000200:22.0:1616513093.677650:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.678050:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513093.678059:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.678064:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513093.678067:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513093.678071:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.678080:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 272 into portal 26 MB=0x603dfad685ac0 00000400:00000200:2.0:1616513093.678085:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 272/272 into md 0xf97ceb29 [1] + 0 00000400:00000200:2.0:1616513093.678089:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf980 00000400:00000200:2.0:1616513093.678091:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.678093:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513093.678096:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf980 00000800:00000200:2.0:1616513093.678098:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.678100:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.678101:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.678106:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109082151616 00010000:00000200:0.0:1616513093.678127:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1693109082151616/t0(0) o503->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:50/0 lens 272/8416 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.678137:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 8384 bytes to portal 25, xid 1693109082151616, offset 192 00000400:00000200:0.0:1616513093.678141:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.678146:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.678157:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.678162:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 8384 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.678168:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.678170:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.678172:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.678174:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.678178:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.678179:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513093.678189:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.678194:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513093.678197:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.678230:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.678239:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.678244:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513093.678246:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513093.678250:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.678254:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513093.678255:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.678262:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513093.678270:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.678275:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513093.678278:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513093.678278:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513093.678284:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513093.678291:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfc28 00000800:00000200:12.0:1616513093.678295:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513093.679141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513093.679150:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.679155:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513093.679158:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513093.679162:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:20.0:1616513093.679167:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 328 into portal 26 MB=0x603dfad685b40 00000400:00000200:20.0:1616513093.679172:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 328/328 into md 0xf97ceb49 [1] + 0 00000400:00000200:20.0:1616513093.679176:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae550 00000400:00000200:20.0:1616513093.679178:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513093.679180:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513093.679183:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae550 00000800:00000200:20.0:1616513093.679185:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.679187:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.679189:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.679193:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109082151744 00010000:00000200:0.0:1616513093.679204:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1693109082151744/t0(0) o101->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:50/0 lens 328/344 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.679214:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 312 bytes to portal 25, xid 1693109082151744, offset 192 00000400:00000200:0.0:1616513093.679219:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.679223:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.679236:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.679241:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 312 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.679245:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.679246:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.679248:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.679251:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513093.679255:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513093.679257:0:50918:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.679312:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.679319:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.679322:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513093.679328:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.679334:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfb18 00000800:00000200:18.0:1616513093.679337:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.679340:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.679665:0:50918:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513093.679724:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.679733:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513093.679741:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.679749:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 288 into portal 26 MB=0x603dfad685b80 00000400:00000200:14.0:1616513093.679755:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 288/288 into md 0xf97ceb59 [1] + 0 00000400:00000200:14.0:1616513093.679760:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf650 00000400:00000200:14.0:1616513093.679762:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.679765:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513093.679769:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf650 00000800:00000200:14.0:1616513093.679772:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.679774:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.679776:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513093.679778:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109082151808 00000800:00000200:14.0:1616513093.679780:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513093.679792:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1693109082151808/t0(0) o501->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:50/0 lens 288/272 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.679801:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 240 bytes to portal 25, xid 1693109082151808, offset 192 00000400:00000200:0.0:1616513093.679805:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.679809:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.679821:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.679826:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 240 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.679830:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.679832:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.679833:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.679836:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.679839:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.679841:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.679900:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.679907:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.679910:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513093.679916:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513093.679924:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be6e8 00000800:00000200:16.0:1616513093.679927:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.679931:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513093.680150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513093.680158:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.680163:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513093.680166:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513093.680170:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:28.0:1616513093.680186:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 272 into portal 26 MB=0x603dfad685bc0 00000400:00000200:28.0:1616513093.680192:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 272/272 into md 0xf97ceb79 [1] + 0 00000400:00000200:28.0:1616513093.680208:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf188 00000400:00000200:28.0:1616513093.680210:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.680212:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513093.680216:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf188 00000800:00000200:28.0:1616513093.680218:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.680220:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.680222:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.680226:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109082151872 00010000:00000200:0.0:1616513093.680239:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1693109082151872/t0(0) o503->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:50/0 lens 272/8416 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.680248:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 8384 bytes to portal 25, xid 1693109082151872, offset 192 00000400:00000200:0.0:1616513093.680253:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.680257:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.680268:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.680273:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 8384 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.680277:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.680279:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.680281:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.680283:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513093.680287:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513093.680289:0:50918:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.680299:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.680303:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.680304:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.680346:0:50918:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.680358:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.680365:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d4[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513093.680367:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.680369:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513093.680374:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.680375:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.680383:0:50918:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.680391:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.680395:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.680397:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513093.680400:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513093.680402:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.680407:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beaa0 00000800:00000200:18.0:1616513093.680410:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.681356:0:50918:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513093.681375:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.681381:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513093.681383:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513093.681387:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.681393:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 328 into portal 26 MB=0x603dfad685c00 00000400:00000200:14.0:1616513093.681398:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 328/328 into md 0xf97ceb89 [1] + 0 00000400:00000200:14.0:1616513093.681401:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fafa90 00000400:00000200:14.0:1616513093.681402:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.681405:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513093.681408:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafa90 00000800:00000200:14.0:1616513093.681410:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.681411:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.681413:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.681417:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109082151936 00010000:00000200:0.0:1616513093.681427:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1693109082151936/t0(0) o101->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:50/0 lens 328/344 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.681446:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 312 bytes to portal 25, xid 1693109082151936, offset 192 00000400:00000200:0.0:1616513093.681450:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.681455:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.681466:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.681471:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 312 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.681475:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.681477:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.681478:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.681480:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.681485:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.681486:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.681538:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.681545:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.681548:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513093.681554:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513093.681560:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf188 00000800:00000200:16.0:1616513093.681563:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.681566:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513093.681807:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513093.681860:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.681869:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513093.681878:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.148@tcp1 : PUT - for me 00000400:00000200:20.0:1616513093.681887:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.148@tcp1 of length 304 into portal 26 MB=0x603dfad685c40 00000400:00000200:20.0:1616513093.681893:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.148@tcp1 of length 304/304 into md 0xf97ceba9 [1] + 0 00000400:00000200:20.0:1616513093.681914:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf760 00000400:00000200:20.0:1616513093.681917:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513093.681919:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513093.681924:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf760 00000800:00000200:20.0:1616513093.681926:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.681930:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513093.681931:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513093.681933:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693109082152000 00000800:00000200:20.0:1616513093.681934:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513093.681944:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1693109082152000/t0(0) o501->d7cef53e-356a-9ca3-fcff-d2717af0dbd0@10.242.104.148@tcp1:50/0 lens 304/272 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc -2/0 00000100:00000200:0.0:1616513093.681953:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 240 bytes to portal 25, xid 1693109082152000, offset 192 00000400:00000200:0.0:1616513093.681957:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.148@tcp1 00000400:00000200:0.0:1616513093.681961:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.148@tcp1 routed destination 00000400:00000200:0.0:1616513093.681973:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.148@tcp1(10.242.104.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.681977:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 240 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.681981:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.681983:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.681984:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.681986:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513093.681990:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513093.681994:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.682048:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.682055:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.682058:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513093.682067:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.682072:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bea18 00000800:00000200:22.0:1616513093.682076:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.682079:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.698716:0:50920:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513093.698772:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.698782:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513093.698791:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.110@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.698800:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.110@tcp1 of length 224 into portal 26 MB=0x5ffdee3e1b440 00000400:00000200:2.0:1616513093.698807:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.110@tcp1 of length 224/224 into md 0xf97cebc9 [1] + 0 00000400:00000200:2.0:1616513093.698812:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faf540 00000400:00000200:2.0:1616513093.698815:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.698818:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513093.698822:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faf540 00000800:00000200:14.0:1616513093.698826:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.698826:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.698829:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.698831:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.698888:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707654595648 00010000:00000200:0.0:1616513093.698899:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688707654595648/t0(0) o400->578a9b81-4524-f0c5-91dd-a6a59ff659b0@10.31.132.110@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.698915:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688707654595648, offset 192 00000400:00000200:0.0:1616513093.698921:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.110@tcp1 00000400:00000200:0.0:1616513093.698929:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.110@tcp1 routed destination 00000400:00000200:0.0:1616513093.698944:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.110@tcp1(10.31.132.110@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.698950:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.698956:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.698959:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.698961:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.698966:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.698972:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.698973:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.699027:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.699034:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.699037:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513093.699044:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513093.699049:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bea18 00000800:00000200:16.0:1616513093.699052:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.699056:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513093.700345:0:50920:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513093.700412:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.700422:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513093.700430:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.110@tcp1 : PUT - for me 00000800:00000200:18.0:1616513093.700466:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513093.700469:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.110@tcp1 of length 224 into portal 12 MB=0x5ffdee3e1b780 00000400:00000200:22.0:1616513093.700476:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.110@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 9072 00000400:00000200:22.0:1616513093.700482:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.700484:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513093.700492:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.700496:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.700498:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513093.700557:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707654596480 00010000:00000200:26.0:1616513093.700570:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2ad00 x1688707654596480/t0(0) o400->d97c8819-88a5-0a91-4d90-681cb8912882@10.31.132.110@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513093.700584:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688707654596480, offset 192 00000400:00000200:26.0:1616513093.700590:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.110@tcp1 00000400:00000200:26.0:1616513093.700598:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.110@tcp1 routed destination 00000400:00000200:26.0:1616513093.700612:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.110@tcp1(10.31.132.110@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.700621:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.700627:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.700629:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.700632:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513093.700637:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.700642:0:50920:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.700695:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.700702:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513093.700705:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513093.700711:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.700715:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af540 00000800:00000200:14.0:1616513093.700719:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.700722:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513093.706609:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928640, portal 4 00000100:00000200:28.0:1616513093.706615:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928640, offset 0 00000400:00000200:28.0:1616513093.706621:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:28.0:1616513093.706626:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:28.0:1616513093.706638:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513093.706644:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:28.0:1616513093.706650:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513093.706653:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:28.0:1616513093.706655:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:28.0:1616513093.706660:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:28.0:1616513093.706665:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928624, portal 4 00000100:00000200:28.0:1616513093.706667:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928624, offset 0 00000400:00000200:28.0:1616513093.706672:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:28.0:1616513093.706675:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:28.0:1616513093.706685:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513093.706691:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:28.0:1616513093.706695:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513093.706698:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:28.0:1616513093.706699:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:28.0:1616513093.706701:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.2:1616513093.706707:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513093.706725:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:27.0:1616513093.706731:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:27.0:1616513093.706733:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513093.706740:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513093.706747:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4557cec380 x1693067725928640/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513093.706759:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcca18 00000800:00000200:27.0:1616513093.706762:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513093.706765:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513093.706777:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:3.0:1616513093.706777:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513093.706778:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513093.706783:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513093.706788:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5afb23d580 x1693067725928624/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513093.706799:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccd48 00000800:00000200:27.0:1616513093.706801:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513093.706889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513093.706897:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513093.706903:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513093.706908:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513093.706911:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:31.0:1616513093.706918:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513093.706918:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6278b0 00000800:00000200:3.0:1616513093.706919:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513093.706925:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716717d [1] + 192 00000400:00000200:31.0:1616513093.706927:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513093.706930:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513093.706934:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5afb23d580 x1693067725928624/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513093.706937:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6278c0 00000800:00000200:27.0:1616513093.706948:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:31.0:1616513093.706949:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716716d [1] + 192 00000800:00000200:27.0:1616513093.706950:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513093.706952:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:31.0:1616513093.706955:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.706958:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5afb23d580 x1693067725928624/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513093.706971:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5afb23d580 x1693067725928624/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513093.706981:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccc38 00000100:00000200:31.0:1616513093.706986:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4557cec380 x1693067725928640/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513093.707003:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513093.707006:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:31.0:1616513093.707008:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:28.0:1616513093.707013:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4557cec380 x1693067725928640/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513093.707024:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4557cec380 x1693067725928640/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513093.707033:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccb28 00000800:00000200:7.2:1616513093.714630:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513093.714686:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.714696:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513093.714705:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.231@tcp1 : PUT - for me 00000400:00000200:18.0:1616513093.714715:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.231@tcp1 of length 224 into portal 26 MB=0x603e0ec15f380 00000400:00000200:18.0:1616513093.714723:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.231@tcp1 of length 224/224 into md 0xf97cebd9 [1] + 0 00000400:00000200:18.0:1616513093.714728:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fafed0 00000400:00000200:18.0:1616513093.714731:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.714733:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513093.714738:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafed0 00000800:00000200:22.0:1616513093.714740:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.714741:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.714744:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513093.714747:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513093.714750:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513093.714750:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513093.714759:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.231@tcp1 : PUT - for me 00000400:00000200:22.0:1616513093.714768:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.231@tcp1 of length 224 into portal 12 MB=0x603e0ec15f440 00000400:00000200:22.0:1616513093.714774:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.231@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 9296 00000400:00000200:22.0:1616513093.714780:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.714782:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513093.714789:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.714793:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.714794:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.714803:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114428683136 00010000:00000200:0.0:1616513093.714815:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1693114428683136/t0(0) o400->eecd8dcb-a3f5-e505-1b2d-cd52385233f5@10.31.136.231@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.714830:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114428683136, offset 192 00000400:00000200:0.0:1616513093.714837:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.231@tcp1 00000400:00000200:0.0:1616513093.714845:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.231@tcp1 routed destination 00000100:00000200:26.0:1616513093.714855:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114428683328 00000400:00000200:0.0:1616513093.714860:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.231@tcp1(10.31.136.231@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.714867:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:26.0:1616513093.714868:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2c380 x1693114428683328/t0(0) o400->7a76b8f6-86c4-3cd3-e875-7ad9ade8907e@10.31.136.231@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513093.714873:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.714875:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.714878:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513093.714883:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114428683328, offset 192 00000800:00000200:0.0:1616513093.714883:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513093.714889:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.231@tcp1 00000100:00000200:0.0:1616513093.714889:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.714890:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513093.714896:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.231@tcp1 routed destination 00000400:00000200:26.0:1616513093.714911:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.231@tcp1(10.31.136.231@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.714917:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.714923:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.714926:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.714928:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513093.714932:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.714937:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513093.714945:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.714952:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513093.714961:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513093.714968:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513093.714973:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beaa0 00000800:00000200:28.0:1616513093.714977:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.714979:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513093.715008:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.715016:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.715019:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513093.715019:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513093.715025:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513093.715029:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af430 00000800:00000200:2.0:1616513093.715035:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.720291:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513093.720345:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.720354:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513093.720363:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.103@tcp1 : PUT - for me 00000400:00000200:16.0:1616513093.720372:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.103@tcp1 of length 224 into portal 26 MB=0x6001d372d5440 00000400:00000200:16.0:1616513093.720378:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.103@tcp1 of length 224/224 into md 0xf97cebf9 [1] + 0 00000400:00000200:16.0:1616513093.720383:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fafb18 00000400:00000200:16.0:1616513093.720386:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.720389:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513093.720393:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafb18 00000800:00000200:16.0:1616513093.720396:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.720397:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.720399:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513093.720401:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.720468:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975340033088 00010000:00000200:0.0:1616513093.720480:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688975340033088/t0(0) o400->219f6a46-fb9d-538b-fbb3-8217041f4224@10.31.142.103@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.720494:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688975340033088, offset 192 00000400:00000200:0.0:1616513093.720500:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.103@tcp1 00000400:00000200:0.0:1616513093.720508:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.103@tcp1 routed destination 00000400:00000200:0.0:1616513093.720523:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.103@tcp1(10.31.142.103@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.720530:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.720535:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.720538:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.720540:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.720545:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.720550:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.720552:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513093.720605:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.720611:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513093.720614:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513093.720621:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513093.720626:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfb18 00000800:00000200:28.0:1616513093.720630:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.720633:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.720639:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513093.720641:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513093.720646:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.103@tcp1 : PUT - for me 00000400:00000200:28.0:1616513093.720654:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.103@tcp1 of length 224 into portal 12 MB=0x6001d372d56c0 00000400:00000200:28.0:1616513093.720662:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.103@tcp1 of length 224/224 into md 0x865be291 [1] + 47024 00000400:00000200:28.0:1616513093.720667:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.720670:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513093.720678:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.720680:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.720682:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513093.720699:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975340033728 00010000:00000200:12.0:1616513093.720711:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38ec00 x1688975340033728/t0(0) o400->d3be5659-6829-150c-a91c-e57f3173c585@10.31.142.103@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513093.720725:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688975340033728, offset 192 00000400:00000200:12.0:1616513093.720731:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.103@tcp1 00000400:00000200:12.0:1616513093.720738:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.103@tcp1 routed destination 00000400:00000200:12.0:1616513093.720753:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.103@tcp1(10.31.142.103@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513093.720759:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513093.720765:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513093.720768:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513093.720769:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513093.720782:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.720790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.720849:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.720855:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.720857:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513093.720863:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.720867:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e5d8 00000800:00000200:22.0:1616513093.720871:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.720874:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513093.722512:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928656, portal 4 00000100:00000200:10.0:1616513093.722518:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928656, offset 0 00000400:00000200:10.0:1616513093.722524:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:10.0:1616513093.722530:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:10.0:1616513093.722543:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513093.722549:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:10.0:1616513093.722555:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513093.722558:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513093.722560:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513093.722565:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513093.722613:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513093.722669:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513093.722676:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513093.722679:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513093.722686:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513093.722692:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9646780 x1693067725928656/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513093.722704:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82990 00000800:00000200:18.0:1616513093.722708:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513093.722711:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513093.722790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513093.722846:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513093.722866:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:14.0:1616513093.722866:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513093.722876:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513093.722883:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6278d0 00000400:00000200:22.0:1616513093.722890:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cf1c9 [1] + 192 00000400:00000200:22.0:1616513093.722894:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.722899:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9646780 x1693067725928656/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513093.722913:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513093.722916:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513093.722918:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:10.0:1616513093.722920:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9646780 x1693067725928656/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513093.722932:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9646780 x1693067725928656/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513093.722941:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82c38 00000800:00000200:11.2:1616513093.724967:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513093.725023:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.725033:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513093.725041:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.114@tcp1 : PUT - for me 00000400:00000200:16.0:1616513093.725050:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.114@tcp1 of length 224 into portal 12 MB=0x5ffdfb05aadc0 00000400:00000200:16.0:1616513093.725057:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.114@tcp1 of length 224/224 into md 0x865be291 [1] + 47248 00000400:00000200:16.0:1616513093.725064:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.725067:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513093.725074:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.725076:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.725077:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513093.725078:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513093.725080:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513093.725137:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711085075904 00000800:00000200:18.0:1616513093.725142:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:20.0:1616513093.725150:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6721f80 x1688711085075904/t0(0) o400->d1cbcfad-3d8b-bd87-f5a1-22b2ce4a5f63@10.31.141.114@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513093.725151:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513093.725159:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.114@tcp1 : PUT - for me 00000100:00000200:20.0:1616513093.725164:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688711085075904, offset 192 00000400:00000200:18.0:1616513093.725167:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.114@tcp1 of length 224 into portal 26 MB=0x5ffdfb05aacc0 00000400:00000200:20.0:1616513093.725171:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.114@tcp1 00000400:00000200:18.0:1616513093.725173:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.114@tcp1 of length 224/224 into md 0xf97cec09 [1] + 0 00000400:00000200:20.0:1616513093.725178:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.114@tcp1 routed destination 00000400:00000200:18.0:1616513093.725178:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae330 00000400:00000200:18.0:1616513093.725180:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.725183:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513093.725187:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae330 00000800:00000200:18.0:1616513093.725189:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.725192:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513093.725193:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.114@tcp1(10.31.141.114@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513093.725194:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.725196:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513093.725199:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513093.725205:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513093.725208:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513093.725210:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.725215:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.725220:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513093.725234:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.725238:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.725240:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.725246:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.725249:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a110 00000100:00000200:0.0:1616513093.725250:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711085075648 00000800:00000200:20.0:1616513093.725251:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.725254:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513093.725261:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1688711085075648/t0(0) o400->bc4c70fd-44fa-fb42-ee8a-926abb41eb30@10.31.141.114@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.725275:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688711085075648, offset 192 00000400:00000200:0.0:1616513093.725282:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.114@tcp1 00000400:00000200:0.0:1616513093.725289:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.114@tcp1 routed destination 00000400:00000200:0.0:1616513093.725304:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.114@tcp1(10.31.141.114@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.725315:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.725321:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.725323:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.725326:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.725331:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.725336:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.725337:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513093.725390:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.725397:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513093.725400:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513093.725407:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.725412:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be198 00000800:00000200:14.0:1616513093.725416:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.725418:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.768736:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513093.768757:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513093.768770:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513093.768780:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.113@tcp1 : PUT - for me 00000400:00000200:18.0:1616513093.768790:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.113@tcp1 of length 224 into portal 12 MB=0x5ffdf6b238340 00000400:00000200:18.0:1616513093.768796:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.113@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 9520 00000400:00000200:18.0:1616513093.768802:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.768805:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513093.768815:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.768815:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.768818:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.768820:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513093.768891:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709923832640 00000800:00000200:11.2:1616513093.768892:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:26.0:1616513093.768906:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd702400 x1688709923832640/t0(0) o400->22d1d69e-2c51-f49a-fd7b-b3e1736aacd0@10.31.142.113@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513093.768922:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688709923832640, offset 192 00000400:00000200:26.0:1616513093.768928:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.113@tcp1 00000400:00000200:26.0:1616513093.768936:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.113@tcp1 routed destination 00000400:00000200:26.0:1616513093.768952:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.113@tcp1(10.31.142.113@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513093.768956:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513093.768959:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513093.768965:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513093.768966:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513093.768968:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513093.768970:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:26.0:1616513093.768975:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:12.0:1616513093.768976:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.113@tcp1 : PUT - for me 00000400:00000200:12.0:1616513093.768985:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.113@tcp1 of length 224 into portal 26 MB=0x5ffdf6b2382c0 00000400:00000200:12.0:1616513093.768991:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.113@tcp1 of length 224/224 into md 0xf97cec19 [1] + 0 00000400:00000200:12.0:1616513093.768996:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae7f8 00000400:00000200:12.0:1616513093.768999:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513093.769002:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513093.769008:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae7f8 00000800:00000200:16.0:1616513093.769011:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513093.769011:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513093.769014:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513093.769018:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513093.769019:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513093.769021:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513093.769022:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513093.769029:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513093.769033:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aebb0 00000800:00000200:16.0:1616513093.769037:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.769071:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688709923832512 00010000:00000200:0.0:1616513093.769082:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688709923832512/t0(0) o400->8a2e2732-c516-88cd-d37b-d7e5617cf424@10.31.142.113@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.769097:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688709923832512, offset 192 00000400:00000200:0.0:1616513093.769103:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.113@tcp1 00000400:00000200:0.0:1616513093.769110:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.113@tcp1 routed destination 00000400:00000200:0.0:1616513093.769124:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.113@tcp1(10.31.142.113@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.769130:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.769136:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.769139:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513093.769141:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513093.769146:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.769151:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513093.769153:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513093.769205:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.769211:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513093.769214:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513093.769216:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513093.769221:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.769226:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be198 00000800:00000200:14.0:1616513093.769229:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.789816:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513093.789873:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.789884:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513093.789892:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.135@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.789902:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.135@tcp1 of length 224 into portal 12 MB=0x5ffdebdf311c0 00000400:00000200:2.0:1616513093.789909:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.135@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 9744 00000400:00000200:2.0:1616513093.789914:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.789917:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513093.789924:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.789927:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.789927:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.789931:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.789954:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513093.790005:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707018199488 00000800:00000200:28.0:1616513093.790015:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:26.0:1616513093.790017:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d3180 x1688707018199488/t0(0) o400->e00f2fab-a587-fcfb-464a-0453a6ce3294@10.31.142.135@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:28.0:1616513093.790024:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:26.0:1616513093.790031:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688707018199488, offset 192 00000400:00000200:28.0:1616513093.790032:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.135@tcp1 : PUT - for me 00000400:00000200:26.0:1616513093.790037:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.135@tcp1 00000400:00000200:28.0:1616513093.790041:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.135@tcp1 of length 224 into portal 26 MB=0x5ffdebdf31000 00000400:00000200:26.0:1616513093.790043:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.135@tcp1 routed destination 00000400:00000200:28.0:1616513093.790047:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.135@tcp1 of length 224/224 into md 0xf97cec29 [1] + 0 00000400:00000200:28.0:1616513093.790052:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae088 00000400:00000200:28.0:1616513093.790054:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.790057:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513093.790058:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.135@tcp1(10.31.142.135@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513093.790061:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae088 00000800:00000200:28.0:1616513093.790064:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513093.790064:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513093.790067:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513093.790068:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513093.790069:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513093.790071:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513093.790072:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513093.790074:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513093.790078:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.790083:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.790087:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688707018199040 00010000:00000200:0.0:1616513093.790099:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688707018199040/t0(0) o400->c14e7d60-6210-dc27-ba18-651bc4ca80fd@10.31.142.135@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.790113:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688707018199040, offset 192 00000400:00000200:0.0:1616513093.790119:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.135@tcp1 00000400:00000200:0.0:1616513093.790127:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.135@tcp1 routed destination 00000400:00000200:0.0:1616513093.790142:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.135@tcp1(10.31.142.135@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513093.790148:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513093.790148:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513093.790155:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513093.790155:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.790157:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513093.790158:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513093.790161:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513093.790165:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513093.790167:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513093.790169:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:7.2:1616513093.790171:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513093.790173:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513093.790174:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513093.790176:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513093.790183:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.790190:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.790193:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513093.790199:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.790205:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beaa0 00000800:00000200:22.0:1616513093.790209:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.790212:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513093.802558:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928672, portal 4 00000100:00000200:8.0:1616513093.802562:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928672, offset 0 00000400:00000200:8.0:1616513093.802568:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:8.0:1616513093.802573:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:8.0:1616513093.802584:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513093.802590:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:8.0:1616513093.802595:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513093.802598:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:8.0:1616513093.802600:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:8.0:1616513093.802604:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513093.802652:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513093.802707:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513093.802714:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513093.802717:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513093.802724:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513093.802730:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cde86cc80 x1693067725928672/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513093.802742:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000800:00000200:18.0:1616513093.802746:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513093.802749:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513093.802754:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:2.0:1616513093.802761:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513093.802762:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513093.802769:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6278e0 00000400:00000200:18.0:1616513093.802775:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cf239 [1] + 192 00000400:00000200:18.0:1616513093.802780:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513093.802785:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cde86cc80 x1693067725928672/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513093.802798:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513093.802800:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513093.802802:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:8.0:1616513093.802803:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cde86cc80 x1693067725928672/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:8.0:1616513093.802814:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cde86cc80 x1693067725928672/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513093.802823:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3298 00000100:00000200:3.0:1616513093.818655:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928688, portal 4 00000100:00000200:3.0:1616513093.818661:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928688, offset 0 00000400:00000200:3.0:1616513093.818668:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:3.0:1616513093.818675:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:3.0:1616513093.818688:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513093.818695:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:3.0:1616513093.818702:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513093.818705:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:3.0:1616513093.818708:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:3.0:1616513093.818713:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513093.818761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513093.818781:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513093.818786:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513093.818788:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513093.818790:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513093.818795:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513093.818801:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bfd100 x1693067725928688/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513093.818814:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd3430 00000800:00000200:27.0:1616513093.818818:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513093.818936:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513093.818945:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513093.818951:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513093.818959:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513093.818967:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6278f0 00000400:00000200:27.0:1616513093.818973:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716718d [1] + 192 00000400:00000200:27.0:1616513093.818978:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513093.818983:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bfd100 x1693067725928688/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513093.818996:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513093.818998:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513093.818999:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513093.819002:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:3.0:1616513093.819002:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bfd100 x1693067725928688/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513093.819016:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bfd100 x1693067725928688/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513093.819026:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:11.2:1616513093.830063:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513093.830133:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513093.830142:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513093.830149:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513093.830152:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.160@tcp1 : PUT - for me 00000800:00000200:28.0:1616513093.830159:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513093.830161:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.160@tcp1 of length 224 into portal 26 MB=0x6048d8b97ee00 00000400:00000200:28.0:1616513093.830168:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.160@tcp1 : PUT - for me 00000400:00000200:12.0:1616513093.830168:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.160@tcp1 of length 224/224 into md 0xf97cec39 [1] + 0 00000400:00000200:12.0:1616513093.830172:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae770 00000400:00000200:12.0:1616513093.830175:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513093.830177:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.160@tcp1 of length 224 into portal 12 MB=0x6048d8b97ee80 00000100:00000200:12.0:1616513093.830178:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513093.830182:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae770 00000400:00000200:28.0:1616513093.830183:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.160@tcp1 of length 224/224 into md 0x865be291 [1] + 47472 00000800:00000200:12.0:1616513093.830185:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513093.830188:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513093.830189:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513093.830190:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:28.0:1616513093.830191:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513093.830193:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.830199:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.830202:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.830204:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.830265:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855839153664 00000100:00000200:20.0:1616513093.830267:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855839153792 00010000:00000200:0.0:1616513093.830276:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1693855839153664/t0(0) o400->87122ca9-64c8-12cd-cea7-8f307566a9b7@10.31.140.160@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:20.0:1616513093.830280:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38d100 x1693855839153792/t0(0) o400->7ed0cd10-4212-095d-b31d-50e3393b80a3@10.31.140.160@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513093.830291:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855839153664, offset 192 00000100:00000200:20.0:1616513093.830294:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855839153792, offset 192 00000400:00000200:0.0:1616513093.830298:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.160@tcp1 00000400:00000200:20.0:1616513093.830303:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.160@tcp1 00000400:00000200:0.0:1616513093.830306:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.160@tcp1 routed destination 00000400:00000200:0.0:1616513093.830321:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.160@tcp1(10.31.140.160@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.830328:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513093.830330:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.160@tcp1 routed destination 00000800:00000200:0.0:1616513093.830337:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.830340:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.830343:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:20.0:1616513093.830345:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.160@tcp1(10.31.140.160@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.830348:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.830351:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513093.830353:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.830354:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513093.830357:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513093.830360:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513093.830369:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513093.830374:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.830383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513093.830387:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.830392:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.830394:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513093.830396:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513093.830400:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.830404:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beaa0 00000800:00000200:20.0:1616513093.830407:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513093.830451:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.830458:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513093.830461:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513093.830467:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.830471:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a110 00000800:00000200:14.0:1616513093.830475:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513093.830478:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.849692:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513093.849749:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.849758:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513093.849767:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.25@tcp1 : PUT - for me 00000400:00000200:16.0:1616513093.849776:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.25@tcp1 of length 224 into portal 26 MB=0x60183559fdbc0 00000400:00000200:16.0:1616513093.849783:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.25@tcp1 of length 224/224 into md 0xf97cec49 [1] + 0 00000400:00000200:16.0:1616513093.849787:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae220 00000400:00000200:16.0:1616513093.849790:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.849793:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513093.849797:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae220 00000800:00000200:16.0:1616513093.849800:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.849802:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513093.849803:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513093.849806:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.849861:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513449147328 00010000:00000200:0.0:1616513093.849872:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1690513449147328/t0(0) o400->7185a3d6-6344-9522-72fd-9eedbd21699b@10.31.136.25@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.849887:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513449147328, offset 192 00000400:00000200:0.0:1616513093.849894:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.25@tcp1 00000400:00000200:0.0:1616513093.849902:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.25@tcp1 routed destination 00000400:00000200:0.0:1616513093.849917:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.25@tcp1(10.31.136.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.849924:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513093.849929:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.849932:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.849934:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.849939:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513093.849945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.849946:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513093.849997:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513093.850003:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513093.850006:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513093.850013:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513093.850020:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beaa0 00000800:00000200:12.0:1616513093.850024:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513093.850027:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513093.850217:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513093.850229:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.850237:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513093.850238:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513093.850247:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.25@tcp1 : PUT - for me 00000400:00000200:28.0:1616513093.850256:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.25@tcp1 of length 224 into portal 12 MB=0x60183559fdc40 00000400:00000200:28.0:1616513093.850262:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.25@tcp1 of length 224/224 into md 0x865be291 [1] + 47696 00000400:00000200:28.0:1616513093.850268:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.850270:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513093.850277:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.850280:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.850282:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513093.850298:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513449147456 00010000:00000200:20.0:1616513093.850311:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38d580 x1690513449147456/t0(0) o400->2d3c97bf-475a-6cb6-cda3-5073b3cb25e3@10.31.136.25@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513093.850325:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513449147456, offset 192 00000400:00000200:20.0:1616513093.850331:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.25@tcp1 00000400:00000200:20.0:1616513093.850339:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.25@tcp1 routed destination 00000400:00000200:20.0:1616513093.850354:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.25@tcp1(10.31.136.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513093.850360:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513093.850366:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513093.850368:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513093.850371:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513093.850376:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.850384:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.850449:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.850457:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513093.850459:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513093.850466:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:12.0:1616513093.850468:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928704, portal 4 00000400:00000200:22.0:1616513093.850470:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bcb0 00000100:00000200:12.0:1616513093.850473:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928704, offset 0 00000800:00000200:22.0:1616513093.850474:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.850477:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513093.850479:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:12.0:1616513093.850484:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:12.0:1616513093.850497:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513093.850503:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:12.0:1616513093.850509:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513093.850512:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:12.0:1616513093.850513:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:12.0:1616513093.850518:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513093.850564:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000100:00000200:23.0:1616513093.850596:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928720, portal 4 00000100:00000200:23.0:1616513093.850602:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928720, offset 0 00000400:00000200:23.0:1616513093.850619:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000800:00000200:14.0:1616513093.850620:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000400:00000200:23.0:1616513093.850625:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000800:00000200:14.0:1616513093.850627:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513093.850630:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513093.850637:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:23.0:1616513093.850638:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000100:00000200:14.0:1616513093.850644:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdeb89b00 x1693067725928704/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:23.0:1616513093.850645:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:23.0:1616513093.850651:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513093.850654:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:23.0:1616513093.850656:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:14.0:1616513093.850656:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e880 00000800:00000200:14.0:1616513093.850660:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513093.850663:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:23.0:1616513093.850664:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513093.850711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513093.850724:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513093.850729:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513093.850731:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513093.850734:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:12.0:1616513093.850736:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:12.0:1616513093.850742:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded1f80 x1693067725928720/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:25.2:1616513093.850745:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000400:00000200:12.0:1616513093.850754:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e110 00000800:00000200:22.0:1616513093.850756:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:12.0:1616513093.850757:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:22.0:1616513093.850762:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:18.0:1616513093.850766:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513093.850770:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513093.850777:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627900 00000400:00000200:22.0:1616513093.850784:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cf269 [1] + 192 00000400:00000200:22.0:1616513093.850789:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.850794:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdeb89b00 x1693067725928704/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513093.850819:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513093.850821:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513093.850823:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:12.0:1616513093.850825:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdeb89b00 x1693067725928704/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:12.0:1616513093.850836:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdeb89b00 x1693067725928704/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513093.850845:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:5.2:1616513093.850858:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513093.850868:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513093.850873:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:28.0:1616513093.850876:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:16.0:1616513093.850881:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513093.850888:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627910 00000400:00000200:16.0:1616513093.850894:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dbc91 [1] + 192 00000400:00000200:16.0:1616513093.850898:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.850903:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded1f80 x1693067725928720/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513093.850917:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513093.850920:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513093.850921:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:23.0:1616513093.850928:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded1f80 x1693067725928720/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513093.850940:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded1f80 x1693067725928720/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513093.850950:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fb18 00000400:00000200:31.0:1616513093.871548:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513093.871556:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000100:00000200:27.0:1616513093.882794:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928736, portal 4 00000100:00000200:27.0:1616513093.882800:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928736, offset 0 00000400:00000200:27.0:1616513093.882807:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:27.0:1616513093.882812:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:27.0:1616513093.882826:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513093.882833:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:27.0:1616513093.882840:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513093.882843:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:27.0:1616513093.882845:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:27.0:1616513093.882850:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513093.882898:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.882917:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513093.882922:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513093.882927:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513093.882934:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513093.882941:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be46780 x1693067725928736/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513093.882953:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248bb0 00000800:00000200:1.0:1616513093.882958:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513093.882960:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513093.883079:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513093.883088:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.883094:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513093.883097:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513093.883102:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513093.883111:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627920 00000400:00000200:1.0:1616513093.883117:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700075 [1] + 192 00000400:00000200:1.0:1616513093.883122:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513093.883127:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be46780 x1693067725928736/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513093.883140:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.883143:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513093.883144:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:27.0:1616513093.883148:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be46780 x1693067725928736/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513093.883173:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be46780 x1693067725928736/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513093.883182:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249650 00000800:00000200:7.2:1616513093.892987:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513093.893044:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.893054:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513093.893063:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.37@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.893074:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.37@tcp1 of length 224 into portal 26 MB=0x5ffdc39939dc0 00000400:00000200:14.0:1616513093.893080:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.37@tcp1 of length 224/224 into md 0xf97cec79 [1] + 0 00000400:00000200:14.0:1616513093.893085:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fae3b8 00000400:00000200:14.0:1616513093.893088:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.893091:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513093.893096:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fae3b8 00000800:00000200:2.0:1616513093.893097:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.893099:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.893102:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513093.893104:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.893107:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.893108:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513093.893117:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.37@tcp1 : PUT - for me 00000400:00000200:2.0:1616513093.893126:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.37@tcp1 of length 224 into portal 12 MB=0x5ffdc3993a000 00000400:00000200:2.0:1616513093.893132:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.37@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 9968 00000400:00000200:2.0:1616513093.893138:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.893140:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513093.893148:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513093.893151:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513093.893153:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.893159:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696207416768 00010000:00000200:0.0:1616513093.893169:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688696207416768/t0(0) o400->343ddfa4-589e-8a98-433d-ff5c4fade341@10.31.142.37@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.893184:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696207416768, offset 192 00000400:00000200:0.0:1616513093.893191:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.37@tcp1 00000400:00000200:0.0:1616513093.893197:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.37@tcp1 routed destination 00000100:00000200:26.0:1616513093.893212:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696207417344 00000400:00000200:0.0:1616513093.893212:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.37@tcp1(10.31.142.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.893218:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:26.0:1616513093.893224:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d2880 x1688696207417344/t0(0) o400->61fb08f2-1f71-62cd-ef08-329d80fc67dc@10.31.142.37@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513093.893224:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.893226:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.893228:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.893233:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:26.0:1616513093.893238:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696207417344, offset 192 00000100:00000200:0.0:1616513093.893239:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513093.893241:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513093.893244:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.37@tcp1 00000400:00000200:26.0:1616513093.893252:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.37@tcp1 routed destination 00000400:00000200:26.0:1616513093.893266:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.37@tcp1(10.31.142.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.893273:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.893279:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.893282:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.893284:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513093.893289:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.893294:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513093.893296:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.893304:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513093.893305:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513093.893306:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513093.893312:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513093.893313:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513093.893315:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513093.893316:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513093.893319:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cd48 00000400:00000200:22.0:1616513093.893321:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513093.893322:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513093.893325:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aedd0 00000800:00000200:20.0:1616513093.893325:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513093.893328:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513093.915978:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513093.916035:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.916045:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513093.916054:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.122@tcp1 : PUT - for me 00000400:00000200:28.0:1616513093.916064:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.122@tcp1 of length 224 into portal 26 MB=0x603ff793ff6c0 00000400:00000200:28.0:1616513093.916070:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.122@tcp1 of length 224/224 into md 0xf97cec89 [1] + 0 00000400:00000200:28.0:1616513093.916075:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fafa08 00000400:00000200:28.0:1616513093.916077:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.916080:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513093.916085:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafa08 00000800:00000200:28.0:1616513093.916088:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.916090:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.916091:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513093.916094:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513093.916137:0:50943:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513093.916148:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245646042816 00010000:00000200:0.0:1616513093.916159:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1693245646042816/t0(0) o400->39ce0356-d294-990d-a06e-0a930160975d@10.31.140.122@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.916174:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693245646042816, offset 192 00000400:00000200:0.0:1616513093.916180:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.122@tcp1 00000400:00000200:0.0:1616513093.916187:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.122@tcp1 routed destination 00000800:00000200:2.0:1616513093.916194:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513093.916202:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.122@tcp1(10.31.140.122@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513093.916204:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513093.916209:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513093.916213:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.122@tcp1 : PUT - for me 00000800:00000200:0.0:1616513093.916215:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.916218:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.916220:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513093.916222:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.122@tcp1 of length 224 into portal 12 MB=0x603ff793ff7c0 00000800:00000200:0.0:1616513093.916225:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513093.916228:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.122@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 10192 00000100:00000200:0.0:1616513093.916232:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513093.916233:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513093.916236:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:11.2:1616513093.916238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513093.916243:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.916246:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513093.916248:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513093.916252:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513093.916291:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513093.916298:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.916301:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:26.0:1616513093.916307:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693245646043072 00000400:00000200:20.0:1616513093.916307:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.916313:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c770 00000800:00000200:20.0:1616513093.916318:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:26.0:1616513093.916319:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d6780 x1693245646043072/t0(0) o400->c401ffd1-8b20-2693-5b5c-ca6d2459c9d1@10.31.140.122@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513093.916321:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513093.916333:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693245646043072, offset 192 00000400:00000200:26.0:1616513093.916340:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.122@tcp1 00000400:00000200:26.0:1616513093.916347:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.122@tcp1 routed destination 00000400:00000200:26.0:1616513093.916364:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.122@tcp1(10.31.140.122@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513093.916370:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.916376:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.916379:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.916381:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513093.916386:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.916391:0:50943:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.916449:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513093.916456:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513093.916459:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513093.916462:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513093.916466:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513093.916472:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae3b8 00000800:00000200:14.0:1616513093.916476:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:27.0:1616513093.946686:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928752, portal 4 00000100:00000200:27.0:1616513093.946692:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928752, offset 0 00000400:00000200:27.0:1616513093.946698:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:27.0:1616513093.946704:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:27.0:1616513093.946718:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000100:00000200:3.0:1616513093.946722:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928768, portal 4 00000800:00000200:27.0:1616513093.946725:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000100:00000200:3.0:1616513093.946730:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928768, offset 0 00000800:00000200:27.0:1616513093.946731:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513093.946734:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:27.0:1616513093.946737:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:3.0:1616513093.946737:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:27.0:1616513093.946741:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:3.0:1616513093.946743:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:3.0:1616513093.946756:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513093.946763:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:3.0:1616513093.946770:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513093.946773:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:3.0:1616513093.946775:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:3.0:1616513093.946781:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:1.2:1616513093.946788:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.946806:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513093.946811:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513093.946813:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513093.946820:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:25.2:1616513093.946826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000100:00000200:1.0:1616513093.946827:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456ecc80 x1693067725928752/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513093.946839:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d248770 00000800:00000200:1.0:1616513093.946844:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513093.946847:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:18.0:1616513093.946883:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513093.946892:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513093.946896:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513093.946902:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513093.946908:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be41680 x1693067725928768/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513093.946920:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd3980 00000800:00000200:18.0:1616513093.946924:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513093.946927:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:1.2:1616513093.946960:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513093.946969:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.946975:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513093.946978:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513093.946984:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:25.2:1616513093.946992:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000400:00000200:1.0:1616513093.946992:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627930 00000400:00000200:1.0:1616513093.946998:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700085 [1] + 192 00000400:00000200:1.0:1616513093.947005:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513093.947010:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456ecc80 x1693067725928752/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513093.947024:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513093.947026:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513093.947028:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:27.0:1616513093.947032:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456ecc80 x1693067725928752/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513093.947045:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456ecc80 x1693067725928752/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513093.947045:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000400:00000200:27.0:1616513093.947055:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249650 00000800:00000200:22.0:1616513093.947055:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513093.947064:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513093.947072:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627940 00000400:00000200:22.0:1616513093.947079:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cf2b9 [1] + 192 00000400:00000200:22.0:1616513093.947085:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.947090:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be41680 x1693067725928768/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513093.947099:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513093.947105:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513093.947108:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513093.947110:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:3.0:1616513093.947115:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be41680 x1693067725928768/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513093.947129:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be41680 x1693067725928768/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513093.947139:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:11.2:1616513093.954287:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513093.954344:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513093.954353:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513093.954362:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.154@tcp1 : PUT - for me 00000400:00000200:16.0:1616513093.954371:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.154@tcp1 of length 224 into portal 26 MB=0x5ffd628cd2800 00000400:00000200:16.0:1616513093.954378:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.154@tcp1 of length 224/224 into md 0xf97cec99 [1] + 0 00000400:00000200:16.0:1616513093.954383:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3fafc28 00000400:00000200:16.0:1616513093.954386:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513093.954388:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513093.954393:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3fafc28 00000800:00000200:16.0:1616513093.954396:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513093.954398:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513093.954399:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513093.954405:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513093.954408:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.954409:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513093.954417:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.154@tcp1 : PUT - for me 00000400:00000200:28.0:1616513093.954426:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.154@tcp1 of length 224 into portal 12 MB=0x5ffd628cd28c0 00000400:00000200:28.0:1616513093.954445:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.154@tcp1 of length 224/224 into md 0x865be291 [1] + 47920 00000400:00000200:28.0:1616513093.954451:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513093.954454:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513093.954462:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513093.954466:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513093.954467:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513093.954472:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688670156171264 00010000:00000200:0.0:1616513093.954483:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688670156171264/t0(0) o400->172edc3c-caa6-54d0-5f0e-26a2022f3a00@10.31.138.154@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.954498:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688670156171264, offset 192 00000400:00000200:0.0:1616513093.954505:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.154@tcp1 00000400:00000200:0.0:1616513093.954512:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.154@tcp1 routed destination 00000100:00000200:24.0:1616513093.954528:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688670156171456 00000400:00000200:0.0:1616513093.954529:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.154@tcp1(10.31.138.154@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513093.954536:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:24.0:1616513093.954541:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38ba80 x1688670156171456/t0(0) o400->1d9ffdaf-3e56-7e03-145d-2826dfc13af7@10.31.138.154@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513093.954542:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.954544:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.954547:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.954552:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513093.954556:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688670156171456, offset 192 00000800:00000200:11.2:1616513093.954557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513093.954559:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513093.954562:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.154@tcp1 00000400:00000200:24.0:1616513093.954569:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.154@tcp1 routed destination 00000400:00000200:24.0:1616513093.954584:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.154@tcp1(10.31.138.154@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513093.954591:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513093.954597:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513093.954599:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513093.954602:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513093.954606:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513093.954612:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513093.954613:0:50947:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513093.954619:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513093.954622:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513093.954623:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513093.954628:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513093.954634:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922ddc0 00000800:00000200:12.0:1616513093.954638:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513093.954669:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513093.954676:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513093.954679:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513093.954686:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513093.954690:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2aa0 00000800:00000200:18.0:1616513093.954694:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513093.954697:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513093.965514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513093.965570:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513093.965579:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513093.965588:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.207@tcp1 : PUT - for me 00000400:00000200:14.0:1616513093.965597:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.207@tcp1 of length 224 into portal 26 MB=0x5ffe346ce8cc0 00000400:00000200:14.0:1616513093.965604:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.207@tcp1 of length 224/224 into md 0xf97ceca9 [1] + 0 00000400:00000200:14.0:1616513093.965609:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faeaa0 00000400:00000200:14.0:1616513093.965611:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513093.965614:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513093.965618:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faeaa0 00000800:00000200:14.0:1616513093.965621:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513093.965624:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513093.965624:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.965631:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513093.965634:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.965635:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513093.965643:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.207@tcp1 : PUT - for me 00000400:00000200:22.0:1616513093.965652:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.207@tcp1 of length 224 into portal 12 MB=0x5ffe346ce8d40 00000400:00000200:22.0:1616513093.965658:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.207@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 10416 00000400:00000200:22.0:1616513093.965664:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513093.965666:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513093.965674:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513093.965678:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513093.965679:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513093.965709:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726494153920 00010000:00000200:0.0:1616513093.965721:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688726494153920/t0(0) o400->446b1079-1bda-e86a-8dd5-1544ad673ef6@10.31.141.207@tcp1:50/0 lens 224/224 e 0 to 0 dl 1616513185 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513093.965735:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726494153920, offset 192 00000100:00000200:26.0:1616513093.965738:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726494154048 00000400:00000200:0.0:1616513093.965741:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.207@tcp1 00000400:00000200:0.0:1616513093.965748:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.207@tcp1 routed destination 00010000:00000200:26.0:1616513093.965750:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2ec00 x1688726494154048/t0(0) o400->90823477-8f9a-b0e9-d50a-5ae90bf3b262@10.31.141.207@tcp1:719/0 lens 224/224 e 0 to 0 dl 1616513099 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513093.965763:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726494154048, offset 192 00000400:00000200:0.0:1616513093.965763:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.207@tcp1(10.31.141.207@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513093.965769:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.207@tcp1 00000800:00000200:0.0:1616513093.965770:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513093.965775:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.207@tcp1 routed destination 00000800:00000200:0.0:1616513093.965776:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513093.965779:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513093.965781:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513093.965786:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513093.965789:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.207@tcp1(10.31.141.207@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513093.965792:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513093.965795:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513093.965800:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513093.965803:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513093.965805:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513093.965809:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513093.965814:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513093.965832:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513093.965868:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513093.965875:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513093.965878:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513093.965879:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513093.965885:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513093.965889:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513093.965889:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aff58 00000800:00000200:2.0:1616513093.965893:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513093.965897:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513093.965899:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513093.965906:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513093.965911:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922ca18 00000800:00000200:20.0:1616513093.965915:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513093.965918:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513094.001092:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513094.001148:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.001158:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513094.001167:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.216@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.001176:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.216@tcp1 of length 224 into portal 26 MB=0x602167588c1c0 00000400:00000200:22.0:1616513094.001182:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.216@tcp1 of length 224/224 into md 0xf97cecc9 [1] + 0 00000400:00000200:22.0:1616513094.001187:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd3faee58 00000400:00000200:22.0:1616513094.001190:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.001193:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513094.001197:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd3faee58 00000800:00000200:22.0:1616513094.001199:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.001202:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.001202:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513094.001209:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.001212:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.001213:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.001221:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.216@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.001230:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.216@tcp1 of length 224 into portal 12 MB=0x602167588c240 00000400:00000200:18.0:1616513094.001236:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.216@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 10640 00000400:00000200:18.0:1616513094.001241:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.001244:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.001251:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.001254:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.001256:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.001259:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145344696768 00010000:00000200:0.0:1616513094.001270:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1691145344696768/t0(0) o400->3e477f1c-5187-0074-c406-bf1053db8d85@10.31.140.216@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.001283:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691145344696768, offset 192 00000400:00000200:0.0:1616513094.001289:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.216@tcp1 00000400:00000200:0.0:1616513094.001296:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.216@tcp1 routed destination 00000400:00000200:0.0:1616513094.001309:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.216@tcp1(10.31.140.216@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.001315:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:26.0:1616513094.001316:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145344696896 00000800:00000200:0.0:1616513094.001321:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.001325:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.001327:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:26.0:1616513094.001328:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd703a80 x1691145344696896/t0(0) o400->b6909258-4e72-a61c-593e-f4f2b04a7f7d@10.31.140.216@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513094.001332:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513094.001338:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513094.001340:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513094.001344:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691145344696896, offset 192 00000400:00000200:26.0:1616513094.001350:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.216@tcp1 00000400:00000200:26.0:1616513094.001357:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.216@tcp1 routed destination 00000400:00000200:26.0:1616513094.001372:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.216@tcp1(10.31.140.216@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.001379:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.001390:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.001393:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513094.001396:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513094.001397:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.001401:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513094.001403:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513094.001406:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513094.001407:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513094.001412:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513094.001418:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cff0 00000800:00000200:16.0:1616513094.001421:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.001424:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513094.001473:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.001480:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.001483:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.001484:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513094.001490:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.001494:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aff58 00000800:00000200:2.0:1616513094.001498:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.028169:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513094.028194:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513094.028224:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513094.028234:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513094.028244:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.206@tcp1 : PUT - for me 00000800:00000200:18.0:1616513094.028254:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513094.028255:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.206@tcp1 of length 224 into portal 12 MB=0x60187036ddac0 00000400:00000200:20.0:1616513094.028261:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.206@tcp1 of length 224/224 into md 0x865be291 [1] + 48144 00000800:00000200:18.0:1616513094.028264:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513094.028266:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.028269:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513094.028273:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.206@tcp1 : PUT - for me 00000800:00000200:20.0:1616513094.028276:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.028279:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.028281:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513094.028284:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513094.028284:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.206@tcp1 of length 224 into portal 26 MB=0x60187036dda00 00000400:00000200:18.0:1616513094.028290:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.206@tcp1 of length 224/224 into md 0xf97cecd9 [1] + 0 00000400:00000200:18.0:1616513094.028295:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bee58 00000400:00000200:18.0:1616513094.028298:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.028303:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513094.028307:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bee58 00000800:00000200:18.0:1616513094.028310:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.028313:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.028315:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.028318:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513094.028345:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690529250007744 00010000:00000200:24.0:1616513094.028364:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ef500 x1690529250007744/t0(0) o400->13fd1262-3184-db45-0e4e-43a03fc78510@10.31.142.206@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513094.028373:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690529250007552 00000100:00000200:24.0:1616513094.028378:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690529250007744, offset 192 00000400:00000200:24.0:1616513094.028384:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.206@tcp1 00010000:00000200:0.0:1616513094.028385:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1690529250007552/t0(0) o400->09c3fd86-ce3d-462e-df42-f8afbca8fd3d@10.31.142.206@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:24.0:1616513094.028392:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.206@tcp1 routed destination 00000100:00000200:0.0:1616513094.028399:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690529250007552, offset 192 00000400:00000200:0.0:1616513094.028406:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.206@tcp1 00000400:00000200:24.0:1616513094.028407:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.206@tcp1(10.31.142.206@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.028413:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513094.028416:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.206@tcp1 routed destination 00000800:00000200:24.0:1616513094.028424:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.028427:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513094.028429:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:0.0:1616513094.028431:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.206@tcp1(10.31.142.206@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513094.028442:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513094.028451:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513094.028452:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513094.028458:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.028461:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513094.028463:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513094.028468:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.028473:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513094.028475:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513094.028502:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.028514:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513094.028517:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513094.028526:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513094.028528:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513094.028530:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2bb0 00000800:00000200:16.0:1616513094.028534:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513094.028536:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513094.028537:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513094.028538:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513094.028545:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.028551:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922dba0 00000800:00000200:22.0:1616513094.028554:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.028558:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513094.058577:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928784, portal 4 00000100:00000200:26.0:1616513094.058583:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928784, offset 0 00000400:00000200:26.0:1616513094.058590:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:26.0:1616513094.058596:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:26.0:1616513094.058610:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.058617:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:26.0:1616513094.058623:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.058627:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:26.0:1616513094.058629:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:26.0:1616513094.058635:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513094.058682:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.058703:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513094.058709:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513094.058712:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513094.058719:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513094.058727:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d0d80 x1693067725928784/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513094.058739:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aedd0 00000800:00000200:27.0:1616513094.058744:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513094.058747:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513094.058854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513094.058863:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.058869:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513094.058872:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513094.058877:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513094.058886:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627950 00000400:00000200:27.0:1616513094.058894:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371671ed [1] + 192 00000400:00000200:27.0:1616513094.058900:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513094.058904:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d0d80 x1693067725928784/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513094.058919:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.058922:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513094.058924:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:26.0:1616513094.058929:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d0d80 x1693067725928784/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513094.058942:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d0d80 x1693067725928784/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513094.058953:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae3b8 00000800:00000200:11.2:1616513094.068327:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513094.068383:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513094.068393:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513094.068403:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.128@tcp1 : PUT - for me 00000400:00000200:20.0:1616513094.068413:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.128@tcp1 of length 224 into portal 26 MB=0x5ffde865179c0 00000400:00000200:20.0:1616513094.068419:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.128@tcp1 of length 224/224 into md 0xf97cece9 [1] + 0 00000400:00000200:20.0:1616513094.068424:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf540 00000400:00000200:20.0:1616513094.068427:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.068430:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513094.068451:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf540 00000800:00000200:28.0:1616513094.068452:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.068455:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.068458:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513094.068460:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513094.068463:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513094.068463:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513094.068472:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.128@tcp1 : PUT - for me 00000400:00000200:28.0:1616513094.068481:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.128@tcp1 of length 224 into portal 12 MB=0x5ffde86517b00 00000400:00000200:28.0:1616513094.068487:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.128@tcp1 of length 224/224 into md 0x865be291 [1] + 48368 00000400:00000200:28.0:1616513094.068493:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513094.068495:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513094.068502:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706084862400 00000800:00000200:28.0:1616513094.068503:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513094.068506:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513094.068508:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513094.068514:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688706084862400/t0(0) o400->59edb4b9-21b8-6147-6b03-19cad337e399@10.31.142.128@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.068529:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706084862400, offset 192 00000400:00000200:0.0:1616513094.068536:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.128@tcp1 00000400:00000200:0.0:1616513094.068543:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.128@tcp1 routed destination 00000400:00000200:0.0:1616513094.068558:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.128@tcp1(10.31.142.128@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.068565:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.068571:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.068574:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.068576:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.068581:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.068587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513094.068588:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706084862720 00000100:00000200:0.0:1616513094.068588:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:24.0:1616513094.068601:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38da00 x1688706084862720/t0(0) o400->ea246c4c-e122-7f40-06dd-aa05667d0e37@10.31.142.128@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.068616:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706084862720, offset 192 00000400:00000200:24.0:1616513094.068622:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.128@tcp1 00000400:00000200:24.0:1616513094.068629:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.128@tcp1 routed destination 00000800:00000200:12.0:1616513094.068640:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513094.068644:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.128@tcp1(10.31.142.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513094.068647:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.068650:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513094.068651:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513094.068651:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513094.068656:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513094.068657:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.068660:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.068662:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513094.068662:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c000 00000800:00000200:12.0:1616513094.068665:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513094.068668:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.068675:0:50959:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.068740:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.068747:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.068750:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.068757:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.068761:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2f68 00000800:00000200:18.0:1616513094.068765:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.068768:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.081016:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513094.081073:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.081083:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513094.081087:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513094.081092:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.177@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.081101:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.177@tcp1 of length 224 into portal 26 MB=0x5ffe9f9c73d80 00000400:00000200:22.0:1616513094.081108:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.177@tcp1 of length 224/224 into md 0xf97cecf9 [1] + 0 00000400:00000200:22.0:1616513094.081113:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf3a8 00000400:00000200:22.0:1616513094.081116:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.081118:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513094.081122:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf3a8 00000800:00000200:22.0:1616513094.081125:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.081128:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.081130:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.081188:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688755266600320 00010000:00000200:0.0:1616513094.081200:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688755266600320/t0(0) o400->dae678e5-50ab-6c32-2a5a-7e9a563ccf80@10.31.141.177@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.081214:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688755266600320, offset 192 00000400:00000200:0.0:1616513094.081220:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.177@tcp1 00000800:00000200:11.2:1616513094.081224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:0.0:1616513094.081228:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.177@tcp1 routed destination 00000400:00000200:0.0:1616513094.081242:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.177@tcp1(10.31.141.177@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.081249:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513094.081252:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.081255:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.081258:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513094.081260:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513094.081261:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.081265:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:28.0:1616513094.081272:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.177@tcp1 : PUT - for me 00000100:00000200:0.0:1616513094.081277:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513094.081280:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.177@tcp1 of length 224 into portal 12 MB=0x5ffe9f9c74540 00000400:00000200:28.0:1616513094.081286:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.177@tcp1 of length 224/224 into md 0x865be291 [1] + 48592 00000400:00000200:28.0:1616513094.081292:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513094.081295:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513094.081301:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513094.081304:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513094.081306:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513094.081309:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513094.081312:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513094.081314:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513094.081315:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513094.081320:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513094.081324:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d760 00000800:00000200:28.0:1616513094.081327:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513094.081365:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688755266602304 00010000:00000200:24.0:1616513094.081378:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38e300 x1688755266602304/t0(0) o400->354846b6-cee7-c74e-3a0c-ab8c06c18d7d@10.31.141.177@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.081392:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688755266602304, offset 192 00000400:00000200:24.0:1616513094.081399:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.177@tcp1 00000400:00000200:24.0:1616513094.081406:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.177@tcp1 routed destination 00000400:00000200:24.0:1616513094.081420:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.177@tcp1(10.31.141.177@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.081429:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.081435:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.081439:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.081455:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.081461:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.081465:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.081481:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.081488:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.081490:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.081497:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.081501:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2550 00000800:00000200:18.0:1616513094.081505:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.081508:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.087279:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513094.087334:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.087343:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513094.087352:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.125@tcp1 : PUT - for me 00000400:00000200:12.0:1616513094.087361:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.125@tcp1 of length 224 into portal 26 MB=0x6033f10381ec0 00000400:00000200:12.0:1616513094.087368:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.125@tcp1 of length 224/224 into md 0xf97ced09 [1] + 0 00000400:00000200:12.0:1616513094.087373:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be3b8 00000400:00000200:12.0:1616513094.087375:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.087378:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513094.087383:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be3b8 00000800:00000200:12.0:1616513094.087386:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.087387:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.087388:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.087394:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.087397:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.087398:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513094.087406:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.125@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.087415:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.125@tcp1 of length 224 into portal 12 MB=0x6033f10382280 00000400:00000200:16.0:1616513094.087422:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.125@tcp1 of length 224/224 into md 0x865be291 [1] + 48816 00000400:00000200:16.0:1616513094.087427:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.087430:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513094.087456:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692419250200256 00000800:00000200:16.0:1616513094.087461:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.087464:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.087466:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513094.087466:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1692419250200256/t0(0) o400->66f2387f-51ca-4fc3-19ad-39e9fc8b3378@10.31.143.125@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.087480:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692419250200256, offset 192 00000400:00000200:0.0:1616513094.087486:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.125@tcp1 00000400:00000200:0.0:1616513094.087493:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.125@tcp1 routed destination 00000400:00000200:0.0:1616513094.087507:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.125@tcp1(10.31.143.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.087513:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.087518:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.087521:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.087523:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513094.087526:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692419250201216 00000800:00000200:0.0:1616513094.087528:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.087533:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.087534:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:24.0:1616513094.087539:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6722400 x1692419250201216/t0(0) o400->b457c8eb-40b6-f38e-9865-8a0757e0b0a5@10.31.143.125@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.087553:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692419250201216, offset 192 00000400:00000200:24.0:1616513094.087559:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.125@tcp1 00000400:00000200:24.0:1616513094.087566:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.125@tcp1 routed destination 00000400:00000200:24.0:1616513094.087581:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.125@tcp1(10.31.143.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513094.087586:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513094.087588:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513094.087593:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513094.087594:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.087596:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513094.087596:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513094.087597:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513094.087598:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.087603:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513094.087603:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513094.087608:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922ce58 00000800:00000200:7.2:1616513094.087609:0:50961:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513094.087612:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513094.087664:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.087673:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.087675:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.087682:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.087686:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2660 00000800:00000200:2.0:1616513094.087690:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.087693:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.089725:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.089794:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.089804:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513094.089809:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513094.089813:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.145@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.089822:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.145@tcp1 of length 224 into portal 12 MB=0x60393b5104280 00000400:00000200:18.0:1616513094.089828:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.145@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 10864 00000400:00000200:18.0:1616513094.089834:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.089836:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.089844:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.089847:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.089849:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513094.089911:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692782793081472 00010000:00000200:26.0:1616513094.089923:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd702880 x1692782793081472/t0(0) o400->80acf1b9-b175-b6b4-b388-d6cf67a79ace@10.31.140.145@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.089937:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692782793081472, offset 192 00000400:00000200:26.0:1616513094.089943:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.145@tcp1 00000400:00000200:26.0:1616513094.089951:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.145@tcp1 routed destination 00000400:00000200:26.0:1616513094.089965:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.145@tcp1(10.31.140.145@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.089972:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513094.089977:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.089980:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513094.089982:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513094.089987:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.089993:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513094.090048:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513094.090055:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513094.090058:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513094.090065:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513094.090069:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae3b8 00000800:00000200:28.0:1616513094.090073:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513094.090076:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513094.091228:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513094.091298:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.091307:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513094.091316:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.145@tcp1 : PUT - for me 00000400:00000200:12.0:1616513094.091324:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.145@tcp1 of length 224 into portal 26 MB=0x60393b5104180 00000400:00000200:12.0:1616513094.091331:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.145@tcp1 of length 224/224 into md 0xf97ced29 [1] + 0 00000400:00000200:12.0:1616513094.091335:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be7f8 00000400:00000200:12.0:1616513094.091338:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.091341:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513094.091345:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be7f8 00000800:00000200:12.0:1616513094.091348:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.091351:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.091353:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513094.091355:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513094.091406:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692782793081216 00010000:00000200:0.0:1616513094.091416:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1692782793081216/t0(0) o400->8745b68f-059f-23a0-cbaa-151b34de39d5@10.31.140.145@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.091429:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692782793081216, offset 192 00000400:00000200:0.0:1616513094.091447:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.145@tcp1 00000400:00000200:0.0:1616513094.091454:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.145@tcp1 routed destination 00000400:00000200:0.0:1616513094.091468:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.145@tcp1(10.31.140.145@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.091474:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513094.091479:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.091482:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513094.091484:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513094.091488:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.091493:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513094.091495:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513094.091549:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.091556:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.091558:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.091565:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.091571:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c5d8 00000800:00000200:2.0:1616513094.091574:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.091577:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.104748:0:50963:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513094.104804:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.104814:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513094.104823:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.163@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.104832:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.163@tcp1 of length 224 into portal 26 MB=0x5ffdc19aa9e40 00000400:00000200:14.0:1616513094.104839:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.163@tcp1 of length 224/224 into md 0xf97ced39 [1] + 0 00000400:00000200:14.0:1616513094.104843:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be000 00000400:00000200:14.0:1616513094.104846:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.104849:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513094.104855:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be000 00000800:00000200:18.0:1616513094.104858:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.104859:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.104862:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513094.104864:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.104867:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.104868:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.104876:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.163@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.104884:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.163@tcp1 of length 224 into portal 12 MB=0x5ffdc19aaa0c0 00000400:00000200:18.0:1616513094.104892:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.163@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 11088 00000400:00000200:18.0:1616513094.104897:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.104900:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.104906:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.104909:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.104911:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.104920:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695672053312 00010000:00000200:0.0:1616513094.104930:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688695672053312/t0(0) o400->b7b352e6-0878-f656-ee9e-c2b5215d57c1@10.31.141.163@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.104943:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695672053312, offset 192 00000400:00000200:0.0:1616513094.104949:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.163@tcp1 00000400:00000200:0.0:1616513094.104956:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.163@tcp1 routed destination 00000400:00000200:0.0:1616513094.104970:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.163@tcp1(10.31.141.163@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.104976:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.104981:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.104984:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.104986:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513094.104989:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695672053952 00000800:00000200:0.0:1616513094.104990:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513094.104996:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513094.104997:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:26.0:1616513094.105003:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd703600 x1688695672053952/t0(0) o400->962ef3c6-c998-bbbf-46b7-da92c5b7acb6@10.31.141.163@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.105018:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695672053952, offset 192 00000400:00000200:26.0:1616513094.105024:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.163@tcp1 00000400:00000200:26.0:1616513094.105031:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.163@tcp1 routed destination 00000400:00000200:26.0:1616513094.105046:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.163@tcp1(10.31.141.163@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.105052:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513094.105054:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513094.105058:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513094.105060:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513094.105061:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513094.105063:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513094.105063:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.105068:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513094.105069:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513094.105072:0:50963:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513094.105074:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c880 00000800:00000200:28.0:1616513094.105078:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513094.105080:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513094.105085:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.105093:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513094.105095:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513094.105097:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513094.105102:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.105106:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae4c8 00000800:00000200:22.0:1616513094.105110:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513094.106525:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928800, portal 4 00000100:00000200:8.0:1616513094.106530:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928800, offset 0 00000400:00000200:8.0:1616513094.106537:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:8.0:1616513094.106543:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:8.0:1616513094.106556:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513094.106562:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:8.0:1616513094.106569:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513094.106572:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:8.0:1616513094.106574:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:8.0:1616513094.106579:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513094.106626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.106644:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513094.106650:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513094.106653:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513094.106660:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513094.106667:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b38e780 x1693067725928800/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513094.106679:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf27f8 00000800:00000200:27.0:1616513094.106683:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513094.106686:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513094.106781:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513094.106791:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.106797:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513094.106801:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513094.106805:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513094.106812:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627960 00000400:00000200:27.0:1616513094.106819:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371671fd [1] + 192 00000400:00000200:27.0:1616513094.106824:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513094.106828:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b38e780 x1693067725928800/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513094.106842:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.106845:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513094.106847:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:8.0:1616513094.106849:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b38e780 x1693067725928800/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:8.0:1616513094.106873:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b38e780 x1693067725928800/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513094.106883:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3298 00000800:00000200:7.2:1616513094.132045:0:50966:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513094.132065:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513094.132101:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.132112:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.132121:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.32@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.132131:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.32@tcp1 of length 224 into portal 12 MB=0x60393c3408e00 00000800:00000200:12.0:1616513094.132132:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513094.132138:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.32@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 11312 00000400:00000200:18.0:1616513094.132143:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513094.132144:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:18.0:1616513094.132146:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.132153:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513094.132153:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.32@tcp1 : PUT - for me 00000800:00000200:18.0:1616513094.132156:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.132158:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.132161:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513094.132163:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.32@tcp1 of length 224 into portal 26 MB=0x60393c3408d80 00000400:00000200:12.0:1616513094.132170:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.32@tcp1 of length 224/224 into md 0xf97ced49 [1] + 0 00000400:00000200:12.0:1616513094.132174:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bedd0 00000400:00000200:12.0:1616513094.132177:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.132180:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513094.132185:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bedd0 00000800:00000200:12.0:1616513094.132188:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.132191:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.132193:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.132196:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513094.132224:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783031127552 00010000:00000200:26.0:1616513094.132236:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd707980 x1692783031127552/t0(0) o400->27048e73-bbde-63f1-91d1-b2166723c821@10.31.140.32@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.132250:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783031127552, offset 192 00000100:00000200:0.0:1616513094.132250:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783031127424 00000400:00000200:26.0:1616513094.132256:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.32@tcp1 00010000:00000200:0.0:1616513094.132260:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1692783031127424/t0(0) o400->26a14279-ea92-0ffb-ca9b-8bde2cea7a3d@10.31.140.32@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513094.132263:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.32@tcp1 routed destination 00000100:00000200:0.0:1616513094.132273:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783031127424, offset 192 00000400:00000200:26.0:1616513094.132277:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.32@tcp1(10.31.140.32@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513094.132279:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.32@tcp1 00000800:00000200:26.0:1616513094.132283:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513094.132286:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.32@tcp1 routed destination 00000800:00000200:26.0:1616513094.132288:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.132291:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513094.132293:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513094.132297:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513094.132299:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.32@tcp1(10.31.140.32@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513094.132302:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.132309:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513094.132315:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.132317:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513094.132319:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513094.132324:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.132328:0:50966:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513094.132330:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513094.132355:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513094.132362:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513094.132365:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513094.132372:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513094.132376:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aebb0 00000800:00000200:28.0:1616513094.132379:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513094.132382:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513094.132386:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.132395:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513094.132398:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.132400:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513094.132404:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513094.132410:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cbb0 00000800:00000200:14.0:1616513094.132413:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.144370:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.144426:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.144449:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.144458:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.42@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.144468:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.42@tcp1 of length 224 into portal 26 MB=0x60196194d15c0 00000400:00000200:18.0:1616513094.144474:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.42@tcp1 of length 224/224 into md 0xf97ced59 [1] + 0 00000400:00000200:18.0:1616513094.144479:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf6d8 00000400:00000200:18.0:1616513094.144482:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513094.144482:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:18.0:1616513094.144484:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513094.144488:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf6d8 00000800:00000200:18.0:1616513094.144492:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.144495:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.144496:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.144520:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690594041468352 00010000:00000200:0.0:1616513094.144531:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1690594041468352/t0(0) o400->5c82b6a9-d705-4a57-fa4f-efe89fc76560@10.31.141.42@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.144546:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690594041468352, offset 192 00000400:00000200:0.0:1616513094.144552:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.42@tcp1 00000400:00000200:0.0:1616513094.144559:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.42@tcp1 routed destination 00000400:00000200:0.0:1616513094.144574:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.42@tcp1(10.31.141.42@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.144581:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.144586:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.144589:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.144591:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.144596:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.144603:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.144603:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513094.144657:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.144664:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.144667:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513094.144673:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.144679:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d210 00000800:00000200:12.0:1616513094.144682:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.144685:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513094.145494:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513094.145564:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513094.145573:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513094.145582:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.42@tcp1 : PUT - for me 00000400:00000200:28.0:1616513094.145590:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.42@tcp1 of length 224 into portal 12 MB=0x60196194d1e00 00000400:00000200:28.0:1616513094.145597:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.42@tcp1 of length 224/224 into md 0x865be291 [1] + 49040 00000400:00000200:28.0:1616513094.145602:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513094.145604:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513094.145612:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513094.145614:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513094.145616:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.145619:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513094.145677:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690594041470464 00010000:00000200:24.0:1616513094.145689:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b38e780 x1690594041470464/t0(0) o400->94bcf895-19f2-0a79-85c2-ac32ff6cf753@10.31.141.42@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.145704:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690594041470464, offset 192 00000400:00000200:24.0:1616513094.145710:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.42@tcp1 00000400:00000200:24.0:1616513094.145717:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.42@tcp1 routed destination 00000400:00000200:24.0:1616513094.145732:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.42@tcp1(10.31.141.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.145738:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.145744:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.145747:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.145749:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.145754:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.145758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.145811:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.145818:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513094.145821:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513094.145827:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.145831:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2660 00000800:00000200:22.0:1616513094.145835:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.145838:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:9.2:1616513094.160548:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da656200] (20)++ 00000800:00000200:2.0:1616513094.160605:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da656200] (21)++ 00000800:00000200:2.0:1616513094.160615:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.161@o2ib1 00000400:00000200:2.0:1616513094.160624:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.161@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513094.160633:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.161@o2ib1 of length 224 into portal 26 MB=0x5ffeee710ba80 00000400:00000200:2.0:1616513094.160640:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.161@o2ib1 of length 224/224 into md 0xf97ced69 [1] + 0 00000400:00000200:2.0:1616513094.160649:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be990 00000400:00000200:2.0:1616513094.160652:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513094.160655:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513094.160658:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (22)-- 00000400:00000200:2.0:1616513094.160659:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be990 00000800:00000200:2.0:1616513094.160662:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da656200] (21)++ 00000800:00000200:2.0:1616513094.160665:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da656200] (22)-- 00000800:00000200:2.0:1616513094.160667:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (21)-- 00000800:00000200:9.2:1616513094.160674:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da656200] (20)++ 00000100:00000200:0.0:1616513094.160684:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688776427485824 00010000:00000200:0.0:1616513094.160694:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688776427485824/t0(0) o400->1362588c-42ef-7a52-81b5-56fe9de631e5@10.242.46.161@o2ib1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.160708:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688776427485824, offset 192 00000400:00000200:0.0:1616513094.160714:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.161@o2ib1 00000400:00000200:0.0:1616513094.160721:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.161@o2ib1 local destination 00000400:00000200:0.0:1616513094.160733:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.161@o2ib1(10.242.46.161@o2ib1:10.242.46.161@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513094.160738:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da656200] (21)++ 00000800:00000200:0.0:1616513094.160739:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.161@o2ib1 00000800:00000200:0.0:1616513094.160745:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9d80] -> 10.242.46.161@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513094.160748:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.161@o2ib1 00000800:00000200:0.0:1616513094.160748:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da656200] (22)++ 00000800:00000200:0.0:1616513094.160751:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da656200] (23)++ 00000400:00000200:22.0:1616513094.160756:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.161@o2ib1 : PUT - for me 00000800:00000200:0.0:1616513094.160756:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da656200] (24)-- 00000100:00000200:0.0:1616513094.160763:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513094.160764:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.161@o2ib1 of length 224 into portal 12 MB=0x5ffeee710bb00 00000400:00000200:22.0:1616513094.160771:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.161@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 11536 00000400:00000200:22.0:1616513094.160776:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.160778:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513094.160786:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da656200] (23)++ 00000800:00000200:22.0:1616513094.160789:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da656200] (24)-- 00000800:00000200:22.0:1616513094.160791:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (23)-- 00000800:00000200:14.0:1616513094.160793:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da656200] (22)++ 00000800:00000200:14.0:1616513094.160802:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da656200] (23)-- 00000800:00000200:18.0:1616513094.160805:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (22)-- 00000400:00000200:14.0:1616513094.160805:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513094.160811:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.161@o2ib1: PUT: OK 00000400:00000200:14.0:1616513094.160817:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c660 00000800:00000200:14.0:1616513094.160820:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (21)-- 00000100:00000200:26.0:1616513094.160850:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688776427485952 00010000:00000200:26.0:1616513094.160862:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a28480 x1688776427485952/t0(0) o400->c1c098a5-4d9c-25e4-ad0e-8a772977eb00@10.242.46.161@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.160876:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688776427485952, offset 192 00000400:00000200:26.0:1616513094.160883:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.161@o2ib1 00000400:00000200:26.0:1616513094.160890:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.161@o2ib1 local destination 00000400:00000200:26.0:1616513094.160903:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.161@o2ib1(10.242.46.161@o2ib1:10.242.46.161@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.160912:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.161@o2ib1 00000800:00000200:26.0:1616513094.160917:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9d80] -> 10.242.46.161@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.160920:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da656200] (20)++ 00000800:00000200:26.0:1616513094.160922:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da656200] (21)++ 00000800:00000200:26.0:1616513094.160927:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da656200] (22)-- 00000800:00000200:9.2:1616513094.160932:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da656200] (21)++ 00000800:00000200:2.0:1616513094.160942:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da656200] (22)++ 00000800:00000200:2.0:1616513094.160946:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da656200] (23)-- 00000400:00000200:2.0:1616513094.160949:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.160951:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (22)-- 00000400:00000200:2.0:1616513094.160954:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.161@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.160957:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af430 00000800:00000200:2.0:1616513094.160960:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da656200] (21)-- 00000800:00000200:7.2:1616513094.169795:0:50970:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.169852:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.169863:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.169873:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.140@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.169882:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.140@tcp1 of length 224 into portal 26 MB=0x60393cafb1480 00000400:00000200:18.0:1616513094.169889:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.140@tcp1 of length 224/224 into md 0xf97ced79 [1] + 0 00000400:00000200:18.0:1616513094.169894:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf760 00000400:00000200:18.0:1616513094.169896:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.169899:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513094.169904:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf760 00000800:00000200:18.0:1616513094.169907:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.169909:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.169910:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.169916:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.169919:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.169920:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513094.169928:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.140@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.169938:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.140@tcp1 of length 224 into portal 12 MB=0x60393cafb1500 00000400:00000200:14.0:1616513094.169944:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.140@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 11760 00000400:00000200:14.0:1616513094.169949:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.169952:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513094.169959:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.169962:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.169964:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.169969:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783160792192 00010000:00000200:0.0:1616513094.169980:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1692783160792192/t0(0) o400->e6116f41-df48-51ca-11c1-c7bf0061a15f@10.31.140.140@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.169994:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783160792192, offset 192 00000400:00000200:0.0:1616513094.170001:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.140@tcp1 00000400:00000200:0.0:1616513094.170009:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.140@tcp1 routed destination 00000100:00000200:26.0:1616513094.170022:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783160792320 00000400:00000200:0.0:1616513094.170024:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.140@tcp1(10.31.140.140@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.170031:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:26.0:1616513094.170035:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e05e80 x1692783160792320/t0(0) o400->a9beb37b-1a17-eb13-d225-d8538a236edc@10.31.140.140@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513094.170036:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.170039:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.170041:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.170046:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:26.0:1616513094.170049:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783160792320, offset 192 00000800:00000200:11.2:1616513094.170053:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.170053:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513094.170055:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.140@tcp1 00000400:00000200:26.0:1616513094.170063:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.140@tcp1 routed destination 00000400:00000200:26.0:1616513094.170078:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.140@tcp1(10.31.140.140@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.170085:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.170090:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.170093:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.170095:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.170100:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.170106:0:50970:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513094.170109:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.170116:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.170118:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513094.170125:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.170131:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cee0 00000800:00000200:12.0:1616513094.170134:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.170137:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513094.170162:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.170169:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.170172:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513094.170173:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513094.170179:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.170183:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af540 00000800:00000200:2.0:1616513094.170187:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.178994:0:50971:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513094.179049:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.179059:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513094.179068:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.253@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.179077:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.253@tcp1 of length 224 into portal 26 MB=0x6040a1cda1840 00000400:00000200:14.0:1616513094.179084:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.253@tcp1 of length 224/224 into md 0xf97ced99 [1] + 0 00000400:00000200:14.0:1616513094.179089:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be660 00000800:00000200:11.2:1616513094.179090:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:14.0:1616513094.179092:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.179094:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513094.179099:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be660 00000800:00000200:14.0:1616513094.179102:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.179104:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.179105:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.179109:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513094.179146:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513094.179155:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513094.179164:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.253@tcp1 : PUT - for me 00000400:00000200:28.0:1616513094.179173:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.253@tcp1 of length 224 into portal 12 MB=0x6040a1cda1900 00000100:00000200:0.0:1616513094.179178:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693291340503104 00000400:00000200:28.0:1616513094.179180:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.253@tcp1 of length 224/224 into md 0x865be291 [1] + 49264 00000400:00000200:28.0:1616513094.179185:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513094.179187:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:0.0:1616513094.179189:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1693291340503104/t0(0) o400->7c68e3f9-776e-09c2-1d52-090eb080571e@10.31.132.253@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513094.179195:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513094.179198:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513094.179200:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513094.179203:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693291340503104, offset 192 00000800:00000200:28.0:1616513094.179204:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:0.0:1616513094.179210:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.253@tcp1 00000400:00000200:0.0:1616513094.179217:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.253@tcp1 routed destination 00000400:00000200:0.0:1616513094.179232:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.253@tcp1(10.31.132.253@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.179238:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.179244:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.179247:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.179249:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.179254:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.179259:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.179261:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513094.179264:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693291340503296 00010000:00000200:24.0:1616513094.179277:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc1b00 x1693291340503296/t0(0) o400->4c03e2c7-90e5-030b-6a12-1f0d5de50fe4@10.31.132.253@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.179291:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693291340503296, offset 192 00000400:00000200:24.0:1616513094.179297:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.253@tcp1 00000400:00000200:24.0:1616513094.179305:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.253@tcp1 routed destination 00000800:00000200:12.0:1616513094.179313:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513094.179320:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.253@tcp1(10.31.132.253@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513094.179320:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.179323:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513094.179326:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:12.0:1616513094.179329:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513094.179332:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.179335:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513094.179335:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922caa0 00000800:00000200:24.0:1616513094.179337:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513094.179338:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.179341:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513094.179342:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.179347:0:50971:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.179411:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.179418:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.179421:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.179427:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.179431:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2550 00000800:00000200:18.0:1616513094.179447:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.179450:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.189684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513094.189695:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513094.189750:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513094.189757:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513094.189759:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513094.189766:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513094.189768:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.80@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.189775:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.80@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.189779:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.80@tcp1 of length 224 into portal 26 MB=0x601894bebcc40 00000400:00000200:22.0:1616513094.189785:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.80@tcp1 of length 224 into portal 12 MB=0x601894bebcec0 00000400:00000200:16.0:1616513094.189786:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.80@tcp1 of length 224/224 into md 0xf97ceda9 [1] + 0 00000400:00000200:16.0:1616513094.189790:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be440 00000400:00000200:16.0:1616513094.189793:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513094.189795:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.80@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 11984 00000100:00000200:16.0:1616513094.189800:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513094.189801:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:28.0:1616513094.189804:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513094.189805:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be440 00000100:00000200:22.0:1616513094.189808:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513094.189808:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.189811:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.189813:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513094.189817:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.189820:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.189822:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.189825:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.189871:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690539056155712 00010000:00000200:0.0:1616513094.189883:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1690539056155712/t0(0) o400->4984a490-f375-d6fd-8591-24425c60b23e@10.31.140.80@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.189899:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690539056155712, offset 192 00000400:00000200:0.0:1616513094.189905:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.80@tcp1 00000100:00000200:26.0:1616513094.189906:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690539056156352 00000400:00000200:0.0:1616513094.189912:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.80@tcp1 routed destination 00010000:00000200:26.0:1616513094.189918:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2f500 x1690539056156352/t0(0) o400->5678821d-ded0-72f0-091a-71590aa63bc4@10.31.140.80@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513094.189927:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.80@tcp1(10.31.140.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513094.189932:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690539056156352, offset 192 00000800:00000200:0.0:1616513094.189933:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513094.189939:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.80@tcp1 00000800:00000200:0.0:1616513094.189939:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.189942:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.189944:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513094.189947:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.80@tcp1 routed destination 00000800:00000200:0.0:1616513094.189949:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.189954:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.189956:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513094.189961:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.80@tcp1(10.31.140.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.189968:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.189974:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.189977:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.189979:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.189983:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.189989:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.190004:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513094.190008:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513094.190011:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.190014:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513094.190017:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.190019:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.190020:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.190024:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afb18 00000400:00000200:12.0:1616513094.190026:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513094.190028:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.190031:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513094.190031:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d078 00000800:00000200:12.0:1616513094.190035:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.190038:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513094.190595:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513094.190608:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.190617:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513094.190619:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513094.190627:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.50@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.190636:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.50@tcp1 of length 224 into portal 26 MB=0x6019810b15300 00000400:00000200:14.0:1616513094.190642:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.50@tcp1 of length 224/224 into md 0xf97cedc9 [1] + 0 00000400:00000200:14.0:1616513094.190647:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfed0 00000400:00000200:14.0:1616513094.190650:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.190653:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513094.190657:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfed0 00000800:00000200:14.0:1616513094.190660:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.190663:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.190665:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.190669:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690602486977280 00010000:00000200:0.0:1616513094.190675:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1690602486977280/t0(0) o400->784ef5b9-30f5-46c3-1bb3-b75bfc2c8edd@10.31.142.50@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.190685:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690602486977280, offset 192 00000400:00000200:0.0:1616513094.190690:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.50@tcp1 00000400:00000200:0.0:1616513094.190696:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.50@tcp1 routed destination 00000400:00000200:0.0:1616513094.190708:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.50@tcp1(10.31.142.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.190713:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.190717:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.190720:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.190721:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.190724:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.190729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.190729:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513094.190739:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513094.190741:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513094.190746:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513094.190748:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513094.190749:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513094.190750:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513094.190755:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513094.190755:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513094.190759:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513094.190761:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c908 00000400:00000200:18.0:1616513094.190763:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.50@tcp1 : PUT - for me 00000800:00000200:28.0:1616513094.190764:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513094.190771:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.50@tcp1 of length 224 into portal 12 MB=0x6019810b15400 00000400:00000200:18.0:1616513094.190777:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.50@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 12208 00000400:00000200:18.0:1616513094.190782:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.190787:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.190794:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.190796:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.190798:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513094.190805:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690602486977536 00010000:00000200:26.0:1616513094.190812:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd704c80 x1690602486977536/t0(0) o400->9ff99390-55da-5b1c-bc5c-605ecbf31f8d@10.31.142.50@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.190823:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690602486977536, offset 192 00000400:00000200:26.0:1616513094.190828:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.50@tcp1 00000400:00000200:26.0:1616513094.190834:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.50@tcp1 routed destination 00000400:00000200:26.0:1616513094.190847:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.50@tcp1(10.31.142.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.190853:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.190857:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.190860:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.190862:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.190865:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.190870:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.190878:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.190882:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513094.190885:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513094.190887:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513094.190890:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.190894:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afc28 00000800:00000200:22.0:1616513094.190897:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:29.0:1616513094.202734:0:6492:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928816, portal 4 00000100:00000200:29.0:1616513094.202740:0:6492:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928816, offset 0 00000400:00000200:29.0:1616513094.202747:0:6492:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:29.0:1616513094.202753:0:6492:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:29.0:1616513094.202766:0:6492:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513094.202778:0:6492:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:29.0:1616513094.202784:0:6492:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513094.202788:0:6492:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:29.0:1616513094.202790:0:6492:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:29.0:1616513094.202795:0:6492:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513094.202844:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.202862:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513094.202867:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513094.202869:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513094.202876:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513094.202882:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8302327500 x1693067725928816/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513094.202896:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b3ba0 00000800:00000200:27.0:1616513094.202901:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513094.202903:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513094.203013:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513094.203022:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.203028:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513094.203032:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513094.203036:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513094.203044:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627970 00000400:00000200:27.0:1616513094.203050:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716720d [1] + 192 00000400:00000200:27.0:1616513094.203055:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513094.203060:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8302327500 x1693067725928816/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513094.203073:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.203076:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513094.203077:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:29.0:1616513094.203082:0:6492:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8302327500 x1693067725928816/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:29.0:1616513094.203095:0:6492:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8302327500 x1693067725928816/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:29.0:1616513094.203105:0:6492:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b23b8 00000800:00000200:7.2:1616513094.226397:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513094.226456:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.226467:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513094.226477:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.235@tcp1 : PUT - for me 00000400:00000200:2.0:1616513094.226487:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.235@tcp1 of length 224 into portal 26 MB=0x603e0ecf196c0 00000400:00000200:2.0:1616513094.226494:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.235@tcp1 of length 224/224 into md 0xf97cedd9 [1] + 0 00000400:00000200:2.0:1616513094.226499:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bed48 00000400:00000200:2.0:1616513094.226502:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513094.226505:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513094.226510:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bed48 00000800:00000200:18.0:1616513094.226512:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.226513:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.226516:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513094.226519:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513094.226522:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.226523:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.226532:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.235@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.226542:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.235@tcp1 of length 224 into portal 12 MB=0x603e0ecf19740 00000400:00000200:18.0:1616513094.226548:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.235@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 12432 00000400:00000200:18.0:1616513094.226553:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.226556:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.226564:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.226567:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.226569:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.226576:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114443077312 00010000:00000200:0.0:1616513094.226587:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1693114443077312/t0(0) o400->9007707d-6d1e-307d-5b3c-a6c76beab57f@10.31.136.235@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.226601:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114443077312, offset 192 00000400:00000200:0.0:1616513094.226607:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.235@tcp1 00000400:00000200:0.0:1616513094.226614:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.235@tcp1 routed destination 00000100:00000200:26.0:1616513094.226628:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114443077440 00000400:00000200:0.0:1616513094.226630:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.235@tcp1(10.31.136.235@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.226636:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:26.0:1616513094.226640:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd705100 x1693114443077440/t0(0) o400->8317c132-c299-5929-28a2-e62d0b0bd498@10.31.136.235@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513094.226642:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.226645:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.226647:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.226652:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:26.0:1616513094.226654:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114443077440, offset 192 00000100:00000200:0.0:1616513094.226659:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513094.226660:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.235@tcp1 00000800:00000200:11.2:1616513094.226660:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513094.226667:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.235@tcp1 routed destination 00000400:00000200:26.0:1616513094.226681:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.235@tcp1(10.31.136.235@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.226687:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.226693:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.226695:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.226697:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.226702:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.226708:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513094.226716:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513094.226721:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513094.226722:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.226725:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.226728:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513094.226731:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513094.226732:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513094.226732:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513094.226738:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.226738:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c7f8 00000400:00000200:22.0:1616513094.226741:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afcb0 00000800:00000200:12.0:1616513094.226741:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.226744:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513094.226745:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.230599:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513094.230652:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.230662:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513094.230670:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.140@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.230680:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.140@tcp1 of length 224 into portal 26 MB=0x5ffd79dfb8500 00000400:00000200:14.0:1616513094.230686:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.140@tcp1 of length 224/224 into md 0xf97cedf9 [1] + 0 00000400:00000200:14.0:1616513094.230691:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf210 00000400:00000200:14.0:1616513094.230694:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.230696:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513094.230701:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf210 00000800:00000200:14.0:1616513094.230704:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.230705:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.230707:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.230710:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.230765:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676417111296 00010000:00000200:0.0:1616513094.230776:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688676417111296/t0(0) o400->789efaf2-d09c-a38b-a9d0-dcc2fe51eb41@10.31.143.140@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.230790:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688676417111296, offset 192 00000400:00000200:0.0:1616513094.230797:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.140@tcp1 00000400:00000200:0.0:1616513094.230804:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.140@tcp1 routed destination 00000400:00000200:0.0:1616513094.230819:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.140@tcp1(10.31.143.140@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.230825:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.230831:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.230833:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:11.2:1616513094.230835:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.230836:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.230841:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513094.230847:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513094.230892:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.230901:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513094.230910:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.140@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.230921:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.140@tcp1 of length 224 into portal 12 MB=0x5ffd79dfb8580 00000400:00000200:16.0:1616513094.230927:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.140@tcp1 of length 224/224 into md 0x865be291 [1] + 49488 00000400:00000200:16.0:1616513094.230932:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.230935:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513094.230941:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513094.230944:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513094.230945:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:16.0:1616513094.230952:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:28.0:1616513094.230953:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513094.230955:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513094.230956:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513094.230962:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513094.230968:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c2a8 00000800:00000200:28.0:1616513094.230971:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513094.231005:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676417111424 00010000:00000200:24.0:1616513094.231017:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6723600 x1688676417111424/t0(0) o400->edf524bd-80b1-550b-6d8f-1c2c613ef9d7@10.31.143.140@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.231032:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688676417111424, offset 192 00000400:00000200:24.0:1616513094.231038:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.140@tcp1 00000400:00000200:24.0:1616513094.231045:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.140@tcp1 routed destination 00000400:00000200:24.0:1616513094.231060:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.140@tcp1(10.31.143.140@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.231067:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.231072:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.231075:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.231077:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.231082:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.231087:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.231141:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.231148:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.231151:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.231158:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.231162:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2f68 00000800:00000200:2.0:1616513094.231166:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.231169:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.258705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.258761:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.258771:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513094.258774:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513094.258780:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.138@tcp1 : PUT - for me 00000800:00000200:7.2:1616513094.258782:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:18.0:1616513094.258791:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.138@tcp1 of length 224 into portal 26 MB=0x5ffdd699e0500 00000400:00000200:18.0:1616513094.258798:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.138@tcp1 of length 224/224 into md 0xf97cee19 [1] + 0 00000400:00000200:18.0:1616513094.258803:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf8f8 00000400:00000200:18.0:1616513094.258806:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.258809:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513094.258813:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf8f8 00000800:00000200:18.0:1616513094.258816:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.258819:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.258821:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.258824:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.258829:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.258836:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.138@tcp1 : PUT - for me 00000800:00000200:2.0:1616513094.258838:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513094.258841:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.138@tcp1 of length 224 into portal 12 MB=0x5ffdd699e0780 00000400:00000200:18.0:1616513094.258846:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.138@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 12656 00000400:00000200:18.0:1616513094.258852:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.258854:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.258859:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.258861:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.258863:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.258899:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701308372224 00010000:00000200:0.0:1616513094.258910:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688701308372224/t0(0) o400->b5d401c2-4724-a8ad-f2ff-62498a1a191c@10.31.142.138@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.258924:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701308372224, offset 192 00000100:00000200:26.0:1616513094.258930:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701308372864 00000400:00000200:0.0:1616513094.258931:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.138@tcp1 00000400:00000200:0.0:1616513094.258939:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.138@tcp1 routed destination 00010000:00000200:26.0:1616513094.258942:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd700480 x1688701308372864/t0(0) o400->0917a45c-3403-e825-17bb-6c9614cfad7b@10.31.142.138@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513094.258955:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.138@tcp1(10.31.142.138@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513094.258957:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701308372864, offset 192 00000800:00000200:0.0:1616513094.258962:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513094.258963:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.138@tcp1 00000800:00000200:0.0:1616513094.258968:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:26.0:1616513094.258971:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.138@tcp1 routed destination 00000800:00000200:0.0:1616513094.258972:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.258974:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.258980:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513094.258987:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.138@tcp1(10.31.142.138@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513094.258987:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.258987:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513094.258994:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.259000:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.259003:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.259005:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.259010:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.259015:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.259024:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.259029:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513094.259031:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513094.259033:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513094.259038:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513094.259042:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeaa0 00000800:00000200:20.0:1616513094.259043:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513094.259046:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513094.259050:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513094.259053:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513094.259059:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513094.259065:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d8f8 00000800:00000200:20.0:1616513094.259068:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513094.259072:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:6.0:1616513094.298476:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928832, portal 4 00000100:00000200:6.0:1616513094.298481:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928832, offset 0 00000400:00000200:6.0:1616513094.298485:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:6.0:1616513094.298490:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:6.0:1616513094.298499:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513094.298504:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:6.0:1616513094.298509:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513094.298511:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:6.0:1616513094.298512:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:6.0:1616513094.298517:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:19.0:1616513094.298566:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928848, portal 4 00000100:00000200:19.0:1616513094.298571:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928848, offset 0 00000800:00000200:5.2:1616513094.298571:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:19.0:1616513094.298578:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:19.0:1616513094.298585:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:19.0:1616513094.298599:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513094.298606:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:19.0:1616513094.298612:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513094.298616:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:19.0:1616513094.298618:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:19.0:1616513094.298624:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:28.0:1616513094.298628:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:28.0:1616513094.298636:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:28.0:1616513094.298639:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513094.298645:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:28.0:1616513094.298652:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf81680 x1693067725928832/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513094.298664:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01b18 00000800:00000200:28.0:1616513094.298668:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:27.2:1616513094.298670:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:28.0:1616513094.298671:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.0:1616513094.298690:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513094.298695:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513094.298698:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513094.298700:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513094.298705:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513094.298711:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cdc567080 x1693067725928848/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513094.298723:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17076d8 00000800:00000200:27.0:1616513094.298727:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:5.2:1616513094.298731:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513094.298787:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513094.298796:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513094.298806:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513094.298814:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627980 00000400:00000200:20.0:1616513094.298821:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dbe81 [1] + 192 00000400:00000200:20.0:1616513094.298826:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.298831:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf81680 x1693067725928832/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.2:1616513094.298842:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:16.0:1616513094.298842:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513094.298846:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513094.298849:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:6.0:1616513094.298850:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf81680 x1693067725928832/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513094.298851:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:20.0:1616513094.298851:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.0:1616513094.298858:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000100:00000200:6.0:1616513094.298859:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf81680 x1693067725928832/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513094.298860:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513094.298866:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:6.0:1616513094.298867:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01650 00000400:00000200:27.0:1616513094.298874:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627990 00000400:00000200:27.0:1616513094.298880:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716721d [1] + 192 00000400:00000200:27.0:1616513094.298886:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513094.298890:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cdc567080 x1693067725928848/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513094.298904:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.298909:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513094.298911:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:19.0:1616513094.298911:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cdc567080 x1693067725928848/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513094.298924:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cdc567080 x1693067725928848/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513094.298934:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000800:00000200:11.2:1616513094.305740:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513094.305796:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513094.305805:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513094.305815:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.74@tcp1 : PUT - for me 00000400:00000200:28.0:1616513094.305824:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.74@tcp1 of length 224 into portal 26 MB=0x604cfcc81df40 00000400:00000200:28.0:1616513094.305831:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.74@tcp1 of length 224/224 into md 0xf97cee29 [1] + 0 00000400:00000200:28.0:1616513094.305836:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bef68 00000400:00000200:28.0:1616513094.305839:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513094.305842:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513094.305847:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bef68 00000800:00000200:12.0:1616513094.305849:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513094.305850:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513094.305853:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513094.305857:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513094.305912:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694140396068672 00010000:00000200:0.0:1616513094.305929:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1694140396068672/t0(0) o400->15eeeff6-290f-74df-820e-81fdda7eb951@10.31.136.74@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.305944:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694140396068672, offset 192 00000400:00000200:0.0:1616513094.305950:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.74@tcp1 00000400:00000200:0.0:1616513094.305957:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.74@tcp1 routed destination 00000400:00000200:0.0:1616513094.305973:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.74@tcp1(10.31.136.74@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.305980:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.305986:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.305989:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.305991:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.305996:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.306001:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.306003:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513094.306055:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.306062:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513094.306064:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513094.306071:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513094.306077:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d320 00000800:00000200:16.0:1616513094.306080:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.306083:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513094.308181:0:50985:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.308237:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.308247:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.308256:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.74@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.308264:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.74@tcp1 of length 224 into portal 12 MB=0x604cfcc81dfc0 00000400:00000200:18.0:1616513094.308271:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.74@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 12880 00000400:00000200:18.0:1616513094.308276:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.308279:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.308286:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.308289:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.308291:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.308295:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513094.308350:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694140396068800 00010000:00000200:26.0:1616513094.308362:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd707080 x1694140396068800/t0(0) o400->9f077256-8c9e-849e-e1d3-a1cb7e9f6a27@10.31.136.74@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.308376:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694140396068800, offset 192 00000400:00000200:26.0:1616513094.308382:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.74@tcp1 00000400:00000200:26.0:1616513094.308389:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.74@tcp1 routed destination 00000400:00000200:26.0:1616513094.308404:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.74@tcp1(10.31.136.74@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.308410:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.308417:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.308420:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.308422:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.308427:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.308431:0:50985:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.308487:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.308494:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.308497:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.308503:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.308507:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae5d8 00000800:00000200:2.0:1616513094.308511:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.308514:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:31.2:1616513094.310385:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:22.0:1616513094.310457:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:22.0:1616513094.310466:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:22.0:1616513094.310474:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513094.310482:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 224 into portal 24 MB=0x603db0154cd30 00000400:00000200:22.0:1616513094.310489:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 18 from 12345-10.242.46.234@o2ib1 of length 224/224 into md 0xf973ca09 [1] + 0 00000400:00000200:22.0:1616513094.310494:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82baa4aee0 00000400:00000200:22.0:1616513094.310497:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.310500:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt_out 00000400:00000200:22.0:1616513094.310505:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aee0 00000800:00000200:22.0:1616513094.310508:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:22.0:1616513094.310512:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:18.0:1616513094.310513:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:22.0:1616513094.310517:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:26.0:1616513094.310572:0:185125:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693089015319856 00010000:00000200:26.0:1616513094.310594:0:185125:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74da74f050 x1693089015319856/t0(0) o41->boslfs02-MDT0000-mdtlov_UUID@10.242.46.234@o2ib1:720/0 lens 224/368 e 0 to 0 dl 1616513100 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513094.310608:0:185125:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 4, xid 1693089015319856, offset 192 00000400:00000200:26.0:1616513094.310614:0:185125:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:26.0:1616513094.310621:0:185125:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:26.0:1616513094.310635:0:185125:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.310642:0:185125:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:26.0:1616513094.310647:0:185125:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.310650:0:185125:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:26.0:1616513094.310654:0:185125:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:26.0:1616513094.310659:0:185125:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513094.310665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:26.0:1616513094.310666:0:185125:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000800:00000200:2.0:1616513094.310719:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:2.0:1616513094.310725:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:2.0:1616513094.310728:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.310735:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.310740:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae000 00000800:00000200:2.0:1616513094.310744:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:2.0:1616513094.310747:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:7.2:1616513094.317500:0:50986:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.317556:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.317566:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.317575:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.128@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.317584:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.128@tcp1 of length 224 into portal 26 MB=0x6018765eba640 00000400:00000200:18.0:1616513094.317590:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.128@tcp1 of length 224/224 into md 0xf97cee59 [1] + 0 00000400:00000200:18.0:1616513094.317595:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf078 00000400:00000200:18.0:1616513094.317598:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.317600:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513094.317604:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf078 00000800:00000200:18.0:1616513094.317607:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.317610:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.317611:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513094.317618:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.317621:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513094.317621:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513094.317630:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.128@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.317638:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.128@tcp1 of length 224 into portal 12 MB=0x6018765eba6c0 00000400:00000200:22.0:1616513094.317646:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.128@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 13104 00000400:00000200:22.0:1616513094.317652:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.317654:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513094.317661:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.317664:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.317666:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.317680:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690530902419008 00010000:00000200:0.0:1616513094.317691:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1690530902419008/t0(0) o400->973bdf1d-8e66-e493-f9ad-e54475465b60@10.31.132.128@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.317705:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690530902419008, offset 192 00000400:00000200:0.0:1616513094.317711:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.128@tcp1 00000400:00000200:0.0:1616513094.317718:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.128@tcp1 routed destination 00000100:00000200:26.0:1616513094.317725:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690530902419136 00000400:00000200:0.0:1616513094.317733:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.128@tcp1(10.31.132.128@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:26.0:1616513094.317737:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2ba80 x1690530902419136/t0(0) o400->d56c65fd-4ea0-3601-29b9-a1b4c892f80b@10.31.132.128@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513094.317740:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.317745:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.317748:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.317750:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513094.317752:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690530902419136, offset 192 00000800:00000200:0.0:1616513094.317755:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513094.317758:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.128@tcp1 00000100:00000200:0.0:1616513094.317761:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513094.317765:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.128@tcp1 routed destination 00000400:00000200:26.0:1616513094.317780:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.128@tcp1(10.31.132.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.317786:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.317792:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.317801:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.317804:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513094.317807:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513094.317808:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.317813:0:50986:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513094.317862:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.317869:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.317872:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513094.317879:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513094.317884:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513094.317884:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922ded0 00000800:00000200:12.0:1616513094.317888:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.317890:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513094.317893:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.317896:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513094.317896:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513094.317902:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513094.317906:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afd38 00000800:00000200:14.0:1616513094.317910:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:31.0:1616513094.346578:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928864, portal 4 00000100:00000200:31.0:1616513094.346584:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928864, offset 0 00000400:00000200:31.0:1616513094.346591:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:31.0:1616513094.346597:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:31.0:1616513094.346611:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513094.346618:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:31.0:1616513094.346624:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513094.346627:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:31.0:1616513094.346630:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:31.0:1616513094.346635:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513094.346683:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513094.346742:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513094.346750:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513094.346753:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513094.346760:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513094.346766:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456ec800 x1693067725928864/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513094.346778:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c3f58 00000800:00000200:22.0:1616513094.346782:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513094.346785:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513094.346851:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513094.346906:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513094.346927:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513094.346936:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513094.346948:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6279a0 00000400:00000200:18.0:1616513094.346954:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cf529 [1] + 192 00000400:00000200:18.0:1616513094.346959:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.346964:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456ec800 x1693067725928864/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513094.346970:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513094.346978:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513094.346981:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513094.346983:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:31.0:1616513094.346989:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456ec800 x1693067725928864/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513094.347001:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456ec800 x1693067725928864/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513094.347012:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000800:00000200:7.2:1616513094.350764:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513094.350829:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.350839:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513094.350848:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.115@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.350858:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.115@tcp1 of length 224 into portal 26 MB=0x5ffe02374e2c0 00000400:00000200:22.0:1616513094.350865:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.115@tcp1 of length 224/224 into md 0xf97cee69 [1] + 0 00000400:00000200:22.0:1616513094.350870:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be550 00000400:00000200:22.0:1616513094.350872:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.350875:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513094.350879:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be550 00000800:00000200:22.0:1616513094.350882:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.350883:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.350885:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.350887:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.350904:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688713016173248 00010000:00000200:0.0:1616513094.350915:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688713016173248/t0(0) o400->bcbafb4a-56d9-1e89-704f-d328a06aa208@10.31.141.115@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.350928:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688713016173248, offset 192 00000400:00000200:0.0:1616513094.350935:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.115@tcp1 00000400:00000200:0.0:1616513094.350942:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.115@tcp1 routed destination 00000400:00000200:0.0:1616513094.350956:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.115@tcp1(10.31.141.115@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.350962:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.350968:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.350970:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.350972:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.350977:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513094.350983:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513094.350985:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.351040:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.351047:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513094.351050:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513094.351056:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513094.351062:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cdd0 00000800:00000200:16.0:1616513094.351065:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.351068:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513094.352806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513094.352871:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.352880:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513094.352887:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.115@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.352895:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.115@tcp1 of length 224 into portal 12 MB=0x5ffe02374e540 00000400:00000200:14.0:1616513094.352901:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.115@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 13328 00000400:00000200:14.0:1616513094.352906:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.352908:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513094.352914:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.352917:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.352919:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.352921:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513094.352975:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688713016173888 00010000:00000200:26.0:1616513094.352987:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06300 x1688713016173888/t0(0) o400->abf2b060-4bc7-c244-5e23-8496ec3fcacd@10.31.141.115@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.353000:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688713016173888, offset 192 00000400:00000200:26.0:1616513094.353006:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.115@tcp1 00000400:00000200:26.0:1616513094.353013:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.115@tcp1 routed destination 00000400:00000200:26.0:1616513094.353026:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.115@tcp1(10.31.141.115@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.353032:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.353038:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.353040:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.353042:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.353047:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.353052:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.353106:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.353113:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.353115:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.353118:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513094.353122:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.353126:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeee0 00000800:00000200:2.0:1616513094.353130:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.361633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513094.361689:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.361698:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513094.361707:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.125@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.361717:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.125@tcp1 of length 224 into portal 26 MB=0x604a356864e40 00000400:00000200:14.0:1616513094.361724:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.125@tcp1 of length 224/224 into md 0xf97cee79 [1] + 0 00000400:00000200:14.0:1616513094.361728:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfdc0 00000400:00000200:14.0:1616513094.361731:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.361734:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513094.361738:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfdc0 00000800:00000200:14.0:1616513094.361741:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.361744:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.361746:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.361748:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.361757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.361764:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.361775:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.361783:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.125@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.361791:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.125@tcp1 of length 224 into portal 12 MB=0x604a356864f80 00000400:00000200:18.0:1616513094.361797:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.125@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 13552 00000400:00000200:18.0:1616513094.361802:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.361804:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513094.361807:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949438086720 00000800:00000200:18.0:1616513094.361810:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.361813:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.361814:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.361817:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513094.361818:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1693949438086720/t0(0) o400->1edd3867-34fd-b7c7-ed30-a263f36debf8@10.31.142.125@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.361832:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949438086720, offset 192 00000400:00000200:0.0:1616513094.361838:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.125@tcp1 00000400:00000200:0.0:1616513094.361846:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.125@tcp1 routed destination 00000400:00000200:0.0:1616513094.361861:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.125@tcp1(10.31.142.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.361868:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.361874:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.361876:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.361879:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.361883:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:26.0:1616513094.361885:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949438087040 00000100:00000200:0.0:1616513094.361890:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:26.0:1616513094.361897:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd703f00 x1693949438087040/t0(0) o400->fb7fc460-fbae-6428-cd17-8cbf7d38a4e0@10.31.142.125@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.361922:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949438087040, offset 192 00000400:00000200:26.0:1616513094.361928:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.125@tcp1 00000800:00000200:11.2:1616513094.361930:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513094.361936:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.125@tcp1 routed destination 00000400:00000200:26.0:1616513094.361950:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.125@tcp1(10.31.142.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.361957:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.361962:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.361965:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.361967:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.361972:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.361977:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.361986:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513094.361986:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513094.361990:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513094.361992:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513094.361994:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513094.361998:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513094.362000:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.362001:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513094.362003:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af8f8 00000800:00000200:14.0:1616513094.362006:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513094.362008:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.362013:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c088 00000800:00000200:12.0:1616513094.362016:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.362019:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.362025:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513094.362028:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.362030:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.362031:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.362033:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:31.0:1616513094.394621:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928880, portal 4 00000100:00000200:31.0:1616513094.394627:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928880, offset 0 00000400:00000200:31.0:1616513094.394634:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:31.0:1616513094.394640:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:31.0:1616513094.394653:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513094.394660:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:31.0:1616513094.394667:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513094.394670:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:31.0:1616513094.394672:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:31.0:1616513094.394678:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513094.394686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513094.394743:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513094.394751:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513094.394755:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513094.394761:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513094.394768:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456e9b00 x1693067725928880/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513094.394780:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c3078 00000800:00000200:20.0:1616513094.394784:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513094.394789:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513094.394913:0:50994:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513094.394969:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513094.394979:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:28.0:1616513094.394988:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:28.0:1616513094.394996:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c6279b0 00000400:00000200:28.0:1616513094.395003:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dbed1 [1] + 192 00000400:00000200:28.0:1616513094.395008:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513094.395014:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456e9b00 x1693067725928880/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513094.395023:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513094.395029:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513094.395032:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:28.0:1616513094.395034:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:31.0:1616513094.395038:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456e9b00 x1693067725928880/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513094.395051:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456e9b00 x1693067725928880/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513094.395061:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000800:00000200:7.2:1616513094.397243:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513094.397299:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.397309:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513094.397318:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.121@tcp1 : PUT - for me 00000400:00000200:2.0:1616513094.397328:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.121@tcp1 of length 224 into portal 26 MB=0x603fe746c7b40 00000400:00000200:2.0:1616513094.397334:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.121@tcp1 of length 224/224 into md 0xf97cee99 [1] + 0 00000400:00000200:2.0:1616513094.397342:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfba0 00000400:00000200:2.0:1616513094.397344:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513094.397347:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513094.397351:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfba0 00000800:00000200:22.0:1616513094.397354:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.397354:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.397357:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513094.397361:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513094.397364:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.397365:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513094.397374:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.121@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.397383:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.121@tcp1 of length 224 into portal 12 MB=0x603fe746c7bc0 00000400:00000200:22.0:1616513094.397389:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.121@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 13776 00000400:00000200:22.0:1616513094.397396:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.397399:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513094.397406:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.397409:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.397411:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.397428:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693241270106944 00010000:00000200:0.0:1616513094.397455:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1693241270106944/t0(0) o400->c5895b71-66ff-1a70-6a76-1557a9599452@10.31.138.121@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:26.0:1616513094.397469:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693241270107072 00000100:00000200:0.0:1616513094.397470:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693241270106944, offset 192 00000400:00000200:0.0:1616513094.397478:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.121@tcp1 00010000:00000200:26.0:1616513094.397482:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2d580 x1693241270107072/t0(0) o400->b9e155e4-1700-8a83-37c9-bd4b66c85e32@10.31.138.121@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513094.397485:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.121@tcp1 routed destination 00000100:00000200:26.0:1616513094.397496:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693241270107072, offset 192 00000400:00000200:0.0:1616513094.397501:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.121@tcp1(10.31.138.121@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513094.397503:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.121@tcp1 00000800:00000200:0.0:1616513094.397507:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513094.397511:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.121@tcp1 routed destination 00000800:00000200:0.0:1616513094.397513:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.397516:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.397518:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.397523:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513094.397526:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.121@tcp1(10.31.138.121@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:0.0:1616513094.397530:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513094.397532:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.397538:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.397541:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.397543:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.397548:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.397553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513094.397569:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513094.397608:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.397616:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513094.397618:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513094.397620:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513094.397625:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513094.397629:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513094.397629:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae7f8 00000800:00000200:14.0:1616513094.397633:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513094.397636:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513094.397638:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513094.397645:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513094.397650:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c4c8 00000800:00000200:20.0:1616513094.397654:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513094.397657:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513094.402218:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513094.402274:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.402283:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513094.402292:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.114@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.402301:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.114@tcp1 of length 224 into portal 26 MB=0x60194fe5727c0 00000400:00000200:16.0:1616513094.402307:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.114@tcp1 of length 224/224 into md 0xf97ceeb9 [1] + 0 00000400:00000200:16.0:1616513094.402313:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bebb0 00000400:00000200:16.0:1616513094.402315:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.402318:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513094.402322:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bebb0 00000800:00000200:28.0:1616513094.402327:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.402327:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.402330:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513094.402332:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513094.402385:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589294176192 00010000:00000200:0.0:1616513094.402396:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1690589294176192/t0(0) o400->f160f0b9-ee0e-5912-9241-293fef8c108e@10.31.140.114@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.402410:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690589294176192, offset 192 00000400:00000200:0.0:1616513094.402417:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.114@tcp1 00000400:00000200:0.0:1616513094.402424:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.114@tcp1 routed destination 00000400:00000200:0.0:1616513094.402451:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.114@tcp1(10.31.140.114@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.402457:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.402463:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.402465:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.402468:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.402472:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.402478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.402479:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513094.402531:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.402538:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513094.402541:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513094.402547:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513094.402553:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d430 00000800:00000200:20.0:1616513094.402556:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513094.402559:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513094.402977:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513094.402989:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513094.402997:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513094.402999:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513094.403006:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.114@tcp1 : PUT - for me 00000400:00000200:28.0:1616513094.403014:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.114@tcp1 of length 224 into portal 12 MB=0x60194fe572b00 00000400:00000200:28.0:1616513094.403021:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.114@tcp1 of length 224/224 into md 0x865be291 [1] + 49712 00000400:00000200:28.0:1616513094.403027:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513094.403029:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513094.403036:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513094.403039:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513094.403041:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513094.403101:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690589294177024 00010000:00000200:24.0:1616513094.403113:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc6300 x1690589294177024/t0(0) o400->b1a9aa1b-ed12-5556-d924-d7b316910745@10.31.140.114@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.403127:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690589294177024, offset 192 00000400:00000200:24.0:1616513094.403133:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.114@tcp1 00000400:00000200:24.0:1616513094.403141:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.114@tcp1 routed destination 00000400:00000200:24.0:1616513094.403157:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.114@tcp1(10.31.140.114@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.403164:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.403169:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.403172:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.403174:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.403179:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.403184:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.403237:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.403244:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513094.403247:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513094.403254:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.403258:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2bb0 00000800:00000200:22.0:1616513094.403262:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.403264:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.412242:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513094.412298:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513094.412307:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513094.412316:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.130@tcp1 : PUT - for me 00000400:00000200:20.0:1616513094.412325:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.130@tcp1 of length 224 into portal 26 MB=0x603fc0c4b2a40 00000400:00000200:20.0:1616513094.412332:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.130@tcp1 of length 224/224 into md 0xf97ceec9 [1] + 0 00000400:00000200:20.0:1616513094.412337:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfd38 00000400:00000200:20.0:1616513094.412340:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.412343:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513094.412347:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfd38 00000800:00000200:20.0:1616513094.412351:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.412352:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.412354:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.412359:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.412362:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.412363:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513094.412373:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.130@tcp1 : PUT - for me 00000400:00000200:12.0:1616513094.412383:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.130@tcp1 of length 224 into portal 12 MB=0x603fc0c4b2b00 00000400:00000200:12.0:1616513094.412389:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.130@tcp1 of length 224/224 into md 0x865be291 [1] + 49936 00000400:00000200:12.0:1616513094.412394:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.412397:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513094.412403:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.412406:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.412408:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513094.412411:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230933158464 00010000:00000200:0.0:1616513094.412422:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1693230933158464/t0(0) o400->cf55d745-c94a-f0f1-05e8-2ced41e5dc0e@10.31.138.130@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.412448:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230933158464, offset 192 00000400:00000200:0.0:1616513094.412455:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.130@tcp1 00000400:00000200:0.0:1616513094.412462:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.130@tcp1 routed destination 00000100:00000200:24.0:1616513094.412469:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230933158656 00000400:00000200:0.0:1616513094.412477:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.130@tcp1(10.31.138.130@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:24.0:1616513094.412481:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d206780 x1693230933158656/t0(0) o400->363525a1-1723-cb3b-51b3-d061e2b0cf72@10.31.138.130@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513094.412492:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513094.412495:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230933158656, offset 192 00000800:00000200:0.0:1616513094.412498:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.412500:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:24.0:1616513094.412502:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.130@tcp1 00000800:00000200:0.0:1616513094.412502:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.412507:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513094.412509:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.130@tcp1 routed destination 00000800:00000200:11.2:1616513094.412512:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.412514:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513094.412524:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513094.412524:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.130@tcp1(10.31.138.130@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513094.412531:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513094.412531:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513094.412533:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513094.412535:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513094.412536:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.412539:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513094.412540:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513094.412541:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513094.412546:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d6d8 00000800:00000200:24.0:1616513094.412546:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513094.412549:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513094.412553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.412611:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.412618:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.412620:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.412627:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.412631:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2aa0 00000800:00000200:2.0:1616513094.412635:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.412638:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.421215:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513094.421271:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.421281:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513094.421288:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513094.421290:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.11@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.421299:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.11@tcp1 of length 224 into portal 26 MB=0x604a29bb44840 00000400:00000200:22.0:1616513094.421306:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.11@tcp1 of length 224/224 into md 0xf97ceee9 [1] + 0 00000400:00000200:22.0:1616513094.421311:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf430 00000400:00000200:22.0:1616513094.421314:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.421317:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513094.421321:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf430 00000800:00000200:22.0:1616513094.421324:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.421327:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.421329:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.421383:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946303760448 00010000:00000200:0.0:1616513094.421393:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1693946303760448/t0(0) o400->ba3c2b42-5af9-d366-e087-4ac08cba0a35@10.31.141.11@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513094.421403:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513094.421407:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946303760448, offset 192 00000400:00000200:0.0:1616513094.421413:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.11@tcp1 00000400:00000200:0.0:1616513094.421420:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.11@tcp1 routed destination 00000400:00000200:0.0:1616513094.421444:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.11@tcp1(10.31.141.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.421450:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.421455:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.421457:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.421460:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.421464:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513094.421466:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513094.421471:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.421471:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513094.421476:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513094.421484:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.11@tcp1 : PUT - for me 00000400:00000200:2.0:1616513094.421492:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.11@tcp1 of length 224 into portal 12 MB=0x604a29bb44bc0 00000400:00000200:2.0:1616513094.421499:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.11@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 14000 00000400:00000200:2.0:1616513094.421504:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513094.421506:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513094.421514:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.421517:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513094.421519:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.421522:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513094.421543:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.421550:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.421552:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513094.421559:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.421565:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d6d8 00000800:00000200:12.0:1616513094.421568:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.421571:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513094.421579:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946303761344 00010000:00000200:26.0:1616513094.421590:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7080 x1693946303761344/t0(0) o400->383d11bf-f0e1-5d5f-3c70-a7592349db9e@10.31.141.11@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.421604:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946303761344, offset 192 00000400:00000200:26.0:1616513094.421609:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.11@tcp1 00000400:00000200:26.0:1616513094.421616:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.11@tcp1 routed destination 00000400:00000200:26.0:1616513094.421629:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.11@tcp1(10.31.141.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.421637:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.421643:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.421645:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.421647:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.421652:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.421657:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.421668:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.421673:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.421675:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.421676:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513094.421682:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.421686:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af320 00000800:00000200:18.0:1616513094.421689:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.431588:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513094.431658:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513094.431659:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513094.431667:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513094.431675:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.189@tcp1 : PUT - for me 00000400:00000200:20.0:1616513094.431684:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.189@tcp1 of length 224 into portal 26 MB=0x5ffd3357c98c0 00000400:00000200:20.0:1616513094.431690:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.189@tcp1 of length 224/224 into md 0xf97ceef9 [1] + 0 00000400:00000200:20.0:1616513094.431696:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be330 00000400:00000200:20.0:1616513094.431699:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.431702:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513094.431706:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be330 00000800:00000200:20.0:1616513094.431708:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513094.431711:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.431711:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.431715:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513094.431715:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.431725:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513094.431734:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.189@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.431743:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.189@tcp1 of length 224 into portal 12 MB=0x5ffd3357c99c0 00000400:00000200:14.0:1616513094.431749:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.189@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 14224 00000400:00000200:14.0:1616513094.431755:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.431757:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513094.431764:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.431767:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.431769:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513094.431769:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657484093632 00000800:00000200:14.0:1616513094.431771:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:0.0:1616513094.431779:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688657484093632/t0(0) o400->7042e8b0-a90b-4f48-eccc-496868585275@10.31.138.189@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.431792:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657484093632, offset 192 00000400:00000200:0.0:1616513094.431799:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.189@tcp1 00000400:00000200:0.0:1616513094.431805:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.189@tcp1 routed destination 00000400:00000200:0.0:1616513094.431819:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.189@tcp1(10.31.138.189@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.431825:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:26.0:1616513094.431827:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657484093888 00000800:00000200:0.0:1616513094.431831:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.431833:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.431837:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:26.0:1616513094.431839:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e06780 x1688657484093888/t0(0) o400->a3caea7d-a993-5956-cd5b-c7f168204b74@10.31.138.189@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513094.431841:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.431847:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.431848:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513094.431854:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657484093888, offset 192 00000400:00000200:26.0:1616513094.431860:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.189@tcp1 00000400:00000200:26.0:1616513094.431867:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.189@tcp1 routed destination 00000400:00000200:26.0:1616513094.431882:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.189@tcp1(10.31.138.189@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.431889:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.431894:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.431897:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.431899:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513094.431903:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513094.431904:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.431909:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513094.431910:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.431913:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513094.431919:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513094.431920:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513094.431925:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c4c8 00000800:00000200:22.0:1616513094.431928:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513094.431928:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513094.431930:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513094.431931:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513094.431937:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.431941:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af980 00000800:00000200:22.0:1616513094.431945:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.431948:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.440528:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513094.440545:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513094.440554:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513094.440562:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.82@tcp1 : PUT - for me 00000400:00000200:28.0:1616513094.440573:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.82@tcp1 of length 224 into portal 26 MB=0x60183d99e2a40 00000400:00000200:28.0:1616513094.440579:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.82@tcp1 of length 224/224 into md 0xf97cef19 [1] + 0 00000400:00000200:28.0:1616513094.440583:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfe48 00000400:00000200:28.0:1616513094.440586:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513094.440588:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513094.440593:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfe48 00000800:00000200:28.0:1616513094.440595:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513094.440598:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.440598:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513094.440602:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513094.440654:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513094.440658:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515663628864 00010000:00000200:0.0:1616513094.440669:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1690515663628864/t0(0) o400->c95b3461-b1be-3260-dce6-379b6893dc96@10.31.142.82@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.440682:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690515663628864, offset 192 00000400:00000200:0.0:1616513094.440688:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.82@tcp1 00000400:00000200:0.0:1616513094.440695:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.82@tcp1 routed destination 00000400:00000200:0.0:1616513094.440709:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.82@tcp1(10.31.142.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.440715:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.440720:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.440723:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513094.440725:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513094.440725:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.440729:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.440734:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513094.440735:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:0.0:1616513094.440735:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513094.440744:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.82@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.440752:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.82@tcp1 of length 224 into portal 12 MB=0x60183d99e3200 00000400:00000200:14.0:1616513094.440759:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.82@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 14448 00000400:00000200:14.0:1616513094.440764:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.440766:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513094.440773:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.440775:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.440777:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.440780:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513094.440802:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.440816:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.440818:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513094.440825:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.440830:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cdd0 00000800:00000200:12.0:1616513094.440834:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.440837:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513094.440848:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515663630848 00010000:00000200:26.0:1616513094.440860:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03600 x1690515663630848/t0(0) o400->da06d8bc-83bb-9f4d-b8b7-ee315e4b8855@10.31.142.82@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.440873:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690515663630848, offset 192 00000400:00000200:26.0:1616513094.440879:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.82@tcp1 00000400:00000200:26.0:1616513094.440886:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.82@tcp1 routed destination 00000400:00000200:26.0:1616513094.440900:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.82@tcp1(10.31.142.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.440906:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.440911:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.440914:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.440916:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.440920:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.440925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.440936:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.440943:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513094.440947:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513094.440953:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.440957:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae6e8 00000800:00000200:22.0:1616513094.440960:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.440963:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.457998:0:51000:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513094.458054:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.458064:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513094.458073:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.220@tcp1 : PUT - for me 00000400:00000200:2.0:1616513094.458083:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.220@tcp1 of length 224 into portal 26 MB=0x5ffd2f49d4740 00000400:00000200:2.0:1616513094.458090:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.220@tcp1 of length 224/224 into md 0xf97cef39 [1] + 0 00000400:00000200:2.0:1616513094.458095:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be110 00000400:00000200:2.0:1616513094.458100:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513094.458103:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513094.458108:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513094.458108:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be110 00000800:00000200:2.0:1616513094.458111:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.458114:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.458115:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.458173:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656395716416 00010000:00000200:0.0:1616513094.458184:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688656395716416/t0(0) o400->62b74559-906e-9f9a-aa14-6c294590b2ba@10.31.128.220@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513094.458186:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:0.0:1616513094.458198:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688656395716416, offset 192 00000400:00000200:0.0:1616513094.458205:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.220@tcp1 00000400:00000200:0.0:1616513094.458213:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.220@tcp1 routed destination 00000400:00000200:0.0:1616513094.458243:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.220@tcp1(10.31.128.220@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.458249:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.458254:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.458256:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513094.458257:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.458258:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513094.458263:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513094.458266:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:0.0:1616513094.458270:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513094.458275:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.220@tcp1 : PUT - for me 00000400:00000200:20.0:1616513094.458284:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.220@tcp1 of length 224 into portal 12 MB=0x5ffd2f49d4a00 00000400:00000200:20.0:1616513094.458291:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.220@tcp1 of length 224/224 into md 0x865be291 [1] + 50160 00000400:00000200:20.0:1616513094.458296:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.458298:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513094.458305:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513094.458308:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513094.458310:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513094.458312:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513094.458321:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513094.458323:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513094.458330:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513094.458336:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d320 00000800:00000200:28.0:1616513094.458339:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513094.458342:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513094.458369:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688656395717120 00010000:00000200:24.0:1616513094.458381:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ed100 x1688656395717120/t0(0) o400->51b8770e-2309-9b21-1c47-31d6357ac38d@10.31.128.220@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.458396:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688656395717120, offset 192 00000400:00000200:24.0:1616513094.458402:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.220@tcp1 00000400:00000200:24.0:1616513094.458409:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.220@tcp1 routed destination 00000400:00000200:24.0:1616513094.458424:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.220@tcp1(10.31.128.220@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.458432:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.458453:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.458455:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.458460:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.458465:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.458470:0:51000:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:6.0:1616513094.458519:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928896, portal 4 00000100:00000200:6.0:1616513094.458523:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928896, offset 0 00000400:00000200:6.0:1616513094.458528:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513094.458532:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:22.0:1616513094.458534:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.458542:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513094.458542:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513094.458545:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:6.0:1616513094.458547:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513094.458552:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:6.0:1616513094.458552:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513094.458553:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513094.458554:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:6.0:1616513094.458555:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000400:00000200:22.0:1616513094.458556:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f26e8 00000800:00000200:6.0:1616513094.458559:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513094.458560:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:25.2:1616513094.458607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513094.458621:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513094.458628:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513094.458631:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513094.458633:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513094.458637:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513094.458643:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd707500 x1693067725928896/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513094.458655:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf004c8 00000800:00000200:14.0:1616513094.458659:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513094.458730:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513094.458740:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513094.458757:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513094.458759:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513094.458766:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513094.458774:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6279c0 00000400:00000200:18.0:1616513094.458782:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cf629 [1] + 192 00000400:00000200:18.0:1616513094.458787:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.458792:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd707500 x1693067725928896/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513094.458807:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513094.458810:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:6.0:1616513094.458810:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd707500 x1693067725928896/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513094.458811:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:6.0:1616513094.458819:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd707500 x1693067725928896/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513094.458826:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01650 00000800:00000200:7.2:1616513094.467688:0:51001:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513094.467745:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.467754:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513094.467763:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.21@tcp1 : PUT - for me 00000400:00000200:2.0:1616513094.467772:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.21@tcp1 of length 224 into portal 26 MB=0x601975e1fd840 00000400:00000200:2.0:1616513094.467779:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.21@tcp1 of length 224/224 into md 0xf97cef59 [1] + 0 00000400:00000200:2.0:1616513094.467789:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf4b8 00000400:00000200:2.0:1616513094.467792:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513094.467795:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513094.467798:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513094.467799:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf4b8 00000800:00000200:2.0:1616513094.467803:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513094.467806:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513094.467808:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.467809:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513094.467810:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513094.467817:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.21@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.467826:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.21@tcp1 of length 224 into portal 12 MB=0x601975e1fd8c0 00000400:00000200:14.0:1616513094.467832:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.21@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 14672 00000400:00000200:14.0:1616513094.467837:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.467840:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513094.467847:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.467850:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.467852:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.467864:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690599491098688 00000100:00000200:26.0:1616513094.467870:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690599491098816 00010000:00000200:0.0:1616513094.467876:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1690599491098688/t0(0) o400->6934a1d5-2804-4b86-4fd5-00d046d241ca@10.31.132.21@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:26.0:1616513094.467882:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e03180 x1690599491098816/t0(0) o400->28917d97-0a2c-521d-6f69-25aaa470ddd1@10.31.132.21@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513094.467892:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690599491098688, offset 192 00000100:00000200:26.0:1616513094.467897:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690599491098816, offset 192 00000400:00000200:0.0:1616513094.467899:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.21@tcp1 00000400:00000200:26.0:1616513094.467903:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.21@tcp1 00000400:00000200:0.0:1616513094.467906:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.21@tcp1 routed destination 00000400:00000200:26.0:1616513094.467910:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.21@tcp1 routed destination 00000400:00000200:0.0:1616513094.467921:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.21@tcp1(10.31.132.21@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513094.467925:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.21@tcp1(10.31.132.21@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.467928:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513094.467931:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:0.0:1616513094.467934:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.467937:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.467937:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.467939:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513094.467940:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.467942:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513094.467944:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513094.467947:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513094.467951:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.467951:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513094.467952:0:51001:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513094.468006:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.468013:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513094.468016:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513094.468018:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513094.468022:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513094.468025:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.468028:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513094.468028:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c2a8 00000800:00000200:2.0:1616513094.468030:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513094.468031:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513094.468034:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513094.468034:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513094.468038:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae220 00000800:00000200:18.0:1616513094.468042:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.480264:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513094.480319:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513094.480328:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:28.0:1616513094.480337:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.178@tcp1 : PUT - for me 00000400:00000200:28.0:1616513094.480347:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.178@tcp1 of length 224 into portal 26 MB=0x604a366be4d40 00000400:00000200:28.0:1616513094.480353:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.178@tcp1 of length 224/224 into md 0xf97cef69 [1] + 0 00000400:00000200:28.0:1616513094.480358:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfa08 00000400:00000200:28.0:1616513094.480361:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513094.480364:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513094.480368:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfa08 00000800:00000200:28.0:1616513094.480371:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.480372:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513094.480374:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513094.480376:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513094.480430:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949710191936 00010000:00000200:0.0:1616513094.480451:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1693949710191936/t0(0) o400->ee7ca7d7-512f-c07b-e14b-fff5afb1adb5@10.31.140.178@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.480466:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949710191936, offset 192 00000400:00000200:0.0:1616513094.480472:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.178@tcp1 00000400:00000200:0.0:1616513094.480480:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.178@tcp1 routed destination 00000400:00000200:0.0:1616513094.480496:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.178@tcp1(10.31.140.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.480502:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.480508:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.480511:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.480513:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.480518:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.480522:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.480525:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513094.480575:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513094.480577:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513094.480582:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513094.480585:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513094.480591:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513094.480597:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c908 00000800:00000200:16.0:1616513094.480600:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.480603:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513094.480632:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.480641:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513094.480650:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.223@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.480660:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.223@tcp1 of length 224 into portal 12 MB=0x6048d68e38e40 00000400:00000200:14.0:1616513094.480666:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.223@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 14896 00000400:00000200:14.0:1616513094.480671:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.480674:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513094.480680:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.480683:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.480685:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.480689:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.480710:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513094.480728:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.480736:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:28.0:1616513094.480738:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513094.480745:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.178@tcp1 : PUT - for me 00000400:00000200:12.0:1616513094.480753:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.178@tcp1 of length 224 into portal 12 MB=0x604a366be5140 00000100:00000200:26.0:1616513094.480754:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855256907328 00000400:00000200:12.0:1616513094.480762:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.178@tcp1 of length 224/224 into md 0x865be291 [1] + 50384 00010000:00000200:26.0:1616513094.480766:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e00000 x1693855256907328/t0(0) o400->18dc9e86-a745-4d02-5806-f1c7728f1c21@10.31.140.223@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513094.480767:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:11.2:1616513094.480768:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513094.480770:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513094.480776:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.480777:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.480779:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:26.0:1616513094.480781:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855256907328, offset 192 00000800:00000200:12.0:1616513094.480781:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513094.480783:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513094.480783:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513094.480787:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.223@tcp1 00000400:00000200:16.0:1616513094.480791:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.223@tcp1 : PUT - for me 00000400:00000200:26.0:1616513094.480794:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.223@tcp1 routed destination 00000400:00000200:16.0:1616513094.480799:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.223@tcp1 of length 224 into portal 26 MB=0x6048d68e38c40 00000400:00000200:16.0:1616513094.480806:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.223@tcp1 of length 224/224 into md 0xf97cef99 [1] + 0 00000400:00000200:26.0:1616513094.480809:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.223@tcp1(10.31.140.223@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513094.480811:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be220 00000400:00000200:16.0:1616513094.480813:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513094.480816:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:16.0:1616513094.480816:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513094.480821:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be220 00000800:00000200:26.0:1616513094.480822:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513094.480823:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513094.480825:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513094.480826:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513094.480827:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513094.480828:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513094.480832:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513094.480833:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855256906816 00000800:00000200:7.2:1616513094.480836:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:0.0:1616513094.480838:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1693855256906816/t0(0) o400->7baf055e-a2d1-e8dc-fc44-3f5aa966d7ed@10.31.140.223@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513094.480841:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949710192960 00000100:00000200:0.0:1616513094.480848:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855256906816, offset 192 00010000:00000200:24.0:1616513094.480853:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d205580 x1693949710192960/t0(0) o400->47b11ab0-c40d-2bfa-1ae0-75e72861e043@10.31.140.178@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513094.480853:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.223@tcp1 00000400:00000200:0.0:1616513094.480858:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.223@tcp1 routed destination 00000100:00000200:24.0:1616513094.480867:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949710192960, offset 192 00000400:00000200:24.0:1616513094.480873:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.178@tcp1 00000400:00000200:0.0:1616513094.480873:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.223@tcp1(10.31.140.223@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.480878:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513094.480881:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.178@tcp1 routed destination 00000800:00000200:0.0:1616513094.480888:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513094.480890:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513094.480890:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.480892:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.480895:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513094.480896:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.178@tcp1(10.31.140.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513094.480897:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513094.480899:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513094.480899:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:0.0:1616513094.480899:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513094.480903:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:2.0:1616513094.480905:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513094.480908:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513094.480909:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513094.480909:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae770 00000800:00000200:28.0:1616513094.480912:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513094.480912:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.480913:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513094.480914:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513094.480914:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513094.480915:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513094.480916:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513094.480919:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513094.480920:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513094.480923:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513094.480926:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922caa0 00000800:00000200:28.0:1616513094.480929:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513094.480935:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.480940:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513094.480943:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513094.480944:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513094.480949:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.480953:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2220 00000800:00000200:22.0:1616513094.480957:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513094.506483:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928912, portal 4 00000100:00000200:10.0:1616513094.506489:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928912, offset 0 00000400:00000200:10.0:1616513094.506496:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:10.0:1616513094.506503:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:10.0:1616513094.506516:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513094.506523:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:10.0:1616513094.506529:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513094.506532:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:10.0:1616513094.506534:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:10.0:1616513094.506540:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:3.0:1616513094.506575:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928928, portal 4 00000100:00000200:3.0:1616513094.506580:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928928, offset 0 00000400:00000200:3.0:1616513094.506587:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:27.2:1616513094.506594:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:3.0:1616513094.506594:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:3.0:1616513094.506607:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513094.506613:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:3.0:1616513094.506614:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513094.506619:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:3.0:1616513094.506621:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:27.0:1616513094.506622:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513094.506624:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:3.0:1616513094.506626:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:27.0:1616513094.506629:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:3.0:1616513094.506632:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (24)-- 00000100:00000200:23.0:1616513094.506633:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928944, portal 4 00000100:00000200:27.0:1616513094.506636:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9645100 x1693067725928912/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513094.506639:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928944, offset 0 00000400:00000200:23.0:1616513094.506645:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:27.0:1616513094.506648:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83dc0 00000400:00000200:23.0:1616513094.506652:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:27.0:1616513094.506658:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513094.506660:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513094.506664:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:23.0:1616513094.506665:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:27.0:1616513094.506666:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513094.506667:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513094.506671:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:23.0:1616513094.506672:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000100:00000200:27.0:1616513094.506676:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be42880 x1693067725928928/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:23.0:1616513094.506679:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:23.0:1616513094.506682:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:23.0:1616513094.506684:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000400:00000200:27.0:1616513094.506685:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd3210 00000800:00000200:27.0:1616513094.506688:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:23.0:1616513094.506690:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513094.506736:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513094.506751:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513094.506759:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513094.506762:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.506769:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513094.506775:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdd639680 x1693067725928944/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.2:1616513094.506781:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513094.506791:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.506797:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000400:00000200:2.0:1616513094.506797:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3f870 00000800:00000200:31.0:1616513094.506800:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:2.0:1616513094.506801:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513094.506804:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000400:00000200:27.0:1616513094.506805:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:31.0:1616513094.506809:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:3.0:1616513094.506810:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513094.506812:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6279e0 00000400:00000200:31.0:1616513094.506818:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513094.506819:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716726d [1] + 192 00000400:00000200:27.0:1616513094.506824:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:31.0:1616513094.506826:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c6279d0 00000100:00000200:27.0:1616513094.506828:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be42880 x1693067725928928/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513094.506841:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:31.0:1616513094.506843:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716725d [1] + 192 00000800:00000200:27.0:1616513094.506843:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513094.506845:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:31.0:1616513094.506848:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:3.0:1616513094.506850:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be42880 x1693067725928928/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513094.506863:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be42880 x1693067725928928/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:25.2:1616513094.506871:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000400:00000200:3.0:1616513094.506873:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000100:00000200:31.0:1616513094.506878:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9645100 x1693067725928912/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513094.506900:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513094.506903:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:31.0:1616513094.506905:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:10.0:1616513094.506910:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9645100 x1693067725928912/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513094.506923:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9645100 x1693067725928912/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513094.506933:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82c38 00000800:00000200:18.0:1616513094.506950:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513094.506959:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513094.506969:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513094.506977:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c6279f0 00000400:00000200:18.0:1616513094.506984:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cf6a9 [1] + 192 00000400:00000200:18.0:1616513094.506989:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.506994:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdd639680 x1693067725928944/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513094.507005:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513094.507010:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513094.507013:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513094.507015:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:23.0:1616513094.507019:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdd639680 x1693067725928944/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:23.0:1616513094.507032:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdd639680 x1693067725928944/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:23.0:1616513094.507042:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fb18 00000800:00000200:7.2:1616513094.536168:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513094.536227:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.536238:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513094.536247:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.4@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.536257:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.4@tcp1 of length 224 into portal 26 MB=0x60182bb1ff140 00000400:00000200:14.0:1616513094.536264:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.4@tcp1 of length 224/224 into md 0xf97cefa9 [1] + 0 00000400:00000200:14.0:1616513094.536271:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf870 00000400:00000200:14.0:1616513094.536274:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.536277:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513094.536281:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf870 00000800:00000200:2.0:1616513094.536283:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.536285:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.536288:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513094.536291:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513094.536294:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.536295:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513094.536303:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.4@tcp1 : PUT - for me 00000100:00000200:0.0:1616513094.536308:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510857072960 00000400:00000200:2.0:1616513094.536312:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.4@tcp1 of length 224 into portal 12 MB=0x60182bb1ff280 00000400:00000200:2.0:1616513094.536319:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.4@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 15120 00010000:00000200:0.0:1616513094.536319:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1690510857072960/t0(0) o400->ce9a659a-f22c-b04c-9b49-cf197bf14e74@10.31.141.4@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513094.536324:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513094.536327:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:0.0:1616513094.536334:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510857072960, offset 192 00000800:00000200:2.0:1616513094.536335:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.536339:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.536341:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513094.536341:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.4@tcp1 00000400:00000200:0.0:1616513094.536350:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.4@tcp1 routed destination 00000400:00000200:0.0:1616513094.536364:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.4@tcp1(10.31.141.4@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.536372:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.536377:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.536380:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.536382:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.536387:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.536394:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.536394:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513094.536419:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510857073280 00010000:00000200:26.0:1616513094.536431:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5e80 x1690510857073280/t0(0) o400->f88d852a-9147-af69-f3bd-78d4c4342cad@10.31.141.4@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513094.536457:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:26.0:1616513094.536459:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510857073280, offset 192 00000800:00000200:20.0:1616513094.536464:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513094.536466:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.4@tcp1 00000400:00000200:20.0:1616513094.536467:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513094.536472:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.4@tcp1 routed destination 00000400:00000200:20.0:1616513094.536474:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513094.536479:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c660 00000800:00000200:20.0:1616513094.536483:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513094.536486:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513094.536488:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.4@tcp1(10.31.141.4@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.536494:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.536500:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.536502:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.536504:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.536509:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.536514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.536566:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.536574:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.536577:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513094.536578:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513094.536583:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.536587:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae110 00000800:00000200:18.0:1616513094.536592:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.559148:0:51012:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513094.559205:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.559215:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513094.559225:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.38@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.559235:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.38@tcp1 of length 224 into portal 26 MB=0x604a353a99700 00000400:00000200:22.0:1616513094.559242:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.38@tcp1 of length 224/224 into md 0xf97cefc9 [1] + 0 00000400:00000200:22.0:1616513094.559247:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf320 00000400:00000200:22.0:1616513094.559250:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.559253:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513094.559258:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf320 00000800:00000200:2.0:1616513094.559260:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.559261:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.559264:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.559266:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.559326:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949390067456 00010000:00000200:0.0:1616513094.559337:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1693949390067456/t0(0) o400->42dcd879-73bf-2a9b-3b34-452656b73924@10.31.141.38@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.559352:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949390067456, offset 192 00000400:00000200:0.0:1616513094.559359:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.38@tcp1 00000400:00000200:0.0:1616513094.559367:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.38@tcp1 routed destination 00000400:00000200:0.0:1616513094.559382:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.38@tcp1(10.31.141.38@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.559389:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.559395:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.559399:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.559401:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.559406:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.559413:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.559413:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513094.559467:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.559474:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.559477:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513094.559484:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.559489:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cbb0 00000800:00000200:12.0:1616513094.559493:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.559496:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513094.559544:0:51012:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513094.559598:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.559609:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513094.559612:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513094.559617:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.38@tcp1 : PUT - for me 00000400:00000200:14.0:1616513094.559626:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.38@tcp1 of length 224 into portal 12 MB=0x604a353a997c0 00000400:00000200:14.0:1616513094.559632:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.38@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 15344 00000400:00000200:14.0:1616513094.559637:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.559640:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513094.559647:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.559650:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.559652:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513094.559724:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949390067648 00010000:00000200:26.0:1616513094.559737:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07500 x1693949390067648/t0(0) o400->2f57e993-ad66-d653-7661-fc6dab459373@10.31.141.38@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.559752:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949390067648, offset 192 00000400:00000200:26.0:1616513094.559758:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.38@tcp1 00000400:00000200:26.0:1616513094.559766:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.38@tcp1 routed destination 00000400:00000200:26.0:1616513094.559780:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.38@tcp1(10.31.141.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.559787:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.559793:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.559795:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.559798:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.559802:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.559807:0:51012:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.559818:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.559824:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.559826:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.559828:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513094.559833:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.559837:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af7e8 00000800:00000200:2.0:1616513094.559841:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.565291:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513094.565345:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513094.565354:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513094.565363:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.20@tcp1 : PUT - for me 00000400:00000200:20.0:1616513094.565373:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.20@tcp1 of length 224 into portal 26 MB=0x60393e91eb500 00000400:00000200:20.0:1616513094.565379:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.20@tcp1 of length 224/224 into md 0xf97cefe9 [1] + 0 00000400:00000200:20.0:1616513094.565384:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be5d8 00000400:00000200:20.0:1616513094.565387:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.565391:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513094.565396:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be5d8 00000800:00000200:20.0:1616513094.565399:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.565399:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.565402:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.565406:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.565409:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.565410:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513094.565418:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.20@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.565427:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.20@tcp1 of length 224 into portal 12 MB=0x60393e91eb600 00000400:00000200:16.0:1616513094.565453:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.20@tcp1 of length 224/224 into md 0x865be291 [1] + 50608 00000400:00000200:16.0:1616513094.565458:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513094.565459:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783666443520 00000100:00000200:16.0:1616513094.565461:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513094.565468:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:0.0:1616513094.565469:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1692783666443520/t0(0) o400->3db00cff-60ba-a80b-2a82-e5fb9cb7bdc3@10.31.140.20@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513094.565471:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.565473:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513094.565482:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783666443520, offset 192 00000400:00000200:0.0:1616513094.565488:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.20@tcp1 00000400:00000200:0.0:1616513094.565495:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.20@tcp1 routed destination 00000400:00000200:0.0:1616513094.565509:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.20@tcp1(10.31.140.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.565515:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.565520:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.565523:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.565525:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.565529:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.565534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.565535:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513094.565538:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783666443776 00010000:00000200:24.0:1616513094.565550:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6721200 x1692783666443776/t0(0) o400->4179e2ca-ba75-38d0-0032-d070c21a69a3@10.31.140.20@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.565564:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783666443776, offset 192 00000400:00000200:24.0:1616513094.565571:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.20@tcp1 00000400:00000200:24.0:1616513094.565578:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.20@tcp1 routed destination 00000800:00000200:28.0:1616513094.565589:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513094.565593:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.20@tcp1(10.31.140.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513094.565596:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513094.565599:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513094.565599:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513094.565601:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513094.565605:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513094.565606:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:24.0:1616513094.565608:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.565610:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513094.565611:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c5d8 00000800:00000200:28.0:1616513094.565615:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513094.565615:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.565622:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.565676:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.565683:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.565686:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.565692:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.565696:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2110 00000800:00000200:18.0:1616513094.565700:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.565703:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.595576:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513094.595633:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.595652:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513094.595661:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.59@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.595671:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.59@tcp1 of length 224 into portal 26 MB=0x6018445e6abc0 00000400:00000200:22.0:1616513094.595678:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.59@tcp1 of length 224/224 into md 0xf97cf009 [1] + 0 00000400:00000200:22.0:1616513094.595683:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02becc0 00000400:00000200:22.0:1616513094.595685:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.595688:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:11.2:1616513094.595691:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513094.595693:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02becc0 00000800:00000200:22.0:1616513094.595697:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.595698:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.595700:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.595702:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513094.595762:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.595763:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517480319936 00000800:00000200:12.0:1616513094.595771:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00010000:00000200:0.0:1616513094.595775:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1690517480319936/t0(0) o400->291f73dd-b7d3-51f0-06fd-f01b5d8e5892@10.31.142.59@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:12.0:1616513094.595781:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.59@tcp1 : PUT - for me 00000100:00000200:0.0:1616513094.595789:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690517480319936, offset 192 00000400:00000200:12.0:1616513094.595790:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.59@tcp1 of length 224 into portal 12 MB=0x6018445e6ac40 00000400:00000200:0.0:1616513094.595796:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.59@tcp1 00000400:00000200:12.0:1616513094.595801:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.59@tcp1 of length 224/224 into md 0x865be291 [1] + 50832 00000400:00000200:0.0:1616513094.595804:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.59@tcp1 routed destination 00000400:00000200:12.0:1616513094.595807:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.595809:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513094.595819:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.59@tcp1(10.31.142.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513094.595825:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513094.595826:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513094.595828:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.595828:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.595832:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:0.0:1616513094.595832:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.595834:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.595837:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.595842:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.595846:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.595848:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:24.0:1616513094.595880:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517480320064 00010000:00000200:24.0:1616513094.595893:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d201f80 x1690517480320064/t0(0) o400->c945ea9e-c58e-cb86-15ad-615a7435a9f6@10.31.142.59@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513094.595900:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513094.595907:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690517480320064, offset 192 00000800:00000200:20.0:1616513094.595908:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513094.595911:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513094.595913:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513094.595915:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.59@tcp1 00000400:00000200:20.0:1616513094.595918:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513094.595923:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.59@tcp1 routed destination 00000400:00000200:20.0:1616513094.595923:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d760 00000800:00000200:20.0:1616513094.595928:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513094.595937:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.59@tcp1(10.31.142.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.595944:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.595950:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.595953:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.595955:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.595960:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.595965:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.595977:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.595984:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.595987:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.595993:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.595997:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2d48 00000800:00000200:18.0:1616513094.596001:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.596004:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.598305:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513094.598359:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.598368:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513094.598377:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.219@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.598386:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.219@tcp1 of length 224 into portal 26 MB=0x5ffdb9609e000 00000400:00000200:16.0:1616513094.598393:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.219@tcp1 of length 224/224 into md 0xf97cf019 [1] + 0 00000400:00000200:16.0:1616513094.598398:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be880 00000400:00000200:16.0:1616513094.598401:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.598403:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513094.598408:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be880 00000800:00000200:16.0:1616513094.598410:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.598413:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.598414:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513094.598418:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513094.598471:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693463703552 00010000:00000200:0.0:1616513094.598482:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688693463703552/t0(0) o400->28f9a612-a440-86e5-89e7-837644952324@10.31.140.219@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.598496:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693463703552, offset 192 00000400:00000200:0.0:1616513094.598502:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.219@tcp1 00000400:00000200:0.0:1616513094.598510:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.219@tcp1 routed destination 00000400:00000200:0.0:1616513094.598525:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.219@tcp1(10.31.140.219@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.598531:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.598537:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.598540:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.598542:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.598547:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.598552:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.598553:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513094.598567:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513094.598571:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513094.598573:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513094.598578:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513094.598582:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922dba0 00000800:00000200:0.0:1616513094.598584:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513094.598587:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.598593:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513094.598595:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513094.598600:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.219@tcp1 : PUT - for me 00000400:00000200:0.0:1616513094.598607:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.219@tcp1 of length 224 into portal 12 MB=0x5ffdb9609e2c0 00000400:00000200:0.0:1616513094.598612:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.219@tcp1 of length 224/224 into md 0x865be291 [1] + 51056 00000400:00000200:0.0:1616513094.598617:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513094.598619:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513094.598625:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.598627:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513094.598629:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513094.598693:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693463704256 00010000:00000200:24.0:1616513094.598706:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde869680 x1688693463704256/t0(0) o400->ef8be3f9-7695-a420-4841-6e40d265df59@10.31.140.219@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.598720:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693463704256, offset 192 00000400:00000200:24.0:1616513094.598726:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.219@tcp1 00000400:00000200:24.0:1616513094.598733:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.219@tcp1 routed destination 00000400:00000200:24.0:1616513094.598748:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.219@tcp1(10.31.140.219@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.598755:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.598760:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.598763:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.598765:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.598770:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.598775:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.598829:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.598836:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.598839:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.598845:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.598849:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f36d8 00000800:00000200:2.0:1616513094.598853:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.598856:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.617300:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513094.617358:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.617367:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513094.617376:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.96@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.617386:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.96@tcp1 of length 224 into portal 26 MB=0x5ffdd16e72a00 00000400:00000200:16.0:1616513094.617392:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.96@tcp1 of length 224/224 into md 0xf97cf029 [1] + 0 00000400:00000200:16.0:1616513094.617397:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be4c8 00000400:00000200:16.0:1616513094.617400:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.617403:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513094.617407:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be4c8 00000800:00000200:16.0:1616513094.617410:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.617412:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.617413:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513094.617419:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513094.617422:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.617422:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513094.617431:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.96@tcp1 : PUT - for me 00000400:00000200:12.0:1616513094.617456:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.96@tcp1 of length 224 into portal 12 MB=0x5ffdd16e72dc0 00000400:00000200:12.0:1616513094.617462:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.96@tcp1 of length 224/224 into md 0x865be291 [1] + 51280 00000100:00000200:0.0:1616513094.617471:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699920656896 00000400:00000200:12.0:1616513094.617472:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.617475:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513094.617481:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:0.0:1616513094.617482:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688699920656896/t0(0) o400->028b26e5-2c63-a7ee-a4b4-c4b15f6b6b0a@10.31.142.96@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:12.0:1616513094.617484:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.617486:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513094.617498:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699920656896, offset 192 00000400:00000200:0.0:1616513094.617504:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.96@tcp1 00000400:00000200:0.0:1616513094.617511:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.96@tcp1 routed destination 00000400:00000200:0.0:1616513094.617526:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.96@tcp1(10.31.142.96@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.617533:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.617539:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.617541:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.617544:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.617549:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513094.617552:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699920657856 00000800:00000200:11.2:1616513094.617554:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.617555:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:24.0:1616513094.617564:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d202880 x1688699920657856/t0(0) o400->d97d47e4-a269-96f9-bdf3-864fd54685c2@10.31.142.96@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513094.617571:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513094.617576:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513094.617578:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:24.0:1616513094.617579:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699920657856, offset 192 00000800:00000200:16.0:1616513094.617580:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513094.617583:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513094.617585:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.96@tcp1 00000400:00000200:0.0:1616513094.617589:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922dba0 00000400:00000200:24.0:1616513094.617593:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.96@tcp1 routed destination 00000800:00000200:0.0:1616513094.617598:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513094.617607:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.96@tcp1(10.31.142.96@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.617614:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.617620:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.617624:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.617626:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.617631:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.617640:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.617655:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.617661:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.617664:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.617670:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.617674:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3cb0 00000800:00000200:18.0:1616513094.617678:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.617680:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513094.618482:0:6487:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928960, portal 4 00000100:00000200:24.0:1616513094.618486:0:6487:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928960, offset 0 00000400:00000200:24.0:1616513094.618490:0:6487:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:24.0:1616513094.618494:0:6487:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:24.0:1616513094.618505:0:6487:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.618511:0:6487:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:24.0:1616513094.618515:0:6487:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.618517:0:6487:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:24.0:1616513094.618519:0:6487:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513094.618522:0:6487:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513094.618569:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513094.618584:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513094.618591:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513094.618594:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513094.618596:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:20.0:1616513094.618601:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513094.618607:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affc0480 x1693067725928960/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513094.618619:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2198 00000800:00000200:20.0:1616513094.618623:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513094.618743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513094.618807:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513094.618816:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:16.0:1616513094.618825:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513094.618833:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627a00 00000400:00000200:16.0:1616513094.618839:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dc061 [1] + 192 00000400:00000200:16.0:1616513094.618844:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.618850:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affc0480 x1693067725928960/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:0.0:1616513094.618859:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513094.618864:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513094.618867:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513094.618869:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:24.0:1616513094.618870:0:6487:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affc0480 x1693067725928960/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:24.0:1616513094.618881:0:6487:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affc0480 x1693067725928960/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513094.618890:0:6487:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3dc0 00000800:00000200:11.2:1616513094.638594:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513094.638609:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.638617:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513094.638625:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.150@tcp1 : PUT - for me 00000400:00000200:12.0:1616513094.638633:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.150@tcp1 of length 224 into portal 26 MB=0x5ffd752370b40 00000400:00000200:12.0:1616513094.638640:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.150@tcp1 of length 224/224 into md 0xf97cf039 [1] + 0 00000400:00000200:12.0:1616513094.638644:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bec38 00000400:00000200:12.0:1616513094.638647:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.638649:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513094.638653:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bec38 00000800:00000200:12.0:1616513094.638656:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.638659:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.638660:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.638663:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513094.638716:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675145943872 00010000:00000200:0.0:1616513094.638726:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688675145943872/t0(0) o400->f32531ad-b02a-657c-ace7-cdb33ee96b0b@10.31.129.150@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.638739:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688675145943872, offset 192 00000400:00000200:0.0:1616513094.638745:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.150@tcp1 00000400:00000200:0.0:1616513094.638752:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.150@tcp1 routed destination 00000400:00000200:0.0:1616513094.638766:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.150@tcp1(10.31.129.150@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.638772:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.638777:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.638780:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.638782:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.638786:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513094.638790:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513094.638792:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513094.638794:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513094.638807:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513094.638811:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513094.638813:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513094.638818:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513094.638821:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922dba0 00000800:00000200:0.0:1616513094.638824:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513094.638826:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513094.638847:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.638857:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513094.638866:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.150@tcp1 : PUT - for me 00000400:00000200:2.0:1616513094.638874:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.150@tcp1 of length 224 into portal 12 MB=0x5ffd752371040 00000400:00000200:2.0:1616513094.638881:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.150@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 15568 00000400:00000200:2.0:1616513094.638888:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513094.638891:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513094.638898:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.638901:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.638902:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513094.638906:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513094.638921:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675145945152 00010000:00000200:26.0:1616513094.638933:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4c80 x1688675145945152/t0(0) o400->cf9bbd8f-51b3-a26f-7936-270b3864a919@10.31.129.150@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513094.638948:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688675145945152, offset 192 00000400:00000200:26.0:1616513094.638954:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.150@tcp1 00000400:00000200:26.0:1616513094.638961:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.150@tcp1 routed destination 00000400:00000200:26.0:1616513094.638976:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.150@tcp1(10.31.129.150@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.638983:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.638988:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.638991:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.638995:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.638999:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.639004:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.639013:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.639019:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.639021:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.639027:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.639030:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae550 00000800:00000200:18.0:1616513094.639034:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.639036:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.650391:0:51022:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513094.650455:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.650471:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513094.650480:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.230@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.650489:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.230@tcp1 of length 224 into portal 26 MB=0x604f603c9e8c0 00000400:00000200:22.0:1616513094.650496:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.230@tcp1 of length 224/224 into md 0xf97cf059 [1] + 0 00000400:00000200:22.0:1616513094.650500:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02beb28 00000400:00000200:22.0:1616513094.650503:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.650506:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513094.650510:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beb28 00000800:00000200:22.0:1616513094.650513:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.650516:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513094.650518:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513094.650525:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.650528:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.650528:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513094.650537:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.230@tcp1 : PUT - for me 00000100:00000200:10.0:1616513094.650546:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928976, portal 4 00000400:00000200:2.0:1616513094.650546:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.230@tcp1 of length 224 into portal 12 MB=0x604f603c9e940 00000400:00000200:2.0:1616513094.650552:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.230@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 15792 00000100:00000200:10.0:1616513094.650557:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928976, offset 0 00000400:00000200:2.0:1616513094.650557:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513094.650560:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513094.650566:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:10.0:1616513094.650569:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:2.0:1616513094.650570:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.650572:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:10.0:1616513094.650574:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000100:00000200:0.0:1616513094.650574:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304532293824 00010000:00000200:0.0:1616513094.650585:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1694304532293824/t0(0) o400->f3ed97b4-e870-1624-f107-77ac40115765@10.31.140.230@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:10.0:1616513094.650588:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513094.650594:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000100:00000200:0.0:1616513094.650598:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694304532293824, offset 192 00000800:00000200:10.0:1616513094.650601:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513094.650604:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:0.0:1616513094.650604:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.230@tcp1 00000800:00000200:10.0:1616513094.650606:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:0.0:1616513094.650610:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.230@tcp1 routed destination 00000800:00000200:10.0:1616513094.650611:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:0.0:1616513094.650624:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.230@tcp1(10.31.140.230@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513094.650630:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304532293952 00000800:00000200:0.0:1616513094.650630:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.650636:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.650638:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.650640:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:26.0:1616513094.650642:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0900 x1694304532293952/t0(0) o400->6bd87081-7695-e11a-8c1f-38a79ce82dc3@10.31.140.230@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513094.650644:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:0.0:1616513094.650651:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:26.0:1616513094.650657:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694304532293952, offset 192 00000800:00000200:1.2:1616513094.650660:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:26.0:1616513094.650663:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.230@tcp1 00000400:00000200:26.0:1616513094.650671:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.230@tcp1 routed destination 00000800:00000200:1.0:1616513094.650680:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:26.0:1616513094.650685:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.230@tcp1(10.31.140.230@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:1.0:1616513094.650686:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513094.650689:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513094.650690:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513094.650692:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:5.0:1616513094.650692:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513094.650697:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:26.0:1616513094.650698:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.650701:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513094.650701:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513094.650703:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513094.650705:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:1.0:1616513094.650705:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9644800 x1693067725928976/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513094.650707:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513094.650708:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.650712:0:51022:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513094.650713:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.650716:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922dba0 00000400:00000200:1.0:1616513094.650718:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82660 00000800:00000200:12.0:1616513094.650719:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.650721:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513094.650724:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:1.0:1616513094.650724:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:14.0:1616513094.650732:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513094.650734:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.650736:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513094.650741:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513094.650745:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af4b8 00000800:00000200:14.0:1616513094.650749:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:1.2:1616513094.650792:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513094.650802:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513094.650809:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513094.650810:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513094.650818:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513094.650827:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627a10 00000400:00000200:1.0:1616513094.650834:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x437000a5 [1] + 192 00000400:00000200:1.0:1616513094.650839:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513094.650844:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9644800 x1693067725928976/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513094.650859:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513094.650862:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513094.650864:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:10.0:1616513094.650868:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9644800 x1693067725928976/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513094.650881:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9644800 x1693067725928976/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513094.650891:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82c38 00000100:00000200:22.0:1616513094.666606:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725928992, portal 4 00000100:00000200:22.0:1616513094.666611:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725928992, offset 0 00000400:00000200:22.0:1616513094.666618:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:22.0:1616513094.666624:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:22.0:1616513094.666637:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513094.666644:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:22.0:1616513094.666652:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513094.666655:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:22.0:1616513094.666657:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:22.0:1616513094.666662:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513094.666711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.666733:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513094.666738:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513094.666741:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513094.666747:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513094.666756:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5dc7e07980 x1693067725928992/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513094.666769:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b980 00000800:00000200:27.0:1616513094.666773:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513094.666776:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513094.666878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513094.666887:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513094.666893:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513094.666900:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513094.666908:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627a20 00000400:00000200:27.0:1616513094.666915:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716729d [1] + 192 00000400:00000200:27.0:1616513094.666920:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513094.666924:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5dc7e07980 x1693067725928992/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513094.666939:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513094.666939:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513094.666942:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513094.666944:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:22.0:1616513094.666949:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5dc7e07980 x1693067725928992/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513094.666962:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5dc7e07980 x1693067725928992/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513094.666973:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aee0 00000800:00000200:7.2:1616513094.669951:0:51024:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.669962:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.669971:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.669978:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.151@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.669986:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.151@tcp1 of length 224 into portal 26 MB=0x5ffdfb8446b40 00000400:00000200:18.0:1616513094.669992:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.151@tcp1 of length 224/224 into md 0xf97cf069 [1] + 0 00000400:00000200:18.0:1616513094.669996:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bff58 00000400:00000200:18.0:1616513094.670000:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.670002:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513094.670006:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bff58 00000800:00000200:18.0:1616513094.670009:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.670011:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.670012:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.670014:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.670018:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.670024:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.151@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.670029:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.151@tcp1 of length 224 into portal 12 MB=0x5ffdfb8446c00 00000400:00000200:18.0:1616513094.670033:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.151@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 16016 00000400:00000200:18.0:1616513094.670037:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.670039:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.670043:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.670045:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.670046:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.670048:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513094.670066:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711217834816 00010000:00000200:0.0:1616513094.670076:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688711217834816/t0(0) o400->e45fd0e6-8267-c0bc-4b2b-bf36e42fca05@10.31.141.151@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513094.670088:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688711217834816, offset 192 00000400:00000200:0.0:1616513094.670093:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.151@tcp1 00000400:00000200:0.0:1616513094.670099:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.151@tcp1 routed destination 00000100:00000200:26.0:1616513094.670110:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688711217835008 00000400:00000200:0.0:1616513094.670112:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.151@tcp1(10.31.141.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.670118:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:26.0:1616513094.670123:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd705580 x1688711217835008/t0(0) o400->565b089f-a256-9087-f7a3-7c6c7e3f8d02@10.31.141.151@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513094.670123:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513094.670125:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.670128:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.670132:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:26.0:1616513094.670137:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688711217835008, offset 192 00000100:00000200:0.0:1616513094.670138:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513094.670144:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.151@tcp1 00000400:00000200:26.0:1616513094.670151:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.151@tcp1 routed destination 00000400:00000200:26.0:1616513094.670166:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.151@tcp1(10.31.141.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.670172:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.670178:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:11.2:1616513094.670178:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513094.670181:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.670183:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.670188:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513094.670191:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513094.670195:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513094.670195:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513094.670197:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513094.670202:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513094.670205:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cd48 00000800:00000200:0.0:1616513094.670207:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513094.670210:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513094.670264:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.670271:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513094.670274:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513094.670277:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513094.670281:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513094.670285:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afdc0 00000800:00000200:14.0:1616513094.670289:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:1.0:1616513094.683153:0:6493:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929008, portal 4 00000100:00000200:1.0:1616513094.683159:0:6493:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929008, offset 0 00000400:00000200:1.0:1616513094.683166:0:6493:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:1.0:1616513094.683173:0:6493:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:1.0:1616513094.683187:0:6493:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513094.683190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:1.0:1616513094.683194:0:6493:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:1.0:1616513094.683203:0:6493:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513094.683206:0:6493:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:1.0:1616513094.683208:0:6493:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:1.0:1616513094.683214:0:6493:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513094.683222:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513094.683246:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.683257:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513094.683266:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.65@tcp1 : PUT - for me 00000400:00000200:12.0:1616513094.683276:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.65@tcp1 of length 224 into portal 26 MB=0x5ffdd99de1300 00000800:00000200:20.0:1616513094.683278:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:12.0:1616513094.683282:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.65@tcp1 of length 224/224 into md 0xf97cf0a9 [1] + 0 00000800:00000200:20.0:1616513094.683286:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513094.683287:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf100 00000400:00000200:20.0:1616513094.683289:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513094.683290:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.683295:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513094.683296:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.683301:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf100 00000100:00000200:20.0:1616513094.683303:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1392d00 x1693067725929008/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513094.683304:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513094.683305:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513094.683310:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:20.0:1616513094.683322:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84d48 00000800:00000200:12.0:1616513094.683324:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.683325:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:20.0:1616513094.683328:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:7.2:1616513094.683359:0:51025:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.683369:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.683374:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:4.0:1616513094.683377:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702117876480 00000400:00000200:18.0:1616513094.683380:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.65@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.683386:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.65@tcp1 of length 224 into portal 12 MB=0x5ffdd99de1640 00010000:00000200:4.0:1616513094.683388:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688702117876480/t0(0) o400->07c05088-3296-d930-d526-462bcc46f37b@10.31.142.65@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513094.683390:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.65@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 16240 00000400:00000200:18.0:1616513094.683393:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.683395:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.683398:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.683400:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.683401:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.683403:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513094.683403:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702117876480, offset 192 00000400:00000200:4.0:1616513094.683409:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.65@tcp1 00000400:00000200:4.0:1616513094.683417:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.65@tcp1 routed destination 00000800:00000200:5.2:1616513094.683430:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000400:00000200:4.0:1616513094.683431:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.65@tcp1(10.31.142.65@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513094.683445:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513094.683453:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:12.0:1616513094.683455:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:4.0:1616513094.683456:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513094.683462:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:4.0:1616513094.683463:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513094.683465:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513094.683468:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513094.683469:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702117877312 00000400:00000200:0.0:1616513094.683470:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627a30 00000800:00000200:4.0:1616513094.683473:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513094.683476:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dc0b1 [1] + 192 00000800:00000200:11.2:1616513094.683478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513094.683480:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:26.0:1616513094.683482:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd706780 x1688702117877312/t0(0) o400->7c5493a0-2da1-a983-52f5-6ecbc16b057c@10.31.142.65@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513094.683482:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513094.683487:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1392d00 x1693067725929008/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513094.683488:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.683492:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513094.683494:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:26.0:1616513094.683496:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702117877312, offset 192 00000800:00000200:16.0:1616513094.683498:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513094.683499:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513094.683502:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:20.0:1616513094.683503:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb320 00000400:00000200:26.0:1616513094.683505:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.65@tcp1 00000800:00000200:0.0:1616513094.683505:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513094.683507:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:20.0:1616513094.683509:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513094.683512:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.65@tcp1 routed destination 00000100:00000200:1.0:1616513094.683514:0:6493:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1392d00 x1693067725929008/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513094.683527:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.65@tcp1(10.31.142.65@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:1.0:1616513094.683527:0:6493:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1392d00 x1693067725929008/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513094.683533:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:1.0:1616513094.683538:0:6493:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85a08 00000800:00000200:26.0:1616513094.683539:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.683542:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.683544:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.683549:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.683553:0:51025:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.683610:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.683617:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.683620:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.683626:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.683630:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af870 00000800:00000200:2.0:1616513094.683634:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.683637:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.707883:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.707940:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.707950:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.707959:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.63@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.707968:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.63@tcp1 of length 224 into portal 12 MB=0x5ffe2b33bf1c0 00000400:00000200:18.0:1616513094.707974:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.63@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 16464 00000400:00000200:18.0:1616513094.707981:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.707984:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.707991:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.707995:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.707995:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.707999:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.708013:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:26.0:1616513094.708057:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724018295232 00010000:00000200:26.0:1616513094.708069:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd700d80 x1688724018295232/t0(0) o400->dc42ba01-1665-cc3f-d0e6-e84feb1d3cd8@10.31.140.63@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513094.708070:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.708080:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000100:00000200:26.0:1616513094.708082:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688724018295232, offset 192 00000400:00000200:26.0:1616513094.708089:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.63@tcp1 00000400:00000200:12.0:1616513094.708089:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.63@tcp1 : PUT - for me 00000400:00000200:26.0:1616513094.708096:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.63@tcp1 routed destination 00000400:00000200:12.0:1616513094.708098:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.63@tcp1 of length 224 into portal 26 MB=0x5ffe2b33bef80 00000400:00000200:12.0:1616513094.708104:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.63@tcp1 of length 224/224 into md 0xf97cf0c9 [1] + 0 00000400:00000200:12.0:1616513094.708109:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf7e8 00000400:00000200:26.0:1616513094.708110:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.63@tcp1(10.31.140.63@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513094.708112:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.708115:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513094.708116:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513094.708119:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf7e8 00000800:00000200:26.0:1616513094.708121:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513094.708122:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513094.708124:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.708124:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513094.708125:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:12.0:1616513094.708125:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.708128:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513094.708130:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.708135:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513094.708183:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724018294656 00000800:00000200:0.0:1616513094.708192:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:4.0:1616513094.708194:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688724018294656/t0(0) o400->0429ab19-2ee9-872a-fd65-9f568a4654b8@10.31.140.63@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513094.708199:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513094.708201:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513094.708208:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:4.0:1616513094.708210:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688724018294656, offset 192 00000400:00000200:0.0:1616513094.708212:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa08 00000800:00000200:0.0:1616513094.708216:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513094.708217:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.63@tcp1 00000800:00000200:0.0:1616513094.708219:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:4.0:1616513094.708224:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.63@tcp1 routed destination 00000400:00000200:4.0:1616513094.708239:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.63@tcp1(10.31.140.63@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513094.708246:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513094.708252:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513094.708254:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513094.708256:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513094.708261:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.708267:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513094.708268:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513094.708322:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.708329:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.708332:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.708338:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.708344:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baf68 00000800:00000200:2.0:1616513094.708347:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.708350:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.766108:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513094.766165:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513094.766175:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513094.766185:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.129@tcp1 : PUT - for me 00000400:00000200:22.0:1616513094.766197:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.129@tcp1 of length 224 into portal 26 MB=0x60184f14088c0 00000400:00000200:22.0:1616513094.766203:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.129@tcp1 of length 224/224 into md 0xf97cf0d9 [1] + 0 00000400:00000200:22.0:1616513094.766208:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf298 00000400:00000200:22.0:1616513094.766212:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.766215:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513094.766219:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf298 00000800:00000200:18.0:1616513094.766220:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.766222:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.766226:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.766227:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513094.766287:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690520355113152 00010000:00000200:4.0:1616513094.766300:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1690520355113152/t0(0) o400->827fa20b-4b40-acad-e4f7-b36c50fc6710@10.31.142.129@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513094.766315:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690520355113152, offset 192 00000400:00000200:4.0:1616513094.766321:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.129@tcp1 00000400:00000200:4.0:1616513094.766330:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.129@tcp1 routed destination 00000400:00000200:4.0:1616513094.766345:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.129@tcp1(10.31.142.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513094.766352:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513094.766358:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513094.766361:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513094.766363:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513094.766369:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.766376:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513094.766376:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513094.766431:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.766454:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513094.766457:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513094.766464:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513094.766470:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000800:00000200:16.0:1616513094.766473:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.766476:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.766482:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513094.766484:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513094.766490:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.129@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.766499:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.129@tcp1 of length 224 into portal 12 MB=0x60184f1409600 00000400:00000200:16.0:1616513094.766506:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.129@tcp1 of length 224/224 into md 0x865be291 [1] + 51504 00000400:00000200:16.0:1616513094.766510:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.766513:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513094.766520:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.766522:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.766524:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513094.766588:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690520355116544 00010000:00000200:24.0:1616513094.766601:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6727980 x1690520355116544/t0(0) o400->f249399c-c7f2-2bef-c340-12085911121a@10.31.142.129@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.766615:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690520355116544, offset 192 00000400:00000200:24.0:1616513094.766622:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.129@tcp1 00000400:00000200:24.0:1616513094.766630:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.129@tcp1 routed destination 00000400:00000200:24.0:1616513094.766645:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.129@tcp1(10.31.142.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.766652:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.766658:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.766661:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.766663:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.766669:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.766673:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.766730:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.766737:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.766739:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.766746:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.766752:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3dc0 00000800:00000200:2.0:1616513094.766757:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513094.766759:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513094.778525:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929024, portal 4 00000100:00000200:26.0:1616513094.778529:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929024, offset 0 00000400:00000200:26.0:1616513094.778535:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:26.0:1616513094.778539:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:26.0:1616513094.778552:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.778557:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:26.0:1616513094.778563:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.778565:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:26.0:1616513094.778567:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:26.0:1616513094.778572:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513094.778618:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513094.778677:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513094.778685:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513094.778687:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513094.778694:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513094.778700:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bdf980 x1693067725929024/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513094.778712:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae088 00000800:00000200:20.0:1616513094.778716:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513094.778719:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513094.778802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513094.778856:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513094.778866:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:16.0:1616513094.778875:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513094.778884:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627a40 00000400:00000200:16.0:1616513094.778891:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dc101 [1] + 192 00000400:00000200:16.0:1616513094.778896:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.778901:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bdf980 x1693067725929024/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:0.0:1616513094.778911:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513094.778916:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513094.778919:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513094.778920:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:26.0:1616513094.778924:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bdf980 x1693067725929024/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513094.778936:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bdf980 x1693067725929024/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513094.778946:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af3a8 00000800:00000200:11.2:1616513094.821128:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513094.821186:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513094.821195:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513094.821205:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.211@tcp1 : PUT - for me 00000400:00000200:20.0:1616513094.821214:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.211@tcp1 of length 224 into portal 26 MB=0x5ffe390b24440 00000400:00000200:20.0:1616513094.821221:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.211@tcp1 of length 224/224 into md 0xf97cf0e9 [1] + 0 00000400:00000200:20.0:1616513094.821226:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfa90 00000400:00000200:20.0:1616513094.821229:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.821232:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513094.821236:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfa90 00000800:00000200:20.0:1616513094.821240:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.821240:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.821243:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.821247:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.821250:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.821250:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513094.821259:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.211@tcp1 : PUT - for me 00000400:00000200:12.0:1616513094.821268:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.211@tcp1 of length 224 into portal 12 MB=0x5ffe390b246c0 00000400:00000200:12.0:1616513094.821274:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.211@tcp1 of length 224/224 into md 0x865be291 [1] + 51728 00000400:00000200:12.0:1616513094.821280:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.821282:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513094.821289:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.821293:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.821295:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513094.821313:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727733814336 00010000:00000200:4.0:1616513094.821325:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688727733814336/t0(0) o400->d96e6452-525f-2f7b-081c-3023c25afd42@10.31.141.211@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513094.821340:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688727733814336, offset 192 00000400:00000200:4.0:1616513094.821346:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.211@tcp1 00000400:00000200:4.0:1616513094.821353:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.211@tcp1 routed destination 00000100:00000200:24.0:1616513094.821356:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727733814976 00010000:00000200:24.0:1616513094.821369:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d204380 x1688727733814976/t0(0) o400->6a825daa-68d9-08fe-0649-0a7d80feb299@10.31.141.211@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:4.0:1616513094.821371:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.211@tcp1(10.31.141.211@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513094.821378:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513094.821384:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688727733814976, offset 192 00000800:00000200:4.0:1616513094.821384:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513094.821386:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513094.821389:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513094.821390:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.211@tcp1 00000800:00000200:4.0:1616513094.821394:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513094.821397:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.211@tcp1 routed destination 00000800:00000200:11.2:1616513094.821399:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513094.821401:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513094.821413:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.211@tcp1(10.31.141.211@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.821419:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.821426:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.821428:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.821431:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.821451:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513094.821452:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.821459:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513094.821460:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513094.821461:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513094.821463:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513094.821467:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513094.821473:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb5c8 00000800:00000200:18.0:1616513094.821475:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513094.821476:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513094.821482:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.821484:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.821491:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.821495:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3210 00000800:00000200:18.0:1616513094.821499:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.821502:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.856919:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513094.856977:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.856986:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513094.856995:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.50@tcp1 : PUT - for me 00000400:00000200:12.0:1616513094.857004:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.50@tcp1 of length 224 into portal 12 MB=0x5ffe1a1da3840 00000400:00000200:12.0:1616513094.857012:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.50@tcp1 of length 224/224 into md 0x865be291 [1] + 51952 00000400:00000200:12.0:1616513094.857018:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.857021:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513094.857028:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.857031:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513094.857031:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.857035:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513094.857091:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688719431710784 00000800:00000200:7.2:1616513094.857104:0:51043:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:24.0:1616513094.857105:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576d206c00 x1688719431710784/t0(0) o400->695cea4f-4190-8682-af51-e29d09b4e3db@10.31.140.50@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.857119:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688719431710784, offset 192 00000400:00000200:24.0:1616513094.857125:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.50@tcp1 00000400:00000200:24.0:1616513094.857133:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.50@tcp1 routed destination 00000400:00000200:24.0:1616513094.857148:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.50@tcp1(10.31.140.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.857154:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513094.857160:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.857163:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513094.857165:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513094.857165:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.857170:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.857175:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513094.857175:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513094.857184:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.50@tcp1 : PUT - for me 00000400:00000200:2.0:1616513094.857193:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.50@tcp1 of length 224 into portal 26 MB=0x5ffe1a1da3780 00000400:00000200:2.0:1616513094.857199:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.50@tcp1 of length 224/224 into md 0xf97cf0f9 [1] + 0 00000400:00000200:2.0:1616513094.857204:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be908 00000400:00000200:2.0:1616513094.857207:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513094.857209:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513094.857214:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be908 00000800:00000200:2.0:1616513094.857217:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.857219:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513094.857220:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513094.857223:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513094.857242:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.857249:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513094.857252:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513094.857259:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513094.857263:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2e58 00000800:00000200:20.0:1616513094.857266:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513094.857269:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513094.857282:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688719431710592 00010000:00000200:4.0:1616513094.857293:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688719431710592/t0(0) o400->ed064001-7e74-69f4-5d5a-e3e82a52b047@10.31.140.50@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513094.857308:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688719431710592, offset 192 00000400:00000200:4.0:1616513094.857314:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.50@tcp1 00000400:00000200:4.0:1616513094.857321:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.50@tcp1 routed destination 00000400:00000200:4.0:1616513094.857336:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.50@tcp1(10.31.140.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513094.857343:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513094.857349:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513094.857352:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513094.857354:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513094.857359:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.857366:0:51043:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513094.857366:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513094.857421:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.857428:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.857451:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.857458:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.857464:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba5d8 00000800:00000200:18.0:1616513094.857468:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.857470:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.860211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.860267:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.860277:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513094.860287:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.14@tcp1 : PUT - for me 00000400:00000200:0.0:1616513094.860296:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.14@tcp1 of length 224 into portal 26 MB=0x604a467e75f40 00000400:00000200:0.0:1616513094.860303:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.14@tcp1 of length 224/224 into md 0xf97cf109 [1] + 0 00000400:00000200:0.0:1616513094.860307:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf650 00000400:00000200:0.0:1616513094.860310:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513094.860313:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513094.860317:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf650 00000800:00000200:0.0:1616513094.860320:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.860321:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513094.860323:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513094.860325:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513094.860381:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954024628032 00010000:00000200:4.0:1616513094.860391:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1693954024628032/t0(0) o400->4988e163-0783-6f8e-a522-58b19502bd4a@10.31.141.14@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513094.860406:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954024628032, offset 192 00000400:00000200:4.0:1616513094.860412:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.14@tcp1 00000400:00000200:4.0:1616513094.860420:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.14@tcp1 routed destination 00000400:00000200:4.0:1616513094.860457:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.14@tcp1(10.31.141.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513094.860464:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513094.860469:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513094.860474:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513094.860476:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513094.860481:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.860486:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513094.860488:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513094.860540:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.860547:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513094.860549:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513094.860558:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513094.860563:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb188 00000800:00000200:20.0:1616513094.860567:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513094.860570:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513094.860898:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513094.860909:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.860917:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.860919:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513094.860926:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.14@tcp1 : PUT - for me 00000400:00000200:12.0:1616513094.860934:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.14@tcp1 of length 224 into portal 12 MB=0x604a467e762c0 00000400:00000200:12.0:1616513094.860940:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.14@tcp1 of length 224/224 into md 0x865be291 [1] + 52176 00000400:00000200:12.0:1616513094.860945:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.860948:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513094.860955:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513094.860958:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.860960:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513094.861020:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954024628928 00010000:00000200:24.0:1616513094.861032:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4536057080 x1693954024628928/t0(0) o400->52d8abd3-076d-bc4f-d5cd-0dc064421529@10.31.141.14@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.861047:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954024628928, offset 192 00000400:00000200:24.0:1616513094.861053:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.14@tcp1 00000400:00000200:24.0:1616513094.861060:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.14@tcp1 routed destination 00000400:00000200:24.0:1616513094.861077:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.14@tcp1(10.31.141.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.861083:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.861089:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.861091:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.861094:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.861099:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.861104:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513094.861158:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.861164:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513094.861169:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513094.861176:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513094.861180:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3b18 00000800:00000200:14.0:1616513094.861184:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.861187:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:31.0:1616513094.871549:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513094.871557:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000100:00000200:31.0:1616513094.891263:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929040, portal 4 00000100:00000200:31.0:1616513094.891271:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929040, offset 0 00000400:00000200:31.0:1616513094.891278:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:31.0:1616513094.891286:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:31.0:1616513094.891299:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513094.891307:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:31.0:1616513094.891314:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513094.891318:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:31.0:1616513094.891320:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:31.0:1616513094.891327:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513094.891374:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513094.891446:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513094.891455:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513094.891458:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.891465:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513094.891472:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be41f80 x1693067725929040/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513094.891484:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c38f8 00000800:00000200:18.0:1616513094.891489:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513094.891492:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513094.891546:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513094.891602:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513094.891613:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513094.891622:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513094.891631:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627a50 00000400:00000200:14.0:1616513094.891638:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cf849 [1] + 192 00000400:00000200:14.0:1616513094.891644:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513094.891649:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be41f80 x1693067725929040/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513094.891658:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513094.891665:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513094.891668:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513094.891670:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:31.0:1616513094.891675:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be41f80 x1693067725929040/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513094.891688:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be41f80 x1693067725929040/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513094.891699:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000800:00000200:7.2:1616513094.891863:0:51046:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.891876:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.891882:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513094.891885:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513094.891890:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.180@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.891901:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.180@tcp1 of length 224 into portal 26 MB=0x6033cd6dbcb80 00000400:00000200:18.0:1616513094.891907:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.180@tcp1 of length 224/224 into md 0xf97cf119 [1] + 0 00000400:00000200:18.0:1616513094.891914:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02beff0 00000400:00000200:18.0:1616513094.891917:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.891919:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513094.891923:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beff0 00000800:00000200:18.0:1616513094.891926:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513094.891929:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.891931:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513094.891992:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692409697913728 00010000:00000200:4.0:1616513094.892004:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1692409697913728/t0(0) o400->d55efd6e-6d7a-0062-cc73-93ac23db6d34@10.31.138.180@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513094.892019:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692409697913728, offset 192 00000400:00000200:4.0:1616513094.892025:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.180@tcp1 00000400:00000200:4.0:1616513094.892034:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.180@tcp1 routed destination 00000400:00000200:4.0:1616513094.892050:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.180@tcp1(10.31.138.180@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513094.892057:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513094.892063:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513094.892066:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513094.892068:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513094.892073:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:4.0:1616513094.892080:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513094.892081:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513094.892137:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.892144:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513094.892147:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513094.892154:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513094.892159:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb6d8 00000800:00000200:20.0:1616513094.892163:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513094.892166:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513094.892363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.892418:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.892428:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513094.892453:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.180@tcp1 : PUT - for me 00000400:00000200:0.0:1616513094.892462:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.180@tcp1 of length 224 into portal 12 MB=0x6033cd6dbcc00 00000400:00000200:0.0:1616513094.892469:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.180@tcp1 of length 224/224 into md 0x865be291 [1] + 52400 00000800:00000200:12.0:1616513094.892472:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513094.892475:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513094.892477:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513094.892485:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.892488:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513094.892490:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513094.892550:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692409697913856 00010000:00000200:24.0:1616513094.892563:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86cc80 x1692409697913856/t0(0) o400->336a528a-ec60-74d3-c8d0-3e0b26b4fad3@10.31.138.180@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.892578:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692409697913856, offset 192 00000400:00000200:24.0:1616513094.892584:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.180@tcp1 00000400:00000200:24.0:1616513094.892592:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.180@tcp1 routed destination 00000400:00000200:24.0:1616513094.892608:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.180@tcp1(10.31.138.180@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.892614:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.892620:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.892623:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.892625:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.892630:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.892638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513094.892650:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513094.892655:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513094.892658:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513094.892661:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513094.892664:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.892668:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2330 00000800:00000200:2.0:1616513094.892673:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.895566:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513094.895596:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513094.895621:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513094.895630:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513094.895639:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.189@tcp1 : PUT - for me 00000400:00000200:20.0:1616513094.895647:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.189@tcp1 of length 224 into portal 26 MB=0x605a073b1d880 00000800:00000200:22.0:1616513094.895652:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513094.895654:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.189@tcp1 of length 224/224 into md 0xf97cf129 [1] + 0 00000400:00000200:20.0:1616513094.895659:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfcb0 00000800:00000200:22.0:1616513094.895661:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:20.0:1616513094.895662:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.895664:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513094.895668:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfcb0 00000400:00000200:22.0:1616513094.895670:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.189@tcp1 : PUT - for me 00000800:00000200:20.0:1616513094.895671:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.895674:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.895676:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513094.895679:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.189@tcp1 of length 224 into portal 12 MB=0x605a073b1da00 00000800:00000200:16.0:1616513094.895679:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513094.895685:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.189@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 16688 00000400:00000200:22.0:1616513094.895690:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.895693:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513094.895701:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.895704:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513094.895708:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.895711:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513094.895737:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036554205312 00010000:00000200:4.0:1616513094.895748:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1695036554205312/t0(0) o400->1ecf14bb-7fda-b60f-c27e-874491adedfe@10.31.129.189@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513094.895763:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036554205312, offset 192 00000400:00000200:4.0:1616513094.895769:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.189@tcp1 00000100:00000200:26.0:1616513094.895771:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036554205696 00000400:00000200:4.0:1616513094.895777:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.189@tcp1 routed destination 00010000:00000200:26.0:1616513094.895784:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a29b00 x1695036554205696/t0(0) o400->63067e44-3dd9-62ee-f96c-910b9469c2b0@10.31.129.189@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:4.0:1616513094.895792:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.189@tcp1(10.31.129.189@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513094.895798:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036554205696, offset 192 00000800:00000200:4.0:1616513094.895798:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513094.895804:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:26.0:1616513094.895805:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.189@tcp1 00000800:00000200:4.0:1616513094.895807:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513094.895809:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513094.895812:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.189@tcp1 routed destination 00000800:00000200:4.0:1616513094.895814:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.895818:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513094.895820:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:26.0:1616513094.895827:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.189@tcp1(10.31.129.189@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513094.895834:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513094.895839:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513094.895842:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513094.895844:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513094.895849:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.895854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513094.895872:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.895879:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.895882:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513094.895888:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.895894:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba440 00000800:00000200:12.0:1616513094.895897:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513094.895900:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513094.895906:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513094.895913:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513094.895916:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513094.895922:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513094.895926:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af3a8 00000800:00000200:14.0:1616513094.895931:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513094.895934:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:31.0:1616513094.906600:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929056, portal 4 00000100:00000200:31.0:1616513094.906605:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929056, offset 0 00000400:00000200:31.0:1616513094.906612:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:31.0:1616513094.906618:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:31.0:1616513094.906631:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513094.906638:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:31.0:1616513094.906644:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513094.906647:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:31.0:1616513094.906650:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:31.0:1616513094.906655:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513094.906701:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513094.906727:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513094.906733:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513094.906736:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513094.906743:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513094.906750:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded1b00 x1693067725929056/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513094.906761:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c25d8 00000800:00000200:1.0:1616513094.906766:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513094.906769:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513094.906889:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513094.906899:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513094.906905:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513094.906908:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513094.906913:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513094.906922:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627a60 00000400:00000200:1.0:1616513094.906929:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x437000c5 [1] + 192 00000400:00000200:1.0:1616513094.906935:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513094.906939:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded1b00 x1693067725929056/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513094.906952:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513094.906955:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513094.906957:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:31.0:1616513094.906960:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded1b00 x1693067725929056/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513094.906972:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded1b00 x1693067725929056/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513094.906981:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000100:00000200:3.0:1616513094.922626:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929072, portal 4 00000100:00000200:3.0:1616513094.922634:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929072, offset 0 00000400:00000200:3.0:1616513094.922641:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:3.0:1616513094.922646:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:3.0:1616513094.922659:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513094.922665:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513094.922672:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513094.922675:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:3.0:1616513094.922677:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:3.0:1616513094.922682:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513094.922727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513094.922783:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513094.922791:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513094.922794:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513094.922801:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513094.922808:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be44800 x1693067725929072/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513094.922820:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd24c8 00000800:00000200:20.0:1616513094.922825:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513094.922828:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513094.922909:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:12.0:1616513094.922963:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513094.922973:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:12.0:1616513094.922982:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513094.922990:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627a70 00000400:00000200:12.0:1616513094.922998:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dc1a1 [1] + 192 00000400:00000200:12.0:1616513094.923003:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513094.923009:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be44800 x1693067725929072/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:0.0:1616513094.923018:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513094.923026:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513094.923029:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513094.923031:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:3.0:1616513094.923036:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be44800 x1693067725929072/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513094.923049:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be44800 x1693067725929072/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513094.923060:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:25.2:1616513094.941851:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513094.941910:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513094.941921:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513094.941929:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513094.941938:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 26 MB=0x5f9313e1f9a30 00000400:00000200:22.0:1616513094.941945:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf97cf139 [1] + 0 00000400:00000200:22.0:1616513094.941950:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf5c8 00000400:00000200:22.0:1616513094.941953:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513094.941956:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513094.941960:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf5c8 00000800:00000200:22.0:1616513094.941963:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513094.941966:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513094.941967:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:22.0:1616513094.941974:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (23)-- 00000800:00000200:22.0:1616513094.941976:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513094.941978:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513094.941982:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513094.941986:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513094.941989:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513094.941995:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9a90 00000400:00000200:18.0:1616513094.941995:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9b30 00000400:00000200:22.0:1616513094.942000:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 16912 00000400:00000200:22.0:1616513094.942004:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.942007:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 17136 00000100:00000200:22.0:1616513094.942012:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513094.942012:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.942018:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (23)++ 00000100:00000200:18.0:1616513094.942019:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513094.942020:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (24)-- 00000800:00000200:22.0:1616513094.942022:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (23)-- 00000800:00000200:22.0:1616513094.942024:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513094.942027:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (23)++ 00000800:00000200:22.0:1616513094.942028:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000100:00000200:4.0:1616513094.942028:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525488 00000800:00000200:18.0:1616513094.942032:0:6471:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (24)++ 00000400:00000200:22.0:1616513094.942034:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:18.0:1616513094.942036:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (25)-- 00000800:00000200:18.0:1616513094.942038:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (24)-- 00000400:00000200:22.0:1616513094.942039:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9a50 00000800:00000200:18.0:1616513094.942041:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (23)++ 00010000:00000200:4.0:1616513094.942041:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1681364774525488/t0(0) o400->1ae63833-db83-47b2-ef1b-80ab28744988@10.242.46.229@o2ib1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:22.0:1616513094.942044:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 17360 00000400:00000200:22.0:1616513094.942048:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513094.942049:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000100:00000200:22.0:1616513094.942050:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513094.942050:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (24)++ 00000400:00000200:18.0:1616513094.942056:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:14.0:1616513094.942056:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000100:00000200:4.0:1616513094.942056:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681364774525488, offset 192 00000800:00000200:22.0:1616513094.942057:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (25)++ 00000800:00000200:2.0:1616513094.942057:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513094.942061:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (27)-- 00000400:00000200:18.0:1616513094.942061:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9a70 00000800:00000200:22.0:1616513094.942062:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000800:00000200:14.0:1616513094.942063:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:4.0:1616513094.942063:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:22.0:1616513094.942064:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000100:00000200:30.0:1616513094.942065:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525584 00000400:00000200:18.0:1616513094.942066:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 17584 00000800:00000200:22.0:1616513094.942067:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (26)-- 00000400:00000200:2.0:1616513094.942067:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:22.0:1616513094.942069:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (25)-- 00000400:00000200:18.0:1616513094.942070:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.942071:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (24)++ 00000400:00000200:4.0:1616513094.942071:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000100:00000200:18.0:1616513094.942072:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:14.0:1616513094.942072:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:22.0:1616513094.942074:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000100:00000200:26.0:1616513094.942077:0:184144:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525744 00000100:00000200:6.0:1616513094.942077:0:188435:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525520 00010000:00000200:30.0:1616513094.942078:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2d100 x1681364774525584/t0(0) o400->boslfs02-MDT0001-lwp-OST0014_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:4.0:1616513094.942085:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00010000:00000200:26.0:1616513094.942089:0:184144:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd707500 x1681364774525744/t0(0) o400->boslfs02-MDT0001-lwp-OST0000_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:6.0:1616513094.942089:0:188435:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2da00 x1681364774525520/t0(0) o400->boslfs02-MDT0001-lwp-OST0018_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513094.942092:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (25)++ 00000800:00000200:4.0:1616513094.942092:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000100:00000200:30.0:1616513094.942094:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525584, offset 192 00000800:00000200:18.0:1616513094.942094:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (26)-- 00000400:00000200:22.0:1616513094.942095:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:18.0:1616513094.942095:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (25)-- 00000400:00000200:14.0:1616513094.942095:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9b90 00000800:00000200:18.0:1616513094.942097:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (24)++ 00000800:00000200:4.0:1616513094.942098:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513094.942101:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513094.942101:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:4.0:1616513094.942101:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (25)++ 00000800:00000200:4.0:1616513094.942103:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (26)++ 00000100:00000200:2.0:1616513094.942104:0:188430:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525552 00000400:00000200:14.0:1616513094.942106:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 17808 00000400:00000200:18.0:1616513094.942108:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:4.0:1616513094.942108:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (27)-- 00000400:00000200:30.0:1616513094.942109:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:14.0:1616513094.942111:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513094.942112:0:188435:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525520, offset 192 00000100:00000200:26.0:1616513094.942113:0:184144:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525744, offset 192 00000100:00000200:14.0:1616513094.942114:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:2.0:1616513094.942114:0:188430:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd705e80 x1681364774525552/t0(0) o400->boslfs02-MDT0001-lwp-OST000c_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513094.942116:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:30.0:1616513094.942123:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513094.942129:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513094.942129:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (26)++ 00000400:00000200:6.0:1616513094.942129:0:188435:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513094.942132:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9ad0 00000400:00000200:18.0:1616513094.942132:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9ab0 00000400:00000200:26.0:1616513094.942134:0:184144:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513094.942134:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (27)-- 00000400:00000200:6.0:1616513094.942136:0:188435:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:14.0:1616513094.942137:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000100:00000200:2.0:1616513094.942137:0:188430:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525552, offset 192 00000800:00000200:14.0:1616513094.942139:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000400:00000200:22.0:1616513094.942141:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 18032 00000800:00000200:14.0:1616513094.942143:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513094.942145:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513094.942145:0:170088:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525840 00000400:00000200:18.0:1616513094.942147:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 18256 00000400:00000200:6.0:1616513094.942149:0:188435:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513094.942151:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.942154:0:188430:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00010000:00000200:10.0:1616513094.942156:0:170088:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07080 x1681364774525840/t0(0) o400->boslfs02-MDT0001-lwp-OST0030_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:6.0:1616513094.942156:0:188435:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513094.942157:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000100:00000200:22.0:1616513094.942157:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:26.0:1616513094.942158:0:184144:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:30.0:1616513094.942160:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (26)++ 00000800:00000200:30.0:1616513094.942162:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (27)++ 00000100:00000200:18.0:1616513094.942163:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:30.0:1616513094.942167:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (28)-- 00000400:00000200:26.0:1616513094.942173:0:184144:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513094.942173:0:170088:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525840, offset 192 00000400:00000200:10.0:1616513094.942179:0:170088:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:26.0:1616513094.942180:0:184144:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:14.0:1616513094.942182:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513094.942182:0:188430:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000100:00000200:30.0:1616513094.942191:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525648 00000400:00000200:2.0:1616513094.942196:0:188430:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00010000:00000200:30.0:1616513094.942197:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2de80 x1681364774525648/t0(0) o400->boslfs02-MDT0001-lwp-OST003c_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:22.0:1616513094.942205:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (27)++ 00000800:00000200:18.0:1616513094.942206:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (27)++ 00000800:00000200:22.0:1616513094.942207:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (28)-- 00000800:00000200:2.0:1616513094.942207:0:188430:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000100:00000200:30.0:1616513094.942208:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525648, offset 192 00000800:00000200:18.0:1616513094.942208:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (29)-- 00000800:00000200:6.0:1616513094.942208:0:188435:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513094.942209:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (28)-- 00000800:00000200:6.0:1616513094.942210:0:188435:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (26)++ 00000800:00000200:18.0:1616513094.942211:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000400:00000200:10.0:1616513094.942211:0:170088:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:30.0:1616513094.942213:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:22.0:1616513094.942213:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (26)++ 00000800:00000200:6.0:1616513094.942213:0:188435:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (26)++ 00000800:00000200:6.0:1616513094.942217:0:188435:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (28)-- 00000800:00000200:22.0:1616513094.942220:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000100:00000200:18.0:1616513094.942222:0:188394:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525616 00000400:00000200:10.0:1616513094.942224:0:170088:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513094.942231:0:170088:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:26.0:1616513094.942232:0:184144:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00010000:00000200:18.0:1616513094.942232:0:188394:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd701200 x1681364774525616/t0(0) o400->boslfs02-MDT0001-lwp-OST0010_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513094.942233:0:188430:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:30.0:1616513094.942234:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:26.0:1616513094.942235:0:184144:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (27)++ 00000800:00000200:2.0:1616513094.942235:0:188430:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (27)++ 00000800:00000200:26.0:1616513094.942237:0:184144:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (28)++ 00000800:00000200:26.0:1616513094.942242:0:184144:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (30)-- 00000800:00000200:2.0:1616513094.942242:0:188430:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (30)++ 00000100:00000200:18.0:1616513094.942246:0:188394:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525616, offset 192 00000800:00000200:2.0:1616513094.942246:0:188430:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (30)-- 00000400:00000200:30.0:1616513094.942247:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513094.942252:0:188394:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513094.942254:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:14.0:1616513094.942254:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9af0 00000800:00000200:10.0:1616513094.942255:0:170088:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513094.942256:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:10.0:1616513094.942257:0:170088:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (29)++ 00000400:00000200:18.0:1616513094.942258:0:188394:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:14.0:1616513094.942259:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 18480 00000800:00000200:10.0:1616513094.942260:0:170088:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (30)++ 00000400:00000200:14.0:1616513094.942263:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:10.0:1616513094.942264:0:170088:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)-- 00000100:00000200:14.0:1616513094.942265:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513094.942271:0:188394:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513094.942277:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513094.942277:0:188394:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513094.942277:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (30)++ 00000800:00000200:30.0:1616513094.942279:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)++ 00000800:00000200:14.0:1616513094.942279:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (31)-- 00000400:00000200:22.0:1616513094.942280:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9b50 00000800:00000200:14.0:1616513094.942280:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (30)-- 00000400:00000200:2.0:1616513094.942280:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9b10 00000800:00000200:30.0:1616513094.942281:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (31)++ 00000800:00000200:18.0:1616513094.942282:0:188394:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513094.942283:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (31)++ 00000800:00000200:30.0:1616513094.942284:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)-- 00000400:00000200:22.0:1616513094.942284:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 18704 00000800:00000200:18.0:1616513094.942285:0:188394:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)++ 00000800:00000200:14.0:1616513094.942285:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (31)-- 00000800:00000200:18.0:1616513094.942286:0:188394:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (32)++ 00000400:00000200:14.0:1616513094.942287:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:6.0:1616513094.942287:0:188423:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525680 00000400:00000200:22.0:1616513094.942288:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513094.942290:0:188394:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)-- 00000400:00000200:2.0:1616513094.942290:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 18928 00000400:00000200:14.0:1616513094.942293:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00010000:00000200:6.0:1616513094.942293:0:188423:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01200 x1681364774525680/t0(0) o400->boslfs02-MDT0001-lwp-OST0008_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513094.942295:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:2.0:1616513094.942295:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513094.942297:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb078 00000800:00000200:14.0:1616513094.942300:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (31)-- 00000800:00000200:14.0:1616513094.942302:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (30)++ 00000800:00000200:22.0:1616513094.942305:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (31)++ 00000100:00000200:2.0:1616513094.942305:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513094.942306:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513094.942307:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (32)-- 00000800:00000200:22.0:1616513094.942309:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (31)-- 00000800:00000200:22.0:1616513094.942311:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (30)++ 00000400:00000200:14.0:1616513094.942312:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:22.0:1616513094.942314:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513094.942320:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9b70 00000400:00000200:22.0:1616513094.942321:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513094.942327:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9c20 00000100:00000200:6.0:1616513094.942333:0:188423:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525680, offset 192 00000800:00000200:2.0:1616513094.942333:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (31)++ 00000400:00000200:14.0:1616513094.942335:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 19152 00000800:00000200:2.0:1616513094.942336:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (32)-- 00000100:00000200:18.0:1616513094.942337:0:188394:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525776 00000800:00000200:2.0:1616513094.942338:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (31)-- 00000400:00000200:14.0:1616513094.942339:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513094.942340:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (30)++ 00000400:00000200:22.0:1616513094.942341:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 19376 00010000:00000200:18.0:1616513094.942343:0:188394:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a29200 x1681364774525776/t0(0) o400->boslfs02-MDT0001-lwp-OST001c_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513094.942343:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (31)-- 00000400:00000200:22.0:1616513094.942344:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513094.942345:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513094.942346:0:188423:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000100:00000200:14.0:1616513094.942348:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:6.0:1616513094.942350:0:188423:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:2.0:1616513094.942350:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513094.942353:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:26.0:1616513094.942356:0:188438:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525712 00000400:00000200:6.0:1616513094.942359:0:188423:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513094.942359:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef8f8 00000100:00000200:30.0:1616513094.942362:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525808 00000800:00000200:14.0:1616513094.942362:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (30)++ 00010000:00000200:26.0:1616513094.942363:0:188438:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5580 x1681364774525712/t0(0) o400->boslfs02-MDT0001-lwp-OST0004_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:22.0:1616513094.942363:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (30)++ 00000100:00000200:18.0:1616513094.942363:0:188394:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525776, offset 192 00000800:00000200:6.0:1616513094.942363:0:188423:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513094.942364:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (31)-- 00000800:00000200:2.0:1616513094.942365:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (32)-- 00000800:00000200:22.0:1616513094.942366:0:6470:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (30)++ 00010000:00000200:30.0:1616513094.942367:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e01b00 x1681364774525808/t0(0) o400->boslfs02-MDT0001-lwp-OST0024_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513094.942367:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (30)-- 00000800:00000200:6.0:1616513094.942367:0:188423:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513094.942368:0:188394:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:6.0:1616513094.942368:0:188423:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (30)++ 00000800:00000200:2.0:1616513094.942368:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (30)++ 00000800:00000200:22.0:1616513094.942370:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (32)-- 00000100:00000200:10.0:1616513094.942370:0:170088:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525984 00000800:00000200:6.0:1616513094.942370:0:188423:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (32)++ 00000400:00000200:18.0:1616513094.942373:0:188394:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:6.0:1616513094.942373:0:188423:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:6.0:1616513094.942374:0:188423:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)-- 00000100:00000200:26.0:1616513094.942375:0:188438:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525712, offset 192 00010000:00000200:10.0:1616513094.942376:0:170088:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2b180 x1681364774525984/t0(0) o400->boslfs02-MDT0001-lwp-OST0028_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513094.942378:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525808, offset 192 00000400:00000200:26.0:1616513094.942381:0:188438:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:22.0:1616513094.942383:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (31)-- 00000400:00000200:30.0:1616513094.942385:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:22.0:1616513094.942385:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (30)++ 00000800:00000200:2.0:1616513094.942385:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513094.942386:0:188394:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513094.942388:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (31)++ 00000100:00000200:10.0:1616513094.942388:0:170088:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525984, offset 192 00000800:00000200:22.0:1616513094.942389:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (31)-- 00000400:00000200:22.0:1616513094.942391:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513094.942391:0:188394:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513094.942391:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (31)-- 00000400:00000200:26.0:1616513094.942393:0:188438:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:14.0:1616513094.942393:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513094.942394:0:170088:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:22.0:1616513094.942396:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:14.0:1616513094.942398:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.942399:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01650 00000400:00000200:30.0:1616513094.942400:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:14.0:1616513094.942402:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af6d8 00000800:00000200:18.0:1616513094.942406:0:188394:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:10.0:1616513094.942408:0:170088:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:2.0:1616513094.942408:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:18.0:1616513094.942409:0:188394:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (30)++ 00000800:00000200:18.0:1616513094.942410:0:188394:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (31)++ 00000800:00000200:22.0:1616513094.942415:0:6470:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513094.942415:0:188394:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:18.0:1616513094.942416:0:188394:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)-- 00000800:00000200:14.0:1616513094.942416:0:6472:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:2.0:1616513094.942418:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9ba0 00000800:00000200:22.0:1616513094.942420:0:6470:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513094.942422:0:6472:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513094.942423:0:6470:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (31)++ 00000400:00000200:2.0:1616513094.942423:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 19600 00000800:00000200:22.0:1616513094.942424:0:6470:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (32)++ 00000800:00000200:14.0:1616513094.942424:0:6472:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)++ 00000800:00000200:18.0:1616513094.942426:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (34)++ 00000400:00000200:2.0:1616513094.942427:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.942428:0:6470:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:22.0:1616513094.942429:0:6470:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (35)-- 00000100:00000200:2.0:1616513094.942429:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513094.942430:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:14.0:1616513094.942430:0:6472:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (35)++ 00000800:00000200:22.0:1616513094.942431:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (35)-- 00000800:00000200:22.0:1616513094.942433:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (34)++ 00000800:00000200:14.0:1616513094.942444:0:6472:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:14.0:1616513094.942446:0:6472:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (35)-- 00000800:00000200:14.0:1616513094.942448:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (34)-- 00000800:00000200:14.0:1616513094.942450:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (33)++ 00000800:00000200:18.0:1616513094.942451:0:6471:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:2.0:1616513094.942451:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (33)++ 00000800:00000200:14.0:1616513094.942454:0:6472:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:22.0:1616513094.942455:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:14.0:1616513094.942456:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (35)-- 00000400:00000200:14.0:1616513094.942457:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:26.0:1616513094.942458:0:188394:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525856 00000400:00000200:18.0:1616513094.942458:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:2.0:1616513094.942458:0:6469:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:2.0:1616513094.942459:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (34)-- 00000400:00000200:22.0:1616513094.942461:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:2.0:1616513094.942461:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (33)-- 00000400:00000200:14.0:1616513094.942462:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00010000:00000200:26.0:1616513094.942463:0:188394:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7500 x1681364774525856/t0(0) o400->boslfs02-MDT0001-lwp-OST0038_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:18.0:1616513094.942463:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9c00 00000800:00000200:2.0:1616513094.942463:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (32)++ 00000400:00000200:14.0:1616513094.942465:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eecb28 00000800:00000200:2.0:1616513094.942466:0:6469:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000400:00000200:22.0:1616513094.942467:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9bc0 00000800:00000200:2.0:1616513094.942467:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (33)-- 00000400:00000200:18.0:1616513094.942469:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 19824 00000800:00000200:14.0:1616513094.942470:0:6472:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:2.0:1616513094.942470:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.942473:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513094.942474:0:6472:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513094.942475:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.942476:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 20048 00000800:00000200:14.0:1616513094.942476:0:6472:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (32)++ 00000800:00000200:14.0:1616513094.942478:0:6472:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (33)++ 00000100:00000200:26.0:1616513094.942479:0:188394:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525856, offset 192 00000400:00000200:22.0:1616513094.942479:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.942480:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513094.942482:0:6472:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:14.0:1616513094.942482:0:6472:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (34)-- 00000800:00000200:14.0:1616513094.942484:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (33)-- 00000800:00000200:18.0:1616513094.942485:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (33)++ 00000400:00000200:2.0:1616513094.942485:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82c38 00000800:00000200:14.0:1616513094.942486:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (32)++ 00000100:00000200:22.0:1616513094.942487:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513094.942488:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (34)-- 00000800:00000200:18.0:1616513094.942489:0:6471:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:18.0:1616513094.942490:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (33)-- 00000800:00000200:2.0:1616513094.942490:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (33)++ 00000800:00000200:18.0:1616513094.942492:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (33)-- 00000800:00000200:14.0:1616513094.942492:0:6472:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:22.0:1616513094.942493:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (33)++ 00000800:00000200:14.0:1616513094.942493:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (32)-- 00000400:00000200:26.0:1616513094.942494:0:188394:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513094.942494:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (32)++ 00000800:00000200:2.0:1616513094.942494:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513094.942495:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.942496:0:6470:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.229@o2ib1: no credits 00000800:00000200:22.0:1616513094.942498:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (33)-- 00000100:00000200:10.0:1616513094.942498:0:170088:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525952 00000800:00000200:22.0:1616513094.942499:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (32)-- 00000400:00000200:26.0:1616513094.942500:0:188394:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:14.0:1616513094.942500:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:18.0:1616513094.942501:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (31)-- 00000800:00000200:2.0:1616513094.942501:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (32)++ 00000800:00000200:22.0:1616513094.942502:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (31)++ 00000400:00000200:14.0:1616513094.942502:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee088 00000800:00000200:2.0:1616513094.942502:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (32)-- 00000400:00000200:18.0:1616513094.942503:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:10.0:1616513094.942503:0:170088:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd706300 x1681364774525952/t0(0) o400->boslfs02-MDT0001-lwp-OST002c_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513094.942504:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525888 00000800:00000200:2.0:1616513094.942504:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (31)-- 00000800:00000200:22.0:1616513094.942505:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:2.0:1616513094.942506:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (30)++ 00010000:00000200:30.0:1616513094.942508:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2f080 x1681364774525888/t0(0) o400->boslfs02-MDT0001-lwp-OST0034_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513094.942508:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (31)-- 00000400:00000200:18.0:1616513094.942509:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:2.0:1616513094.942510:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (30)-- 00000400:00000200:18.0:1616513094.942511:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc550 00000800:00000200:2.0:1616513094.942511:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (29)++ 00000400:00000200:26.0:1616513094.942513:0:188394:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513094.942513:0:170088:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525952, offset 192 00000800:00000200:2.0:1616513094.942513:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (30)-- 00000400:00000200:2.0:1616513094.942514:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513094.942517:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (29)-- 00000800:00000200:18.0:1616513094.942518:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (28)-- 00000800:00000200:26.0:1616513094.942519:0:188394:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000400:00000200:10.0:1616513094.942519:0:170088:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000100:00000200:30.0:1616513094.942520:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525888, offset 192 00000800:00000200:14.0:1616513094.942520:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (27)++ 00000400:00000200:2.0:1616513094.942520:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.942522:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513094.942523:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf007f8 00000800:00000200:14.0:1616513094.942524:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (28)-- 00000400:00000200:10.0:1616513094.942524:0:170088:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:14.0:1616513094.942526:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:30.0:1616513094.942527:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513094.942529:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (27)++ 00000400:00000200:14.0:1616513094.942531:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:18.0:1616513094.942533:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513094.942533:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc6e8 00000800:00000200:18.0:1616513094.942535:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (28)++ 00000800:00000200:18.0:1616513094.942536:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (29)-- 00000400:00000200:10.0:1616513094.942537:0:170088:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513094.942538:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (28)-- 00000800:00000200:18.0:1616513094.942539:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (27)++ 00000800:00000200:18.0:1616513094.942541:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (28)-- 00000800:00000200:10.0:1616513094.942542:0:170088:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:2.0:1616513094.942542:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (28)-- 00000800:00000200:26.0:1616513094.942543:0:188394:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000400:00000200:18.0:1616513094.942543:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513094.942544:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (26)++ 00000400:00000200:30.0:1616513094.942546:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000800:00000200:26.0:1616513094.942546:0:188394:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (27)++ 00000800:00000200:2.0:1616513094.942546:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (27)-- 00000800:00000200:26.0:1616513094.942547:0:188394:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (27)++ 00000400:00000200:18.0:1616513094.942548:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.942548:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513094.942550:0:188394:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (28)-- 00000400:00000200:18.0:1616513094.942550:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697ae880 00000400:00000200:2.0:1616513094.942552:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.942555:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efd38 00000400:00000200:30.0:1616513094.942558:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513094.942562:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (27)-- 00000800:00000200:30.0:1616513094.942563:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:18.0:1616513094.942563:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000400:00000200:22.0:1616513094.942564:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 224 into portal 12 MB=0x5f9313e1f9be0 00000800:00000200:14.0:1616513094.942565:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (25)++ 00000800:00000200:10.0:1616513094.942565:0:170088:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513094.942567:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513094.942567:0:170088:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (25)++ 00000800:00000200:30.0:1616513094.942569:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (26)++ 00000400:00000200:22.0:1616513094.942569:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.229@o2ib1 of length 224/224 into md 0xf9654ef9 [1] + 20272 00000800:00000200:10.0:1616513094.942569:0:170088:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (26)++ 00000400:00000200:22.0:1616513094.942572:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:30.0:1616513094.942573:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (29)++ 00000800:00000200:10.0:1616513094.942573:0:170088:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (28)-- 00000800:00000200:2.0:1616513094.942574:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (29)-- 00000800:00000200:30.0:1616513094.942576:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (29)-- 00000100:00000200:22.0:1616513094.942576:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513094.942576:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (28)-- 00000800:00000200:2.0:1616513094.942577:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (28)++ 00000400:00000200:14.0:1616513094.942578:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513094.942579:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (27)++ 00000800:00000200:2.0:1616513094.942579:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (27)-- 00000800:00000200:22.0:1616513094.942581:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (27)++ 00000400:00000200:2.0:1616513094.942581:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513094.942583:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (28)-- 00000800:00000200:18.0:1616513094.942583:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513094.942583:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:22.0:1616513094.942585:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (27)-- 00000800:00000200:18.0:1616513094.942585:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (27)++ 00000400:00000200:14.0:1616513094.942585:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82ff0 00000400:00000200:2.0:1616513094.942585:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:22.0:1616513094.942587:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (27)++ 00000800:00000200:18.0:1616513094.942587:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (27)-- 00000400:00000200:2.0:1616513094.942587:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afed0 00000100:00000200:30.0:1616513094.942588:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774525920 00000800:00000200:22.0:1616513094.942589:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (27)-- 00000800:00000200:14.0:1616513094.942589:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (27)-- 00000800:00000200:18.0:1616513094.942590:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (26)-- 00000400:00000200:22.0:1616513094.942591:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:30.0:1616513094.942592:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2bf00 x1681364774525920/t0(0) o400->boslfs02-MDT0001-lwp-OST0020_UUID@10.242.46.229@o2ib1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513094.942592:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (24)-- 00000800:00000200:14.0:1616513094.942594:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (23)++ 00000400:00000200:22.0:1616513094.942596:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.942598:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e834b8 00000800:00000200:14.0:1616513094.942598:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (24)-- 00000400:00000200:14.0:1616513094.942600:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:30.0:1616513094.942601:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774525920, offset 192 00000800:00000200:22.0:1616513094.942601:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (23)-- 00000800:00000200:22.0:1616513094.942604:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:14.0:1616513094.942604:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:30.0:1616513094.942605:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:14.0:1616513094.942608:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee770 00000400:00000200:30.0:1616513094.942611:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:30.0:1616513094.942622:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513094.942626:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:14.0:1616513094.942626:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:30.0:1616513094.942630:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513094.942632:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513094.942634:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513094.942636:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513094.942642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513094.942654:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513094.942658:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513094.942660:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513094.942662:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513094.942664:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000400:00000200:2.0:1616513094.942667:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee198 00000800:00000200:2.0:1616513094.942670:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:11.2:1616513094.960225:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513094.960283:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513094.960292:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513094.960303:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.28@tcp1 : PUT - for me 00000400:00000200:20.0:1616513094.960312:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.28@tcp1 of length 224 into portal 26 MB=0x6048d6db9e340 00000400:00000200:20.0:1616513094.960319:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.28@tcp1 of length 224/224 into md 0xf97cf149 [1] + 0 00000400:00000200:20.0:1616513094.960324:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be2a8 00000400:00000200:20.0:1616513094.960326:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.960329:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513094.960334:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be2a8 00000800:00000200:20.0:1616513094.960337:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.960337:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513094.960340:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.960344:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513094.960347:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.960348:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513094.960357:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.28@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.960365:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.28@tcp1 of length 224 into portal 12 MB=0x6048d6db9e3c0 00000400:00000200:16.0:1616513094.960372:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.28@tcp1 of length 224/224 into md 0x865be291 [1] + 52624 00000400:00000200:16.0:1616513094.960377:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.960380:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513094.960387:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.960390:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.960392:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513094.960409:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855338062656 00010000:00000200:4.0:1616513094.960420:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1693855338062656/t0(0) o400->62b7be0c-2292-f95c-1c85-a6d7d8928a48@10.31.141.28@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513094.960449:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855338062656, offset 192 00000400:00000200:4.0:1616513094.960455:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.28@tcp1 00000400:00000200:4.0:1616513094.960462:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.28@tcp1 routed destination 00000100:00000200:24.0:1616513094.960466:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855338062784 00010000:00000200:24.0:1616513094.960478:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6725a00 x1693855338062784/t0(0) o400->7e0975bf-208d-e13a-34d8-52560df486df@10.31.141.28@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:4.0:1616513094.960478:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.28@tcp1(10.31.141.28@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513094.960485:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513094.960491:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:24.0:1616513094.960492:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855338062784, offset 192 00000800:00000200:4.0:1616513094.960494:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513094.960496:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513094.960499:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.28@tcp1 00000800:00000200:4.0:1616513094.960500:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513094.960506:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.28@tcp1 routed destination 00000800:00000200:11.2:1616513094.960506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513094.960508:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513094.960522:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.28@tcp1(10.31.141.28@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.960528:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.960534:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.960537:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.960541:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.960546:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513094.960554:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513094.960560:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513094.960566:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513094.960569:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513094.960571:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513094.960576:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513094.960581:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb078 00000800:00000200:12.0:1616513094.960585:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513094.960611:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.960618:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513094.960621:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513094.960628:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.960632:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3ba0 00000800:00000200:22.0:1616513094.960636:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.960639:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513094.967437:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513094.967492:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513094.967495:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513094.967501:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513094.967509:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.55@tcp1 : PUT - for me 00000400:00000200:18.0:1616513094.967519:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.55@tcp1 of length 224 into portal 26 MB=0x600194aaaa5c0 00000400:00000200:18.0:1616513094.967525:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.55@tcp1 of length 224/224 into md 0xf97cf159 [1] + 0 00000400:00000200:18.0:1616513094.967529:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be770 00000400:00000200:18.0:1616513094.967531:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513094.967534:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513094.967538:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be770 00000800:00000200:18.0:1616513094.967541:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.967543:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513094.967545:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.967548:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513094.967564:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.967574:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513094.967583:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.55@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.967593:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.55@tcp1 of length 224 into portal 12 MB=0x600194aaaa6c0 00000400:00000200:16.0:1616513094.967599:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.55@tcp1 of length 224/224 into md 0x865be291 [1] + 52848 00000400:00000200:16.0:1616513094.967604:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513094.967604:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688958487143872 00000100:00000200:16.0:1616513094.967609:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513094.967615:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:4.0:1616513094.967615:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688958487143872/t0(0) o400->9a97b66c-c8db-1502-cf17-94c3a01a7a2e@10.31.133.55@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513094.967618:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513094.967620:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513094.967624:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513094.967630:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688958487143872, offset 192 00000400:00000200:4.0:1616513094.967638:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.55@tcp1 00000400:00000200:4.0:1616513094.967645:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.55@tcp1 routed destination 00000400:00000200:4.0:1616513094.967660:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.55@tcp1(10.31.133.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513094.967667:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513094.967672:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513094.967675:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513094.967677:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513094.967678:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688958487144128 00000800:00000200:4.0:1616513094.967682:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.967689:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513094.967689:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:24.0:1616513094.967691:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6720900 x1688958487144128/t0(0) o400->2da0916f-4af8-08fe-a42f-4bf5367e468b@10.31.133.55@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513094.967705:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688958487144128, offset 192 00000400:00000200:24.0:1616513094.967711:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.55@tcp1 00000400:00000200:24.0:1616513094.967719:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.55@tcp1 routed destination 00000400:00000200:24.0:1616513094.967734:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.55@tcp1(10.31.133.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.967740:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513094.967744:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513094.967746:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.967749:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.967751:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513094.967751:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513094.967753:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513094.967756:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513094.967760:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513094.967762:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513094.967765:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb6d8 00000800:00000200:20.0:1616513094.967769:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513094.967772:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513094.967816:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513094.967823:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513094.967826:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513094.967832:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513094.967836:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3760 00000800:00000200:22.0:1616513094.967839:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513094.967842:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513094.970512:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929088, portal 4 00000100:00000200:10.0:1616513094.970517:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929088, offset 0 00000400:00000200:10.0:1616513094.970523:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:10.0:1616513094.970529:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:10.0:1616513094.970540:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513094.970546:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:10.0:1616513094.970551:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513094.970554:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:10.0:1616513094.970556:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:10.0:1616513094.970560:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513094.970609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513094.970627:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:0.0:1616513094.970633:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:0.0:1616513094.970636:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513094.970643:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:0.0:1616513094.970649:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf85e80 x1693067725929088/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513094.970662:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83ba0 00000800:00000200:0.0:1616513094.970666:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513094.970668:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513094.970783:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:20.0:1616513094.970847:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513094.970856:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513094.970865:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513094.970872:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627a80 00000400:00000200:20.0:1616513094.970879:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dc201 [1] + 192 00000400:00000200:20.0:1616513094.970884:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513094.970890:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf85e80 x1693067725929088/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513094.970900:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513094.970904:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513094.970907:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513094.970909:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:10.0:1616513094.970910:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf85e80 x1693067725929088/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513094.970922:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf85e80 x1693067725929088/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513101 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513094.970931:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83d38 00000800:00000200:11.2:1616513094.974195:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513094.974251:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513094.974261:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513094.974270:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.233@tcp1 : PUT - for me 00000400:00000200:0.0:1616513094.974278:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.233@tcp1 of length 224 into portal 26 MB=0x603e102400dc0 00000400:00000200:0.0:1616513094.974285:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.233@tcp1 of length 224/224 into md 0xf97cf169 [1] + 0 00000400:00000200:0.0:1616513094.974289:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02beee0 00000400:00000200:0.0:1616513094.974292:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513094.974294:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513094.974298:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beee0 00000800:00000200:0.0:1616513094.974301:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513094.974303:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513094.974305:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513094.974311:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513094.974314:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513094.974314:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513094.974322:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.233@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.974330:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.233@tcp1 of length 224 into portal 12 MB=0x603e102400e80 00000400:00000200:16.0:1616513094.974336:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.233@tcp1 of length 224/224 into md 0x865be291 [1] + 53072 00000400:00000200:16.0:1616513094.974341:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.974344:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513094.974350:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.974353:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513094.974354:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513094.974361:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114800541120 00010000:00000200:4.0:1616513094.974372:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1693114800541120/t0(0) o400->ad589bbf-2394-3af0-7a99-777c6c11aefe@10.31.136.233@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513094.974387:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114800541120, offset 192 00000400:00000200:4.0:1616513094.974393:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.233@tcp1 00000400:00000200:4.0:1616513094.974400:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.233@tcp1 routed destination 00000100:00000200:24.0:1616513094.974414:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114800541312 00000400:00000200:4.0:1616513094.974415:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.233@tcp1(10.31.136.233@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513094.974422:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:24.0:1616513094.974426:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6720d80 x1693114800541312/t0(0) o400->a969d1d0-4da8-db42-6a74-117dde9e4334@10.31.136.233@tcp1:720/0 lens 224/224 e 0 to 0 dl 1616513100 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:4.0:1616513094.974427:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513094.974430:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513094.974432:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513094.974451:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513094.974455:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114800541312, offset 192 00000800:00000200:11.2:1616513094.974457:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513094.974459:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513094.974462:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.233@tcp1 00000400:00000200:24.0:1616513094.974479:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.233@tcp1 routed destination 00000400:00000200:24.0:1616513094.974493:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.233@tcp1(10.31.136.233@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513094.974499:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513094.974504:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513094.974507:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513094.974509:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513094.974513:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513094.974517:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513094.974519:0:51055:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513094.974524:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513094.974527:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513094.974528:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513094.974535:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513094.974540:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba5d8 00000800:00000200:20.0:1616513094.974543:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513094.974570:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513094.974576:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513094.974579:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513094.974585:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513094.974589:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2cc0 00000800:00000200:18.0:1616513094.974593:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513094.974595:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513094.994904:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513094.994959:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513094.994968:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513094.994976:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:16.0:1616513094.994984:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 224 into portal 26 MB=0x603991342ad80 00000400:00000200:16.0:1616513094.994991:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.12.5@tcp1 of length 224/224 into md 0xf97cf179 [1] + 0 00000400:00000200:16.0:1616513094.994995:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf188 00000400:00000200:16.0:1616513094.994998:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513094.995000:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513094.995006:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf188 00000800:00000200:16.0:1616513094.995009:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513094.995012:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513094.995012:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513094.995016:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513094.995024:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805848280448 00010000:00000200:8.0:1616513094.995034:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1692805848280448/t0(0) o400->17bb2a68-9bfc-c713-623a-851028f8f660@10.242.12.5@tcp1:51/0 lens 224/224 e 0 to 0 dl 1616513186 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513094.995046:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692805848280448, offset 192 00000400:00000200:8.0:1616513094.995052:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:8.0:1616513094.995059:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:8.0:1616513094.995072:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513094.995077:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513094.995082:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513094.995085:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513094.995086:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513094.995091:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513094.995096:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513094.995096:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513094.995149:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513094.995155:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513094.995158:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513094.995164:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513094.995169:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3298 00000800:00000200:0.0:1616513094.995173:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513094.995176:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.049171:0:51062:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513095.049232:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513095.049243:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513095.049253:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.44@tcp1 : PUT - for me 00000400:00000200:22.0:1616513095.049264:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.44@tcp1 of length 224 into portal 26 MB=0x60009f328f3c0 00000400:00000200:22.0:1616513095.049271:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.44@tcp1 of length 224/224 into md 0xf97cf1a9 [1] + 0 00000400:00000200:22.0:1616513095.049277:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be6e8 00000400:00000200:22.0:1616513095.049280:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513095.049283:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513095.049288:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be6e8 00000800:00000200:14.0:1616513095.049289:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513095.049294:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513095.049298:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513095.049299:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.049301:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:22.0:1616513095.049302:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513095.049311:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.44@tcp1 : PUT - for me 00000100:00000200:8.0:1616513095.049316:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688892594516928 00000400:00000200:14.0:1616513095.049321:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.44@tcp1 of length 224 into portal 12 MB=0x60009f328f540 00000400:00000200:14.0:1616513095.049327:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.44@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 20496 00010000:00000200:8.0:1616513095.049329:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688892594516928/t0(0) o400->1d4028ad-0c41-bb76-513b-de826b8ff92a@10.31.142.44@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513095.049333:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.049336:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513095.049345:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513095.049345:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688892594516928, offset 192 00000800:00000200:14.0:1616513095.049348:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.049350:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513095.049351:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.44@tcp1 00000400:00000200:8.0:1616513095.049359:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.44@tcp1 routed destination 00000100:00000200:30.0:1616513095.049370:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688892594517312 00000400:00000200:8.0:1616513095.049375:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.44@tcp1(10.31.142.44@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.049382:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:30.0:1616513095.049384:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e07980 x1688892594517312/t0(0) o400->c4ac6f9e-8466-8ab9-0ed9-81dc78fca58c@10.31.142.44@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:8.0:1616513095.049388:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.049391:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.049393:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:30.0:1616513095.049399:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688892594517312, offset 192 00000800:00000200:8.0:1616513095.049399:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.049404:0:51062:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:30.0:1616513095.049405:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.44@tcp1 00000100:00000200:8.0:1616513095.049406:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:30.0:1616513095.049413:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.44@tcp1 routed destination 00000400:00000200:30.0:1616513095.049429:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.44@tcp1(10.31.142.44@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513095.049443:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513095.049449:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513095.049452:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513095.049454:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513095.049459:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513095.049460:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513095.049467:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513095.049468:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513095.049471:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513095.049472:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513095.049478:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513095.049482:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513095.049484:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2550 00000800:00000200:12.0:1616513095.049489:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513095.049497:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513095.049497:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513095.049504:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513095.049508:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef540 00000800:00000200:12.0:1616513095.049512:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513095.049515:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513095.050483:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929104, portal 4 00000100:00000200:10.0:1616513095.050492:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929104, offset 0 00000400:00000200:10.0:1616513095.050498:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:10.0:1616513095.050504:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:10.0:1616513095.050517:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.050524:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:10.0:1616513095.050530:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.050533:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:10.0:1616513095.050536:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:10.0:1616513095.050541:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513095.050588:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513095.050608:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513095.050614:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513095.050617:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513095.050624:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513095.050631:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9640900 x1693067725929104/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513095.050644:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83078 00000800:00000200:27.0:1616513095.050649:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513095.050652:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513095.050760:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513095.050770:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513095.050776:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513095.050785:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513095.050793:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627a90 00000400:00000200:27.0:1616513095.050800:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371672dd [1] + 192 00000400:00000200:27.0:1616513095.050805:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513095.050812:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9640900 x1693067725929104/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:31.0:1616513095.050821:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513095.050826:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513095.050829:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513095.050831:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:10.0:1616513095.050836:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9640900 x1693067725929104/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513095.050849:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9640900 x1693067725929104/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513095.050860:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83d38 00000800:00000200:7.2:1616513095.054757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513095.054813:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.054823:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513095.054832:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.231@tcp1 : PUT - for me 00000400:00000200:18.0:1616513095.054841:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.231@tcp1 of length 224 into portal 26 MB=0x601878ed29940 00000400:00000200:18.0:1616513095.054847:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.231@tcp1 of length 224/224 into md 0xf97cf1b9 [1] + 0 00000400:00000200:18.0:1616513095.054852:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfc28 00000400:00000200:18.0:1616513095.054855:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513095.054857:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513095.054862:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfc28 00000800:00000200:18.0:1616513095.054865:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.054867:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.054869:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513095.054871:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513095.054881:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690531588643136 00010000:00000200:8.0:1616513095.054890:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1690531588643136/t0(0) o400->26c62256-4449-3020-769b-b1b5b86522a2@10.31.141.231@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.054903:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690531588643136, offset 192 00000800:00000200:7.2:1616513095.054906:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513095.054909:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.231@tcp1 00000400:00000200:8.0:1616513095.054916:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.231@tcp1 routed destination 00000400:00000200:8.0:1616513095.054931:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.231@tcp1(10.31.141.231@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.054937:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513095.054942:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.054945:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.054947:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513095.054952:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:8.0:1616513095.054958:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513095.054959:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513095.054969:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513095.054977:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.231@tcp1 : PUT - for me 00000400:00000200:22.0:1616513095.054986:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.231@tcp1 of length 224 into portal 12 MB=0x601878ed29b40 00000400:00000200:22.0:1616513095.054992:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.231@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 20720 00000400:00000200:22.0:1616513095.054997:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513095.055000:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513095.055007:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513095.055010:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:22.0:1616513095.055012:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513095.055015:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513095.055019:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:30.0:1616513095.055020:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690531588643648 00000400:00000200:22.0:1616513095.055021:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513095.055023:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513095.055026:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:30.0:1616513095.055028:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2a400 x1690531588643648/t0(0) o400->fcfbb072-cb22-22df-aac2-db09f15b3353@10.31.141.231@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:22.0:1616513095.055031:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3d38 00000800:00000200:22.0:1616513095.055034:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513095.055041:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690531588643648, offset 192 00000400:00000200:30.0:1616513095.055046:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.231@tcp1 00000400:00000200:30.0:1616513095.055053:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.231@tcp1 routed destination 00000400:00000200:30.0:1616513095.055066:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.231@tcp1(10.31.141.231@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513095.055072:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513095.055077:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513095.055080:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513095.055083:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513095.055087:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.055095:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.055150:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.055157:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513095.055160:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513095.055167:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513095.055175:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:0.0:1616513095.055179:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.055182:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513095.075951:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.076006:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.076016:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513095.076025:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.83@tcp1 : PUT - for me 00000400:00000200:12.0:1616513095.076034:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.83@tcp1 of length 224 into portal 26 MB=0x604a43a1d5040 00000400:00000200:12.0:1616513095.076041:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.83@tcp1 of length 224/224 into md 0xf97cf1e9 [1] + 0 00000400:00000200:12.0:1616513095.076046:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf980 00000400:00000200:12.0:1616513095.076049:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513095.076052:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513095.076057:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf980 00000800:00000200:16.0:1616513095.076059:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513095.076060:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.076063:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513095.076065:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513095.076076:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693953256411200 00010000:00000200:8.0:1616513095.076084:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1693953256411200/t0(0) o400->81918792-84ae-a108-9a09-3facc7098a08@10.31.140.83@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.076096:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693953256411200, offset 192 00000400:00000200:8.0:1616513095.076102:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.83@tcp1 00000400:00000200:8.0:1616513095.076109:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.83@tcp1 routed destination 00000400:00000200:8.0:1616513095.076121:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.83@tcp1(10.31.140.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.076127:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513095.076133:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.076135:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.076137:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.076142:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513095.076147:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513095.076149:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.076207:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.076214:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513095.076217:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513095.076223:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513095.076229:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ed0 00000800:00000200:18.0:1616513095.076233:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.076236:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.076242:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513095.076250:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.83@tcp1 : PUT - for me 00000400:00000200:18.0:1616513095.076258:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.83@tcp1 of length 224 into portal 12 MB=0x604a43a1d5b80 00000800:00000200:2.0:1616513095.076262:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513095.076265:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.83@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 20944 00000400:00000200:18.0:1616513095.076270:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513095.076272:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513095.076279:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.076281:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.076283:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513095.076298:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693953256414080 00010000:00000200:30.0:1616513095.076308:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd702d00 x1693953256414080/t0(0) o400->4bcf395c-371a-086b-968b-d47e43b6d65d@10.31.140.83@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513095.076322:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693953256414080, offset 192 00000400:00000200:30.0:1616513095.076327:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.83@tcp1 00000400:00000200:30.0:1616513095.076334:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.83@tcp1 routed destination 00000400:00000200:30.0:1616513095.076347:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.83@tcp1(10.31.140.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513095.076353:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513095.076358:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513095.076360:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513095.076362:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513095.076367:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.076372:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.076428:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.076448:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513095.076451:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513095.076457:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513095.076462:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eea18 00000800:00000200:0.0:1616513095.076466:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.076469:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:6.0:1616513095.082475:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929120, portal 4 00000100:00000200:6.0:1616513095.082481:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929120, offset 0 00000400:00000200:6.0:1616513095.082488:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513095.082494:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:6.0:1616513095.082506:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513095.082512:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:6.0:1616513095.082518:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513095.082521:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:6.0:1616513095.082523:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:6.0:1616513095.082528:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513095.082575:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513095.082632:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513095.082639:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513095.082642:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513095.082644:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513095.082649:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513095.082655:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf86300 x1693067725929120/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:25.2:1616513095.082667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000400:00000200:22.0:1616513095.082667:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00110 00000800:00000200:22.0:1616513095.082671:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513095.082674:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513095.082680:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513095.082683:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513095.082685:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000800:00000200:22.0:1616513095.082686:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513095.082691:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513095.082778:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513095.082789:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513095.082797:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.229@o2ib1 00000800:00000200:22.0:1616513095.082799:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513095.082806:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513095.082814:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627aa0 00000400:00000200:2.0:1616513095.082820:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cfa59 [1] + 192 00000400:00000200:2.0:1616513095.082825:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513095.082831:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf86300 x1693067725929120/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513095.082846:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513095.082849:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:6.0:1616513095.082851:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf86300 x1693067725929120/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513095.082851:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:6.0:1616513095.082863:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf86300 x1693067725929120/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513095.082873:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:7.2:1616513095.128465:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513095.128523:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.128533:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513095.128543:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.238@tcp1 : PUT - for me 00000400:00000200:18.0:1616513095.128553:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.238@tcp1 of length 224 into portal 12 MB=0x5fffbcce3cd80 00000400:00000200:18.0:1616513095.128560:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.238@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 21168 00000400:00000200:18.0:1616513095.128566:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513095.128568:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513095.128576:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.128580:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513095.128582:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.128585:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513095.128597:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688831822908800 00010000:00000200:30.0:1616513095.128608:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd700900 x1688831822908800/t0(0) o400->39f3fdc2-47f2-b051-04fb-a942d1307ef1@10.31.141.238@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513095.128621:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688831822908800, offset 192 00000400:00000200:30.0:1616513095.128627:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.238@tcp1 00000400:00000200:30.0:1616513095.128633:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.238@tcp1 routed destination 00000800:00000200:11.2:1616513095.128634:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:30.0:1616513095.128646:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.238@tcp1(10.31.141.238@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513095.128652:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513095.128658:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513095.128660:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513095.128662:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513095.128667:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.128673:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513095.128694:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.128706:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513095.128715:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.238@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.128724:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.238@tcp1 of length 224 into portal 26 MB=0x5fffbcce3ca80 00000800:00000200:22.0:1616513095.128727:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513095.128730:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.238@tcp1 of length 224/224 into md 0xf97cf209 [1] + 0 00000800:00000200:22.0:1616513095.128735:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:16.0:1616513095.128737:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfb18 00000400:00000200:22.0:1616513095.128738:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513095.128740:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.128743:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513095.128744:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.128749:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfb18 00000400:00000200:22.0:1616513095.128752:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef5c8 00000800:00000200:16.0:1616513095.128752:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.128752:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.128755:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513095.128756:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513095.128757:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513095.128759:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513095.128769:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688831822908032 00010000:00000200:8.0:1616513095.128779:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688831822908032/t0(0) o400->3aab4823-bb32-d055-8954-bb5cbe0f1bfa@10.31.141.238@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.128793:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688831822908032, offset 192 00000400:00000200:8.0:1616513095.128799:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.238@tcp1 00000400:00000200:8.0:1616513095.128806:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.238@tcp1 routed destination 00000400:00000200:8.0:1616513095.128820:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.238@tcp1(10.31.141.238@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.128826:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513095.128831:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.128834:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513095.128836:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.128841:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.128845:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513095.128847:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513095.128901:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.128908:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513095.128910:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513095.128917:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513095.128923:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3870 00000800:00000200:0.0:1616513095.128926:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.128929:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:6.0:1616513095.146571:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929136, portal 4 00000100:00000200:6.0:1616513095.146575:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929136, offset 0 00000400:00000200:6.0:1616513095.146581:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:6.0:1616513095.146586:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:6.0:1616513095.146596:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513095.146602:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:6.0:1616513095.146608:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513095.146610:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:6.0:1616513095.146612:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:6.0:1616513095.146617:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513095.146663:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513095.146721:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513095.146729:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513095.146731:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513095.146738:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513095.146744:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d7980 x1693067725929136/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513095.146757:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00220 00000800:00000200:14.0:1616513095.146761:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513095.146764:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513095.147126:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513095.147189:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513095.147199:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:2.0:1616513095.147202:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513095.147207:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513095.147216:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627ab0 00000400:00000200:22.0:1616513095.147223:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cfa99 [1] + 192 00000400:00000200:22.0:1616513095.147228:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513095.147233:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d7980 x1693067725929136/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513095.147248:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513095.147251:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:6.0:1616513095.147252:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d7980 x1693067725929136/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513095.147253:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:6.0:1616513095.147262:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d7980 x1693067725929136/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513095.147270:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:11.2:1616513095.188685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.188743:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.188753:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513095.188764:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.81@tcp1 : PUT - for me 00000400:00000200:12.0:1616513095.188774:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.81@tcp1 of length 224 into portal 12 MB=0x5ffe76de0a840 00000400:00000200:12.0:1616513095.188781:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.81@tcp1 of length 224/224 into md 0x865be291 [1] + 53296 00000400:00000200:12.0:1616513095.188787:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513095.188789:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513095.188797:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513095.188797:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.188800:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513095.188802:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513095.188867:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688744329521216 00000800:00000200:7.2:1616513095.188872:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:24.0:1616513095.188882:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4536053f00 x1688744329521216/t0(0) o400->06222c4a-5478-d3c7-d1d7-2f75fb8c24e0@10.31.141.81@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513095.188898:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688744329521216, offset 192 00000400:00000200:24.0:1616513095.188906:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.81@tcp1 00000400:00000200:24.0:1616513095.188914:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.81@tcp1 routed destination 00000400:00000200:24.0:1616513095.188930:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.81@tcp1(10.31.141.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513095.188933:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513095.188936:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513095.188942:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513095.188944:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513095.188945:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513095.188947:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:24.0:1616513095.188953:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513095.188961:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.81@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.188970:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.81@tcp1 of length 224 into portal 26 MB=0x5ffe76de0a780 00000400:00000200:14.0:1616513095.188977:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.81@tcp1 of length 224/224 into md 0xf97cf229 [1] + 0 00000400:00000200:14.0:1616513095.188982:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bea18 00000400:00000200:14.0:1616513095.188985:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513095.188987:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000100:00000200:14.0:1616513095.188988:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513095.188993:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bea18 00000800:00000200:18.0:1616513095.188995:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513095.188996:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000400:00000200:18.0:1616513095.188998:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513095.188999:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513095.189001:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.189004:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513095.189005:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513095.189011:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3a08 00000800:00000200:18.0:1616513095.189016:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513095.189017:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688744329521024 00010000:00000200:8.0:1616513095.189029:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688744329521024/t0(0) o400->42997430-72ad-d2ed-fdce-a95efae99f13@10.31.141.81@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.189044:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688744329521024, offset 192 00000400:00000200:8.0:1616513095.189051:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.81@tcp1 00000400:00000200:8.0:1616513095.189058:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.81@tcp1 routed destination 00000400:00000200:8.0:1616513095.189073:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.81@tcp1(10.31.141.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.189080:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513095.189086:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.189089:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513095.189091:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.189096:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.189101:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513095.189103:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513095.189156:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.189163:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513095.189166:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513095.189173:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513095.189178:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3870 00000800:00000200:0.0:1616513095.189182:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.189185:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.191342:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513095.191397:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513095.191407:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513095.191416:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.121@tcp1 : PUT - for me 00000400:00000200:22.0:1616513095.191425:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.121@tcp1 of length 224 into portal 26 MB=0x6037f1051a840 00000400:00000200:22.0:1616513095.191432:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.121@tcp1 of length 224/224 into md 0xf97cf249 [1] + 0 00000800:00000200:14.0:1616513095.191451:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513095.191453:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be198 00000400:00000200:22.0:1616513095.191456:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513095.191459:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513095.191462:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513095.191463:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be198 00000800:00000200:22.0:1616513095.191467:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:22.0:1616513095.191469:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513095.191470:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.121@tcp1 : PUT - for me 00000800:00000200:22.0:1616513095.191471:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513095.191474:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513095.191474:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692694129780800 00000400:00000200:14.0:1616513095.191479:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.121@tcp1 of length 224 into portal 12 MB=0x6037f1051a8c0 00010000:00000200:8.0:1616513095.191479:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1692694129780800/t0(0) o400->1a385b23-e3e0-2184-7d28-3032f85d8d91@10.31.143.121@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513095.191485:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.121@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 21392 00000100:00000200:8.0:1616513095.191489:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692694129780800, offset 192 00000400:00000200:14.0:1616513095.191490:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.191493:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513095.191494:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.121@tcp1 00000400:00000200:8.0:1616513095.191499:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.121@tcp1 routed destination 00000800:00000200:14.0:1616513095.191500:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.191503:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.191505:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513095.191511:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.121@tcp1(10.31.143.121@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:30.0:1616513095.191520:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692694129780928 00000800:00000200:8.0:1616513095.191520:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513095.191524:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.191526:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.191528:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:30.0:1616513095.191530:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc7e02880 x1692694129780928/t0(0) o400->e46ab10b-c0b4-d26b-e571-21ed2ec1d700@10.31.143.121@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:8.0:1616513095.191531:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513095.191535:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513095.191536:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:30.0:1616513095.191544:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692694129780928, offset 192 00000400:00000200:30.0:1616513095.191551:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.121@tcp1 00000400:00000200:30.0:1616513095.191558:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.121@tcp1 routed destination 00000400:00000200:30.0:1616513095.191573:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.121@tcp1(10.31.143.121@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513095.191580:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513095.191585:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513095.191588:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513095.191590:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513095.191591:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:30.0:1616513095.191594:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513095.191598:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513095.191599:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513095.191601:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513095.191603:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513095.191608:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513095.191610:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513095.191613:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3d38 00000800:00000200:16.0:1616513095.191617:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513095.191617:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513095.191619:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513095.191626:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.191630:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeb28 00000800:00000200:16.0:1616513095.191634:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.191637:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:3.0:1616513095.194588:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929152, portal 4 00000100:00000200:3.0:1616513095.194594:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929152, offset 0 00000400:00000200:3.0:1616513095.194601:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:3.0:1616513095.194608:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:3.0:1616513095.194622:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513095.194629:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513095.194636:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513095.194639:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:3.0:1616513095.194641:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:31.0:1616513095.194642:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929168, portal 4 00000800:00000200:3.0:1616513095.194647:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:31.0:1616513095.194649:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929168, offset 0 00000400:00000200:31.0:1616513095.194656:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:5.2:1616513095.194656:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:31.0:1616513095.194662:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:31.0:1616513095.194675:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513095.194682:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:31.0:1616513095.194688:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513095.194691:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:31.0:1616513095.194694:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:31.0:1616513095.194699:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:0.0:1616513095.194711:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:0.0:1616513095.194719:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:0.0:1616513095.194722:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513095.194729:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:0.0:1616513095.194735:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bf9680 x1693067725929152/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.2:1616513095.194745:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:0.0:1616513095.194748:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2880 00000800:00000200:0.0:1616513095.194752:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513095.194755:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.0:1616513095.194765:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513095.194770:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513095.194772:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513095.194775:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513095.194779:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513095.194785:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be42400 x1693067725929168/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513095.194798:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c33a8 00000800:00000200:27.0:1616513095.194802:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:5.2:1616513095.194865:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:27.2:1616513095.194922:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:16.0:1616513095.194924:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513095.194932:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:16.0:1616513095.194933:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:27.0:1616513095.194939:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513095.194942:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:16.0:1616513095.194943:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513095.194947:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513095.194951:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627ac0 00000400:00000200:27.0:1616513095.194955:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627ad0 00000400:00000200:16.0:1616513095.194958:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dc2b1 [1] + 192 00000400:00000200:27.0:1616513095.194962:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371672fd [1] + 192 00000400:00000200:16.0:1616513095.194964:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:27.0:1616513095.194967:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.194970:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bf9680 x1693067725929152/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513095.194972:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be42400 x1693067725929168/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:12.0:1616513095.194977:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513095.194985:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513095.194986:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513095.194988:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:16.0:1616513095.194989:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:27.0:1616513095.194990:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:16.0:1616513095.194991:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:31.0:1616513095.194992:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be42400 x1693067725929168/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513095.194993:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bf9680 x1693067725929152/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513095.195005:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be42400 x1693067725929168/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513095.195006:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bf9680 x1693067725929152/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513095.195015:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000400:00000200:3.0:1616513095.195017:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:7.2:1616513095.200024:0:51078:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513095.200079:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.200089:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513095.200098:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.169@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.200108:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.169@tcp1 of length 224 into portal 26 MB=0x5ffee40751dc0 00000400:00000200:14.0:1616513095.200115:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.169@tcp1 of length 224/224 into md 0xf97cf259 [1] + 0 00000400:00000200:14.0:1616513095.200119:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922da08 00000400:00000200:14.0:1616513095.200122:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.200125:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513095.200129:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922da08 00000800:00000200:14.0:1616513095.200132:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.200134:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.200135:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.200139:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513095.200143:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688773632269760 00010000:00000200:8.0:1616513095.200149:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688773632269760/t0(0) o400->93aa3a46-b700-69aa-0f0e-b8402595a3d6@10.31.141.169@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.200162:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688773632269760, offset 192 00000400:00000200:8.0:1616513095.200167:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.169@tcp1 00000400:00000200:8.0:1616513095.200173:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.169@tcp1 routed destination 00000400:00000200:8.0:1616513095.200187:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.169@tcp1(10.31.141.169@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.200192:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513095.200197:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.200199:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.200201:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.200204:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.200208:0:51078:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513095.200209:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513095.200263:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513095.200270:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513095.200273:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513095.200274:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513095.200279:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513095.200285:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3298 00000800:00000200:22.0:1616513095.200289:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513095.200292:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513095.200337:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.200353:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513095.200362:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.169@tcp1 : PUT - for me 00000400:00000200:0.0:1616513095.200370:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.169@tcp1 of length 224 into portal 12 MB=0x5ffee40751f80 00000400:00000200:0.0:1616513095.200377:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.169@tcp1 of length 224/224 into md 0x865be291 [1] + 53520 00000400:00000200:0.0:1616513095.200382:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.200385:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513095.200391:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.200394:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513095.200396:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.200399:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513095.200467:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688773632270208 00010000:00000200:24.0:1616513095.200479:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde868900 x1688773632270208/t0(0) o400->fc3d3ace-fb14-5c68-753d-c0aba56bd645@10.31.141.169@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513095.200494:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688773632270208, offset 192 00000400:00000200:24.0:1616513095.200500:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.169@tcp1 00000400:00000200:24.0:1616513095.200507:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.169@tcp1 routed destination 00000400:00000200:24.0:1616513095.200522:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.169@tcp1(10.31.141.169@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513095.200529:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513095.200534:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513095.200537:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513095.200539:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513095.200544:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.200548:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.200601:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.200608:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513095.200610:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513095.200613:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513095.200617:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513095.200621:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3320 00000800:00000200:12.0:1616513095.200625:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513095.210477:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929184, portal 4 00000100:00000200:10.0:1616513095.210482:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929184, offset 0 00000400:00000200:10.0:1616513095.210488:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:10.0:1616513095.210493:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:10.0:1616513095.210504:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.210510:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:10.0:1616513095.210515:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.210518:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513095.210520:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513095.210524:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513095.210571:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513095.210587:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513095.210593:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513095.210596:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513095.210599:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513095.210602:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513095.210609:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c9bf9680 x1693067725929184/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513095.210621:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82d48 00000800:00000200:18.0:1616513095.210625:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513095.210661:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513095.210674:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513095.210689:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513095.210699:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513095.210706:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627ae0 00000400:00000200:22.0:1616513095.210713:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cfaf9 [1] + 192 00000400:00000200:22.0:1616513095.210719:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513095.210724:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c9bf9680 x1693067725929184/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513095.210736:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513095.210739:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513095.210742:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:10.0:1616513095.210743:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c9bf9680 x1693067725929184/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513095.210744:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:10.0:1616513095.210754:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c9bf9680 x1693067725929184/t0(0) o13->boslfs02-OST0004-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513095.210763:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83d38 00000100:00000200:18.0:1616513095.242639:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929200, portal 4 00000100:00000200:18.0:1616513095.242645:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929200, offset 0 00000400:00000200:18.0:1616513095.242651:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:18.0:1616513095.242657:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:18.0:1616513095.242670:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513095.242676:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:18.0:1616513095.242683:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513095.242685:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:18.0:1616513095.242687:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:18.0:1616513095.242693:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513095.242742:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.242762:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513095.242768:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513095.242771:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513095.242781:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513095.242789:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c9bff980 x1693067725929200/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513095.242802:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6ccff0 00000800:00000200:1.0:1616513095.242808:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513095.242811:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513095.242867:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513095.242877:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.242884:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.242887:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513095.242893:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513095.242901:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627af0 00000400:00000200:1.0:1616513095.242909:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x437000d5 [1] + 192 00000400:00000200:1.0:1616513095.242914:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513095.242918:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c9bff980 x1693067725929200/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513095.242933:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.242936:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513095.242938:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:18.0:1616513095.242942:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c9bff980 x1693067725929200/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:18.0:1616513095.242954:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c9bff980 x1693067725929200/t0(0) o13->boslfs02-OST0013-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513095.242964:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc550 00000400:00000080:7.0:1616513095.259152:0:51086:0:(module.c:90:libcfs_ioctl()) libcfs ioctl cmd 3233310049 00000400:00000080:7.0:1616513095.259184:0:51086:0:(module.c:90:libcfs_ioctl()) libcfs ioctl cmd 3233310049 00000400:00000080:7.0:1616513095.259194:0:51086:0:(module.c:90:libcfs_ioctl()) libcfs ioctl cmd 3233310049 00000800:00000200:11.2:1616513095.261663:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513095.261722:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.261733:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.261743:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.91@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.261752:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.91@tcp1 of length 224 into portal 26 MB=0x602ad74699a40 00000400:00000200:20.0:1616513095.261760:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.91@tcp1 of length 224/224 into md 0xf97cf289 [1] + 0 00000400:00000200:20.0:1616513095.261764:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c440 00000400:00000200:20.0:1616513095.261767:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.261770:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513095.261775:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c440 00000800:00000200:0.0:1616513095.261778:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.261779:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.261782:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513095.261785:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.261788:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.261789:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:8.0:1616513095.261797:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691793865939520 00000400:00000200:0.0:1616513095.261797:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.91@tcp1 : PUT - for me 00000400:00000200:0.0:1616513095.261806:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.91@tcp1 of length 224 into portal 12 MB=0x602ad74699ac0 00010000:00000200:8.0:1616513095.261808:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1691793865939520/t0(0) o400->84a25618-442c-c991-6cee-0113744b00e2@10.31.141.91@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513095.261812:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.91@tcp1 of length 224/224 into md 0x865be291 [1] + 53744 00000400:00000200:0.0:1616513095.261817:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.261820:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:8.0:1616513095.261827:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691793865939520, offset 192 00000800:00000200:0.0:1616513095.261828:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.261831:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513095.261833:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.91@tcp1 00000800:00000200:0.0:1616513095.261833:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513095.261838:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.91@tcp1 routed destination 00000400:00000200:8.0:1616513095.261851:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.91@tcp1(10.31.141.91@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.261856:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513095.261862:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.261864:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.261867:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.261872:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513095.261878:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513095.261879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.261895:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:28.0:1616513095.261900:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691793865939648 00000800:00000200:14.0:1616513095.261902:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.261905:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513095.261912:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00010000:00000200:28.0:1616513095.261915:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86a880 x1691793865939648/t0(0) o400->0c88d4e0-a1cb-9c3d-d821-08b90c3a73c5@10.31.141.91@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513095.261918:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000800:00000200:14.0:1616513095.261922:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.261925:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513095.261930:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691793865939648, offset 192 00000400:00000200:28.0:1616513095.261936:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.91@tcp1 00000400:00000200:28.0:1616513095.261943:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.91@tcp1 routed destination 00000400:00000200:28.0:1616513095.261959:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.91@tcp1(10.31.141.91@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.261965:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513095.261971:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.261974:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513095.261977:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513095.261982:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.261986:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.262039:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.262046:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513095.262049:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513095.262050:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513095.262055:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513095.262059:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccb28 00000800:00000200:12.0:1616513095.262062:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.282516:0:51089:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513095.282573:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.282585:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513095.282595:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.137@tcp1 : PUT - for me 00000400:00000200:2.0:1616513095.282604:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.137@tcp1 of length 224 into portal 26 MB=0x602bfa7b20e00 00000400:00000200:2.0:1616513095.282612:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.137@tcp1 of length 224/224 into md 0xf97cf299 [1] + 0 00000400:00000200:2.0:1616513095.282619:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d5c8 00000400:00000200:2.0:1616513095.282622:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513095.282625:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513095.282629:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d5c8 00000800:00000200:2.0:1616513095.282632:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.282635:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513095.282637:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.282640:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513095.282648:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691872035737088 00010000:00000200:8.0:1616513095.282658:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1691872035737088/t0(0) o400->afb6c4ea-d7da-fb34-4ebb-482392e44fdc@10.31.142.137@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.282671:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691872035737088, offset 192 00000400:00000200:8.0:1616513095.282676:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.137@tcp1 00000400:00000200:8.0:1616513095.282682:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.137@tcp1 routed destination 00000400:00000200:8.0:1616513095.282694:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.137@tcp1(10.31.142.137@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.282699:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513095.282705:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.282707:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.282709:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.282713:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.282717:0:51089:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513095.282719:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513095.282769:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.282776:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.282778:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513095.282785:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513095.282790:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2b28 00000800:00000200:14.0:1616513095.282793:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.282796:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.283100:0:51089:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513095.283110:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.283116:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513095.283120:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513095.283124:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.137@tcp1 : PUT - for me 00000400:00000200:2.0:1616513095.283130:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.137@tcp1 of length 224 into portal 12 MB=0x602bfa7b21c80 00000400:00000200:2.0:1616513095.283136:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.137@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 21616 00000400:00000200:2.0:1616513095.283140:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513095.283142:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513095.283148:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.283150:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.283152:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513095.283169:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691872035740800 00010000:00000200:30.0:1616513095.283180:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d5a00 x1691872035740800/t0(0) o400->94c7f278-5eba-fe95-5187-e322f04e53b1@10.31.142.137@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513095.283192:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691872035740800, offset 192 00000400:00000200:30.0:1616513095.283197:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.137@tcp1 00000400:00000200:30.0:1616513095.283203:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.137@tcp1 routed destination 00000400:00000200:30.0:1616513095.283216:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.137@tcp1(10.31.142.137@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513095.283221:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513095.283226:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513095.283228:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513095.283230:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513095.283234:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.283241:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.283295:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.283302:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.283305:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513095.283312:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.283316:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee2a8 00000800:00000200:16.0:1616513095.283320:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.283325:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513095.286566:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513095.286620:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.286630:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.286639:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.99@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.286649:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.99@tcp1 of length 224 into portal 12 MB=0x5ffd9d6a706c0 00000400:00000200:20.0:1616513095.286655:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.99@tcp1 of length 224/224 into md 0x865be291 [1] + 53968 00000400:00000200:20.0:1616513095.286661:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.286663:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513095.286670:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.286673:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513095.286674:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.286678:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.286689:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:28.0:1616513095.286734:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688685957809856 00000800:00000200:14.0:1616513095.286745:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:28.0:1616513095.286746:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eb180 x1688685957809856/t0(0) o400->cbc7ea47-a59c-956c-a5c2-ef77650cd5d4@10.31.138.99@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513095.286754:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:28.0:1616513095.286760:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688685957809856, offset 192 00000400:00000200:14.0:1616513095.286762:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.99@tcp1 : PUT - for me 00000400:00000200:28.0:1616513095.286767:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.99@tcp1 00000400:00000200:14.0:1616513095.286771:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.99@tcp1 of length 224 into portal 26 MB=0x5ffd9d6a70640 00000400:00000200:28.0:1616513095.286774:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.99@tcp1 routed destination 00000400:00000200:14.0:1616513095.286777:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.99@tcp1 of length 224/224 into md 0xf97cf2c9 [1] + 0 00000400:00000200:14.0:1616513095.286781:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922de48 00000400:00000200:14.0:1616513095.286784:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.286787:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513095.286789:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.99@tcp1(10.31.138.99@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513095.286791:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922de48 00000800:00000200:14.0:1616513095.286794:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513095.286795:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513095.286797:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.286799:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513095.286801:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513095.286801:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513095.286804:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:8.0:1616513095.286805:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688685957809728 00000800:00000200:28.0:1616513095.286808:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:8.0:1616513095.286812:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688685957809728/t0(0) o400->a24a157e-a6fb-292b-ef76-c359f26b768e@10.31.138.99@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513095.286813:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.286817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513095.286823:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688685957809728, offset 192 00000400:00000200:8.0:1616513095.286828:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.99@tcp1 00000400:00000200:8.0:1616513095.286834:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.99@tcp1 routed destination 00000400:00000200:8.0:1616513095.286845:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.99@tcp1(10.31.138.99@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.286850:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513095.286854:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.286856:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513095.286858:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.286862:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.286867:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513095.286867:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513095.286872:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.286879:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513095.286882:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513095.286888:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513095.286892:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccc38 00000800:00000200:26.0:1616513095.286896:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.286899:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513095.286920:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.286927:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.286930:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513095.286933:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.286936:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.286942:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2908 00000800:00000200:16.0:1616513095.286946:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513095.290537:0:6498:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929232, portal 4 00000100:00000200:12.0:1616513095.290542:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929216, portal 4 00000100:00000200:22.0:1616513095.290543:0:6498:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929232, offset 0 00000100:00000200:12.0:1616513095.290548:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929216, offset 0 00000400:00000200:22.0:1616513095.290550:0:6498:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:12.0:1616513095.290554:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:22.0:1616513095.290555:0:6498:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:12.0:1616513095.290560:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:22.0:1616513095.290567:0:6498:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513095.290574:0:6498:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:22.0:1616513095.290580:0:6498:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513095.290581:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513095.290583:0:6498:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:22.0:1616513095.290585:0:6498:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:12.0:1616513095.290587:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:22.0:1616513095.290590:0:6498:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:12.0:1616513095.290593:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.290596:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:12.0:1616513095.290598:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (22)++ 00000100:00000200:19.0:1616513095.290600:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929248, portal 4 00000800:00000200:12.0:1616513095.290603:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (23)-- 00000100:00000200:19.0:1616513095.290606:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929248, offset 0 00000400:00000200:19.0:1616513095.290613:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:19.0:1616513095.290619:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:19.0:1616513095.290633:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513095.290640:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:19.0:1616513095.290647:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513095.290650:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:1.2:1616513095.290651:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:19.0:1616513095.290652:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:19.0:1616513095.290657:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:1.0:1616513095.290670:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000800:00000200:1.0:1616513095.290676:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (24)-- 00000400:00000200:1.0:1616513095.290679:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513095.290686:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513095.290694:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bdb600 x1693067725929232/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.2:1616513095.290703:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:1.0:1616513095.290706:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a550 00000800:00000200:1.0:1616513095.290711:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513095.290714:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:12.0:1616513095.290718:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:1.0:1616513095.290719:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:5.0:1616513095.290720:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513095.290721:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513095.290723:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:12.0:1616513095.290726:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513095.290726:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513095.290731:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdeb89f80 x1693067725929216/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513095.290732:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:12.0:1616513095.290738:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456e8d80 x1693067725929248/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513095.290742:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ee58 00000800:00000200:1.0:1616513095.290746:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000400:00000200:12.0:1616513095.290750:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707e48 00000800:00000200:12.0:1616513095.290754:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513095.290757:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:1.2:1616513095.290835:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513095.290845:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.290851:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.290853:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000400:00000200:1.0:1616513095.290860:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:5.0:1616513095.290862:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:9.0:1616513095.290864:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513095.290868:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627b10 00000400:00000200:5.0:1616513095.290870:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513095.290875:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x437000e5 [1] + 192 00000800:00000200:5.2:1616513095.290877:0:6465:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000400:00000200:1.0:1616513095.290881:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:5.0:1616513095.290885:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627b00 00000100:00000200:1.0:1616513095.290885:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bdb600 x1693067725929232/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:0.0:1616513095.290890:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:1.0:1616513095.290899:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:0.0:1616513095.290899:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:5.0:1616513095.290900:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x437000f5 [1] + 192 00000800:00000200:1.0:1616513095.290902:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513095.290904:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:5.0:1616513095.290906:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513095.290908:0:6498:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bdb600 x1693067725929232/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513095.290909:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:0.0:1616513095.290917:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627b20 00000100:00000200:22.0:1616513095.290920:0:6498:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bdb600 x1693067725929232/t0(0) o13->boslfs02-OST0033-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513095.290924:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dc341 [1] + 192 00000400:00000200:0.0:1616513095.290929:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:22.0:1616513095.290932:0:6498:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aee0 00000100:00000200:0.0:1616513095.290934:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456e8d80 x1693067725929248/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:5.0:1616513095.290937:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdeb89f80 x1693067725929216/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513095.290944:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513095.290950:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:5.0:1616513095.290952:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:0.0:1616513095.290953:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.0:1616513095.290955:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:0.0:1616513095.290955:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.0:1616513095.290957:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:19.0:1616513095.290958:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456e8d80 x1693067725929248/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:12.0:1616513095.290959:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdeb89f80 x1693067725929216/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513095.290971:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456e8d80 x1693067725929248/t0(0) o13->boslfs02-OST0019-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:12.0:1616513095.290971:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdeb89f80 x1693067725929216/t0(0) o13->boslfs02-OST000f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513095.290981:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000400:00000200:12.0:1616513095.290981:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000100:00000200:12.0:1616513095.338520:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929264, portal 4 00000100:00000200:12.0:1616513095.338526:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929264, offset 0 00000400:00000200:12.0:1616513095.338533:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:12.0:1616513095.338540:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:12.0:1616513095.338554:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.338561:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:12.0:1616513095.338568:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.338571:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:12.0:1616513095.338573:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:12.0:1616513095.338579:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513095.338625:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.338643:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513095.338649:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513095.338652:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513095.338659:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513095.338666:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdeb8e300 x1693067725929264/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513095.338678:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eb28 00000800:00000200:1.0:1616513095.338683:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513095.338686:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513095.338792:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513095.338800:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.338806:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513095.338814:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513095.338822:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627b30 00000400:00000200:1.0:1616513095.338828:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700105 [1] + 192 00000400:00000200:1.0:1616513095.338833:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513095.338838:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdeb8e300 x1693067725929264/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:5.0:1616513095.338850:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513095.338852:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513095.338855:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513095.338856:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:12.0:1616513095.338862:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdeb8e300 x1693067725929264/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:12.0:1616513095.338876:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdeb8e300 x1693067725929264/t0(0) o13->boslfs02-OST0003-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513095.338886:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:11.2:1616513095.371544:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.371600:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.371610:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513095.371620:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.89@tcp1 : PUT - for me 00000400:00000200:12.0:1616513095.371632:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.89@tcp1 of length 224 into portal 26 MB=0x5f90941f3e580 00000400:00000200:12.0:1616513095.371639:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.89@tcp1 of length 224/224 into md 0xf97cf2e9 [1] + 0 00000400:00000200:12.0:1616513095.371644:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d4b8 00000400:00000200:12.0:1616513095.371648:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513095.371651:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513095.371656:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513095.371656:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d4b8 00000800:00000200:12.0:1616513095.371659:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.371662:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513095.371664:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.371667:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513095.371667:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513095.371677:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.89@tcp1 : PUT - for me 00000100:00000200:8.0:1616513095.371678:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681193040078208 00000400:00000200:20.0:1616513095.371686:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.89@tcp1 of length 224 into portal 12 MB=0x5f90941f3e600 00010000:00000200:8.0:1616513095.371690:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1681193040078208/t0(0) o400->cd8e2722-e70e-0eda-e2cc-00e1371267ca@10.31.131.89@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513095.371693:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.89@tcp1 of length 224/224 into md 0x865be291 [1] + 54192 00000400:00000200:20.0:1616513095.371700:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.371703:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513095.371710:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513095.371710:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681193040078208, offset 192 00000800:00000200:20.0:1616513095.371713:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.371715:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513095.371716:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.89@tcp1 00000400:00000200:8.0:1616513095.371724:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.89@tcp1 routed destination 00000400:00000200:8.0:1616513095.371740:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.89@tcp1(10.31.131.89@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.371746:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513095.371753:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.371756:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.371758:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.371764:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513095.371772:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513095.371772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513095.371780:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681193040078336 00010000:00000200:28.0:1616513095.371794:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ef980 x1681193040078336/t0(0) o400->7e6a180f-e8c4-9dd0-1522-48799f40e282@10.31.131.89@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513095.371809:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681193040078336, offset 192 00000400:00000200:28.0:1616513095.371816:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.89@tcp1 00000400:00000200:28.0:1616513095.371823:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.89@tcp1 routed destination 00000400:00000200:28.0:1616513095.371838:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.89@tcp1(10.31.131.89@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.371845:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513095.371845:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513095.371851:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513095.371853:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513095.371854:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513095.371856:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513095.371856:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513095.371861:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513095.371862:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513095.371867:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513095.371868:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a08 00000800:00000200:18.0:1616513095.371872:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.371877:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513095.371879:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.371887:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513095.371890:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513095.371891:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513095.371896:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513095.371900:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd078 00000800:00000200:0.0:1616513095.371904:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.383432:0:51100:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513095.383488:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.383499:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513095.383508:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.60@tcp1 : PUT - for me 00000400:00000200:26.0:1616513095.383517:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.60@tcp1 of length 224 into portal 26 MB=0x5ffd84bda5300 00000400:00000200:26.0:1616513095.383524:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.60@tcp1 of length 224/224 into md 0xf97cf2f9 [1] + 0 00000400:00000200:26.0:1616513095.383531:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d298 00000400:00000200:26.0:1616513095.383534:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513095.383537:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513095.383541:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d298 00000800:00000200:26.0:1616513095.383544:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.383544:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.383547:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513095.383549:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513095.383561:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688679334171392 00010000:00000200:8.0:1616513095.383571:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688679334171392/t0(0) o400->6ec9e565-327c-58ed-29fa-98c6b92a85c0@10.31.128.60@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.383584:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688679334171392, offset 192 00000400:00000200:8.0:1616513095.383590:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.60@tcp1 00000400:00000200:8.0:1616513095.383597:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.60@tcp1 routed destination 00000400:00000200:8.0:1616513095.383611:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.60@tcp1(10.31.128.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.383617:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513095.383623:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.383626:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.383627:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.383632:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.383637:0:51100:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513095.383638:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513095.383691:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.383697:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513095.383700:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513095.383707:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513095.383713:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2990 00000800:00000200:18.0:1616513095.383716:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.383719:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.383728:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513095.383783:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.383792:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513095.383801:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.60@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.383810:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.60@tcp1 of length 224 into portal 12 MB=0x5ffd84bda56c0 00000400:00000200:16.0:1616513095.383817:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.60@tcp1 of length 224/224 into md 0x865be291 [1] + 54416 00000400:00000200:16.0:1616513095.383822:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.383824:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513095.383831:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.383834:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.383835:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513095.383840:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513095.383901:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688679334172352 00010000:00000200:28.0:1616513095.383914:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6723f00 x1688679334172352/t0(0) o400->df0c222e-9456-64c6-41cc-9c3fc3992e0e@10.31.128.60@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513095.383929:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688679334172352, offset 192 00000400:00000200:28.0:1616513095.383935:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.60@tcp1 00000400:00000200:28.0:1616513095.383942:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.60@tcp1 routed destination 00000400:00000200:28.0:1616513095.383957:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.60@tcp1(10.31.128.60@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.383964:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513095.383970:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.383972:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513095.383975:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513095.383980:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.383984:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.384038:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.384045:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513095.384047:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513095.384050:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513095.384054:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513095.384058:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcce58 00000800:00000200:12.0:1616513095.384062:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:15.2:1616513095.391573:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:1.0:1616513095.391629:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:1.0:1616513095.391640:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.151@o2ib1 00000400:00000200:1.0:1616513095.391649:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.151@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513095.391658:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.151@o2ib1 of length 224 into portal 26 MB=0x5f9586dcfb9c0 00000400:00000200:1.0:1616513095.391665:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.151@o2ib1 of length 224/224 into md 0xf97cf319 [1] + 0 00000400:00000200:1.0:1616513095.391670:0:6468:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d980 00000400:00000200:1.0:1616513095.391673:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513095.391676:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:9.0:1616513095.391682:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)++ 00000400:00000200:1.0:1616513095.391682:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d980 00000800:00000200:1.0:1616513095.391685:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (23)++ 00000800:00000200:1.0:1616513095.391688:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (24)-- 00000800:00000200:1.0:1616513095.391690:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (23)-- 00000800:00000200:9.0:1616513095.391692:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.151@o2ib1 00000800:00000200:1.0:1616513095.391693:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000100:00000200:8.0:1616513095.391699:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681533078321600 00000400:00000200:9.0:1616513095.391701:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.151@o2ib1 : PUT - for me 00010000:00000200:8.0:1616513095.391707:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1681533078321600/t0(0) o400->9241ddb0-c18f-012c-8e38-794bc45f506e@10.242.46.151@o2ib1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:9.0:1616513095.391709:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.151@o2ib1 of length 224 into portal 12 MB=0x5f9586dcfba40 00000400:00000200:9.0:1616513095.391716:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.151@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 39808 00000100:00000200:8.0:1616513095.391719:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681533078321600, offset 192 00000400:00000200:9.0:1616513095.391721:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:9.0:1616513095.391724:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513095.391725:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.151@o2ib1 00000800:00000200:9.0:1616513095.391732:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (21)++ 00000400:00000200:8.0:1616513095.391732:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.151@o2ib1 local destination 00000800:00000200:9.0:1616513095.391735:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:9.0:1616513095.391737:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000400:00000200:8.0:1616513095.391747:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.151@o2ib1(10.242.46.151@o2ib1:10.242.46.151@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.391753:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.151@o2ib1 00000800:00000200:8.0:1616513095.391769:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4319f80] -> 10.242.46.151@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.391771:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:8.0:1616513095.391773:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:8.0:1616513095.391777:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (22)-- 00000100:00000200:8.0:1616513095.391782:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:15.2:1616513095.391783:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:1.0:1616513095.391792:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:1.0:1616513095.391797:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87e8600] (23)-- 00000400:00000200:1.0:1616513095.391800:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513095.391801:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000100:00000200:29.0:1616513095.391805:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681533078321728 00000400:00000200:1.0:1616513095.391806:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.151@o2ib1: PUT: OK 00000400:00000200:1.0:1616513095.391812:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2110 00000800:00000200:1.0:1616513095.391815:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00010000:00000200:29.0:1616513095.391823:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4558ad0900 x1681533078321728/t0(0) o400->67eb3afe-7574-8f15-013a-b1df12d1e90c@10.242.46.151@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:29.0:1616513095.391838:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681533078321728, offset 192 00000400:00000200:29.0:1616513095.391844:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.151@o2ib1 00000400:00000200:29.0:1616513095.391851:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.151@o2ib1 local destination 00000400:00000200:29.0:1616513095.391865:0:170085:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.151@o2ib1(10.242.46.151@o2ib1:10.242.46.151@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513095.391872:0:170085:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.151@o2ib1 00000800:00000200:29.0:1616513095.391879:0:170085:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4319f80] -> 10.242.46.151@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513095.391881:0:170085:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (20)++ 00000800:00000200:29.0:1616513095.391884:0:170085:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:29.0:1616513095.391889:0:170085:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87e8600] (22)-- 00000800:00000200:15.2:1616513095.391893:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87e8600] (21)++ 00000800:00000200:5.0:1616513095.391901:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)++ 00000800:00000200:5.0:1616513095.391906:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87e8600] (23)-- 00000400:00000200:5.0:1616513095.391909:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:9.0:1616513095.391910:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (22)-- 00000400:00000200:5.0:1616513095.391916:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.151@o2ib1: PUT: OK 00000400:00000200:5.0:1616513095.391920:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b23b8 00000800:00000200:5.0:1616513095.391924:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87e8600] (21)-- 00000800:00000200:11.2:1616513095.399510:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513095.399566:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.399575:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.399585:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.25@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.399594:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.25@tcp1 of length 224 into portal 26 MB=0x6018cb645fe00 00000400:00000200:20.0:1616513095.399601:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.25@tcp1 of length 224/224 into md 0xf97cf339 [1] + 0 00000400:00000200:20.0:1616513095.399606:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c198 00000400:00000200:20.0:1616513095.399608:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.399611:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513095.399616:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c198 00000800:00000200:20.0:1616513095.399619:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.399619:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.399622:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.399624:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513095.399630:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690553725353472 00010000:00000200:8.0:1616513095.399637:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1690553725353472/t0(0) o400->52b5d358-b101-319c-992a-a3c6a6002b4f@10.31.142.25@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.399649:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690553725353472, offset 192 00000400:00000200:8.0:1616513095.399655:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.25@tcp1 00000400:00000200:8.0:1616513095.399661:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.25@tcp1 routed destination 00000400:00000200:8.0:1616513095.399675:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.25@tcp1(10.31.142.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.399681:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513095.399686:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.399688:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.399690:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.399694:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513095.399699:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513095.399702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513095.399735:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513095.399759:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.399766:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513095.399768:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513095.399775:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513095.399781:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3650 00000800:00000200:2.0:1616513095.399784:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.399787:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513095.399788:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.399797:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513095.399805:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.25@tcp1 : PUT - for me 00000400:00000200:0.0:1616513095.399813:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.25@tcp1 of length 224 into portal 12 MB=0x6018cb6460b80 00000400:00000200:0.0:1616513095.399821:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.25@tcp1 of length 224/224 into md 0x865be291 [1] + 54640 00000400:00000200:0.0:1616513095.399826:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.399829:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513095.399834:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.399837:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513095.399839:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.399842:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513095.399899:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690553725356928 00010000:00000200:28.0:1616513095.399911:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86d580 x1690553725356928/t0(0) o400->58e01dbd-4302-3970-4915-4d847119565a@10.31.142.25@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513095.399925:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690553725356928, offset 192 00000400:00000200:28.0:1616513095.399932:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.25@tcp1 00000400:00000200:28.0:1616513095.399939:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.25@tcp1 routed destination 00000400:00000200:28.0:1616513095.399953:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.25@tcp1(10.31.142.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.399959:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513095.399964:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.399967:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513095.399969:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513095.399974:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.399979:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.399992:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.399997:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.400000:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513095.400001:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.400007:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.400011:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccf68 00000800:00000200:16.0:1616513095.400014:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.406741:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513095.406797:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.406806:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513095.406815:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.85@tcp1 : PUT - for me 00000400:00000200:18.0:1616513095.406825:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.85@tcp1 of length 224 into portal 12 MB=0x603705c54e340 00000400:00000200:18.0:1616513095.406832:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.11.85@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 21840 00000400:00000200:18.0:1616513095.406837:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513095.406840:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513095.406852:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.406853:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.406855:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513095.406857:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513095.406872:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692630980551488 00010000:00000200:30.0:1616513095.406882:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd701680 x1692630980551488/t0(0) o400->1e27a69f-11d8-a575-1ac7-d9e5b187f3dd@10.31.11.85@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513095.406895:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692630980551488, offset 192 00000400:00000200:30.0:1616513095.406901:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.85@tcp1 00000400:00000200:30.0:1616513095.406908:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.85@tcp1 routed destination 00000400:00000200:30.0:1616513095.406922:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.85@tcp1(10.31.11.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513095.406927:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513095.406928:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513095.406933:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513095.406936:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513095.406938:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513095.406943:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.406948:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513095.406983:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.406992:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513095.407001:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.85@tcp1 : PUT - for me 00000800:00000200:2.0:1616513095.407004:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513095.407010:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.85@tcp1 of length 224 into portal 26 MB=0x603705c54e0c0 00000800:00000200:2.0:1616513095.407011:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513095.407013:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:12.0:1616513095.407016:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.11.85@tcp1 of length 224/224 into md 0xf97cf349 [1] + 0 00000400:00000200:2.0:1616513095.407020:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513095.407021:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922db18 00000400:00000200:12.0:1616513095.407024:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:2.0:1616513095.407024:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef298 00000100:00000200:12.0:1616513095.407027:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513095.407028:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.407031:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513095.407032:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922db18 00000800:00000200:12.0:1616513095.407035:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.407037:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513095.407038:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513095.407041:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513095.407046:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692630980550848 00010000:00000200:8.0:1616513095.407054:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1692630980550848/t0(0) o400->ec8609f7-c204-68a3-3419-5ada61f72445@10.31.11.85@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.407065:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692630980550848, offset 192 00000400:00000200:8.0:1616513095.407070:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.85@tcp1 00000400:00000200:8.0:1616513095.407076:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.85@tcp1 routed destination 00000400:00000200:8.0:1616513095.407089:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.85@tcp1(10.31.11.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.407095:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513095.407100:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.407103:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513095.407104:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.407108:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.407113:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513095.407113:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513095.407167:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.407173:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513095.407176:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513095.407183:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513095.407189:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf26e8 00000800:00000200:20.0:1616513095.407192:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.407195:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513095.413026:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513095.413066:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513095.413081:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.413090:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513095.413099:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.132@tcp1 : PUT - for me 00000400:00000200:0.0:1616513095.413109:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.132@tcp1 of length 224 into portal 12 MB=0x6054e71468780 00000400:00000200:0.0:1616513095.413115:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.132@tcp1 of length 224/224 into md 0x865be291 [1] + 54864 00000800:00000200:14.0:1616513095.413121:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513095.413121:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.413123:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513095.413130:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513095.413131:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513095.413133:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513095.413135:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.413137:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513095.413140:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.132@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.413149:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.132@tcp1 of length 224 into portal 26 MB=0x6054e71468700 00000400:00000200:14.0:1616513095.413158:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.132@tcp1 of length 224/224 into md 0xf97cf359 [1] + 0 00000400:00000200:14.0:1616513095.413163:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922dc28 00000400:00000200:14.0:1616513095.413166:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.413168:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513095.413173:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922dc28 00000800:00000200:18.0:1616513095.413175:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.413176:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.413179:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.413180:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513095.413187:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694684326299392 00010000:00000200:8.0:1616513095.413194:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1694684326299392/t0(0) o400->f6fb160c-b491-6ee4-532a-6a0b5a921e01@10.31.140.132@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.413206:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694684326299392, offset 192 00000400:00000200:8.0:1616513095.413211:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.132@tcp1 00000100:00000200:28.0:1616513095.413212:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694684326299520 00000400:00000200:8.0:1616513095.413218:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.132@tcp1 routed destination 00010000:00000200:28.0:1616513095.413225:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86c380 x1694684326299520/t0(0) o400->3e89408a-e517-c326-66f6-f59d10e7d16e@10.31.140.132@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:8.0:1616513095.413232:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.132@tcp1(10.31.140.132@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.413248:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:28.0:1616513095.413249:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694684326299520, offset 192 00000800:00000200:8.0:1616513095.413253:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513095.413255:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.132@tcp1 00000800:00000200:8.0:1616513095.413255:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.413256:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.413260:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513095.413262:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.132@tcp1 routed destination 00000100:00000200:8.0:1616513095.413265:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513095.413266:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513095.413276:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.132@tcp1(10.31.140.132@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.413282:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513095.413287:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.413290:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513095.413292:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513095.413296:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.413301:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513095.413321:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.413330:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513095.413333:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513095.413339:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513095.413345:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2440 00000800:00000200:2.0:1616513095.413349:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.413351:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513095.413355:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.413361:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513095.413364:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513095.413371:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513095.413374:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc000 00000800:00000200:20.0:1616513095.413378:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.413381:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513095.449820:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.449876:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.449886:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513095.449895:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.84@tcp1 : PUT - for me 00000400:00000200:12.0:1616513095.449905:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.84@tcp1 of length 224 into portal 26 MB=0x5ffdc4d9f6a40 00000400:00000200:12.0:1616513095.449914:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.84@tcp1 of length 224/224 into md 0xf97cf379 [1] + 0 00000400:00000200:12.0:1616513095.449919:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d188 00000400:00000200:12.0:1616513095.449922:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513095.449925:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513095.449929:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d188 00000800:00000200:0.0:1616513095.449930:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.449933:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.449936:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:12.0:1616513095.449938:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513095.449941:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.449942:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:8.0:1616513095.449947:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696543734336 00000400:00000200:0.0:1616513095.449951:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.84@tcp1 : PUT - for me 00010000:00000200:8.0:1616513095.449956:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688696543734336/t0(0) o400->2a1a7baa-40b4-7e3b-c427-fc9bafb0974a@10.31.141.84@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513095.449960:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.84@tcp1 of length 224 into portal 12 MB=0x5ffdc4d9f6ac0 00000400:00000200:0.0:1616513095.449966:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.84@tcp1 of length 224/224 into md 0x865be291 [1] + 55088 00000100:00000200:8.0:1616513095.449972:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696543734336, offset 192 00000400:00000200:0.0:1616513095.449972:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.449974:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513095.449981:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513095.449983:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.84@tcp1 00000800:00000200:0.0:1616513095.449985:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.449987:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513095.449989:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.84@tcp1 routed destination 00000400:00000200:8.0:1616513095.450003:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.84@tcp1(10.31.141.84@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.450009:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513095.450015:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.450017:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513095.450019:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.450023:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513095.450030:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513095.450049:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696543734464 00010000:00000200:28.0:1616513095.450062:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86f500 x1688696543734464/t0(0) o400->e20e24b8-de18-e35c-64d5-47cc77507493@10.31.141.84@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513095.450069:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513095.450077:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696543734464, offset 192 00000400:00000200:28.0:1616513095.450083:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.84@tcp1 00000400:00000200:28.0:1616513095.450090:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.84@tcp1 routed destination 00000400:00000200:28.0:1616513095.450105:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.84@tcp1(10.31.141.84@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.450112:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513095.450117:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.450122:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513095.450124:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513095.450128:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513095.450129:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.450134:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513095.450135:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513095.450138:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513095.450144:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513095.450150:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf35c8 00000800:00000200:18.0:1616513095.450154:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.450157:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513095.450206:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.450213:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.450216:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513095.450217:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.450222:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.450226:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd188 00000800:00000200:16.0:1616513095.450230:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:29.2:1616513095.464983:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa1000] (20)++ 00000800:00000200:20.0:1616513095.465041:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (21)++ 00000800:00000200:20.0:1616513095.465052:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.204@o2ib1 00000400:00000200:20.0:1616513095.465061:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.204@o2ib1 : PUT - for me 00000400:00000200:20.0:1616513095.465069:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.204@o2ib1 of length 224 into portal 26 MB=0x60386ef461a80 00000400:00000200:20.0:1616513095.465076:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.204@o2ib1 of length 224/224 into md 0xf97cf389 [1] + 0 00000400:00000200:20.0:1616513095.465081:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922cf68 00000400:00000200:20.0:1616513095.465083:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.465086:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513095.465091:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cf68 00000800:00000200:20.0:1616513095.465094:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa1000] (22)++ 00000800:00000200:0.0:1616513095.465096:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (22)++ 00000800:00000200:20.0:1616513095.465097:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa1000] (23)-- 00000800:00000200:20.0:1616513095.465103:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (23)-- 00000800:00000200:20.0:1616513095.465106:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (22)-- 00000800:00000200:0.0:1616513095.465107:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.204@o2ib1 00000100:00000200:8.0:1616513095.465109:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692727935113856 00000400:00000200:0.0:1616513095.465116:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.204@o2ib1 : PUT - for me 00010000:00000200:8.0:1616513095.465119:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1692727935113856/t0(0) o400->c7497b2f-7207-d905-9584-552f06d0affa@10.242.46.204@o2ib1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513095.465126:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.204@o2ib1 of length 224 into portal 12 MB=0x60386ef461b80 00000400:00000200:0.0:1616513095.465132:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.204@o2ib1 of length 224/224 into md 0x865be291 [1] + 55312 00000100:00000200:8.0:1616513095.465137:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692727935113856, offset 192 00000400:00000200:0.0:1616513095.465137:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.465140:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513095.465146:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa1000] (21)++ 00000400:00000200:8.0:1616513095.465148:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.204@o2ib1 00000800:00000200:0.0:1616513095.465150:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa1000] (22)-- 00000800:00000200:0.0:1616513095.465151:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (21)-- 00000400:00000200:8.0:1616513095.465154:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.204@o2ib1 local destination 00000400:00000200:8.0:1616513095.465166:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.204@o2ib1(10.242.46.204@o2ib1:10.242.46.204@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.465172:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.204@o2ib1 00000800:00000200:8.0:1616513095.465177:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6600] -> 10.242.46.204@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.465179:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa1000] (20)++ 00000800:00000200:8.0:1616513095.465181:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa1000] (21)++ 00000800:00000200:8.0:1616513095.465185:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa1000] (22)-- 00000800:00000200:29.2:1616513095.465190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa1000] (21)++ 00000100:00000200:8.0:1616513095.465191:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513095.465214:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692727935114112 00010000:00000200:28.0:1616513095.465227:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86ad00 x1692727935114112/t0(0) o400->a639e6da-5738-1ba4-ca02-11389207dd66@10.242.46.204@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513095.465241:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692727935114112, offset 192 00000800:00000200:16.0:1616513095.465246:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (22)++ 00000400:00000200:28.0:1616513095.465247:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.204@o2ib1 00000800:00000200:16.0:1616513095.465253:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa1000] (23)-- 00000400:00000200:28.0:1616513095.465254:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.204@o2ib1 local destination 00000400:00000200:16.0:1616513095.465255:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513095.465256:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (22)-- 00000400:00000200:16.0:1616513095.465265:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.204@o2ib1: PUT: OK 00000400:00000200:28.0:1616513095.465268:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.204@o2ib1(10.242.46.204@o2ib1:10.242.46.204@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513095.465270:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ee0 00000800:00000200:16.0:1616513095.465274:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (21)-- 00000800:00000200:28.0:1616513095.465275:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.204@o2ib1 00000800:00000200:28.0:1616513095.465281:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6600] -> 10.242.46.204@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.465283:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa1000] (20)++ 00000800:00000200:28.0:1616513095.465286:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa1000] (21)++ 00000800:00000200:28.0:1616513095.465291:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa1000] (22)-- 00000800:00000200:29.2:1616513095.465296:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa1000] (21)++ 00000800:00000200:12.0:1616513095.465307:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (22)++ 00000800:00000200:12.0:1616513095.465313:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa1000] (23)-- 00000400:00000200:12.0:1616513095.465316:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513095.465317:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (22)-- 00000400:00000200:12.0:1616513095.465322:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.204@o2ib1: PUT: OK 00000400:00000200:12.0:1616513095.465326:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd298 00000800:00000200:12.0:1616513095.465329:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa1000] (21)-- 00000100:00000200:16.0:1616513095.466549:0:6482:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929280, portal 4 00000100:00000200:16.0:1616513095.466559:0:6482:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929280, offset 0 00000400:00000200:16.0:1616513095.466566:0:6482:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:16.0:1616513095.466572:0:6482:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:16.0:1616513095.466585:0:6482:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513095.466592:0:6482:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:16.0:1616513095.466598:0:6482:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513095.466601:0:6482:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:16.0:1616513095.466603:0:6482:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:16.0:1616513095.466608:0:6482:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513095.466662:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.466681:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513095.466688:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513095.466691:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513095.466699:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513095.466706:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdeb8e300 x1693067725929280/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513095.466719:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7895c8 00000800:00000200:1.0:1616513095.466724:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513095.466727:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513095.466839:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513095.466849:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.466855:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.466858:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513095.466864:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513095.466872:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627b40 00000400:00000200:1.0:1616513095.466878:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700125 [1] + 192 00000400:00000200:1.0:1616513095.466887:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513095.466892:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdeb8e300 x1693067725929280/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513095.466906:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.466909:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513095.466911:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:16.0:1616513095.466916:0:6482:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdeb8e300 x1693067725929280/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513095.466929:0:6482:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdeb8e300 x1693067725929280/t0(0) o13->boslfs02-OST001b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:16.0:1616513095.466939:0:6482:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788e58 00000800:00000200:11.2:1616513095.468685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513095.468741:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.468750:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.468760:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.147@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.468769:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.147@tcp1 of length 224 into portal 26 MB=0x603fcdb875c40 00000400:00000200:20.0:1616513095.468776:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.147@tcp1 of length 224/224 into md 0xf97cf3b9 [1] + 0 00000400:00000200:20.0:1616513095.468781:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922df58 00000400:00000200:20.0:1616513095.468783:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.468786:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513095.468791:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922df58 00000800:00000200:20.0:1616513095.468794:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.468794:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.468796:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513095.468800:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234409987136 00000800:00000200:20.0:1616513095.468801:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.468804:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.468805:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00010000:00000200:8.0:1616513095.468805:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1693234409987136/t0(0) o400->bb39e619-fcc6-5ad3-9d65-6ec1faa6f1c2@10.31.138.147@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513095.468813:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.147@tcp1 : PUT - for me 00000100:00000200:8.0:1616513095.468815:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234409987136, offset 192 00000400:00000200:8.0:1616513095.468819:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.147@tcp1 00000400:00000200:16.0:1616513095.468822:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.147@tcp1 of length 224 into portal 12 MB=0x603fcdb875d00 00000400:00000200:8.0:1616513095.468824:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.147@tcp1 routed destination 00000400:00000200:16.0:1616513095.468829:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.147@tcp1 of length 224/224 into md 0x865be291 [1] + 55536 00000400:00000200:16.0:1616513095.468834:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513095.468836:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.147@tcp1(10.31.138.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513095.468837:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513095.468841:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513095.468844:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.468846:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513095.468847:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513095.468848:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513095.468849:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513095.468850:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513095.468852:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513095.468856:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513095.468861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513095.468906:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234409987328 00010000:00000200:28.0:1616513095.468917:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6720000 x1693234409987328/t0(0) o400->ed94a6eb-07bc-d767-1fd7-7653ba4c0fca@10.31.138.147@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513095.468919:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.468927:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513095.468929:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:28.0:1616513095.468931:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234409987328, offset 192 00000400:00000200:2.0:1616513095.468936:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513095.468937:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.147@tcp1 00000400:00000200:2.0:1616513095.468941:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3f58 00000400:00000200:28.0:1616513095.468944:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.147@tcp1 routed destination 00000800:00000200:2.0:1616513095.468945:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.468948:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513095.468957:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.147@tcp1(10.31.138.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.468963:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513095.468969:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.468971:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513095.468973:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513095.468977:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.468982:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.469033:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.469040:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:12.0:1616513095.469043:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513095.469044:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513095.469049:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513095.469052:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd8f8 00000800:00000200:12.0:1616513095.469056:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513095.471907:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513095.471964:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.471973:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513095.471979:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.471982:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.12.5@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.471991:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.12.5@tcp1 of length 336 into portal 12 MB=0x603991342dc80 00000400:00000200:16.0:1616513095.472000:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.12.5@tcp1 of length 336/336 into md 0x865be291 [1] + 55760 00000400:00000200:16.0:1616513095.472005:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.472008:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513095.472014:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.472017:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.472019:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513095.472079:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692805848292480 00010000:00000200:28.0:1616513095.472159:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6725100 x1692805848292480/t0(0) o48->78c4c409-a40e-a60c-c88b-e947dd06e880@10.242.12.5@tcp1:721/0 lens 336/336 e 0 to 0 dl 1616513101 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513095.472184:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 304 bytes to portal 10, xid 1692805848292480, offset 192 00000400:00000200:28.0:1616513095.472190:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.12.5@tcp1 00000400:00000200:28.0:1616513095.472198:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.12.5@tcp1 routed destination 00000400:00000200:28.0:1616513095.472212:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.12.5@tcp1(10.242.12.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.472219:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 304 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513095.472225:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.472227:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513095.472230:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513095.472234:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.472240:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.472295:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.472302:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513095.472305:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513095.472312:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513095.472316:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc550 00000800:00000200:18.0:1616513095.472320:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.472323:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.472328:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513095.472333:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.472335:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513095.472336:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.472339:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.479232:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513095.479250:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.479260:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.479268:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.133@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.479278:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.133@tcp1 of length 224 into portal 12 MB=0x60219fb771400 00000400:00000200:20.0:1616513095.479284:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.133@tcp1 of length 224/224 into md 0x865be291 [1] + 56096 00000400:00000200:20.0:1616513095.479291:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.479294:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513095.479300:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.479303:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513095.479303:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513095.479310:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.479313:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513095.479313:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513095.479321:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.133@tcp1 : PUT - for me 00000400:00000200:12.0:1616513095.479330:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.133@tcp1 of length 224 into portal 26 MB=0x60219fb771380 00000400:00000200:12.0:1616513095.479336:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.133@tcp1 of length 224/224 into md 0xf97cf3c9 [1] + 0 00000400:00000200:12.0:1616513095.479340:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d3a8 00000400:00000200:12.0:1616513095.479343:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513095.479345:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:12.0:1616513095.479350:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d3a8 00000800:00000200:12.0:1616513095.479353:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.479355:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513095.479359:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513095.479361:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691160476587008 00000100:00000200:8.0:1616513095.479363:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691160476586880 00010000:00000200:8.0:1616513095.479369:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1691160476586880/t0(0) o400->1221d4bc-adbd-6b0a-197d-e2e1cbe87cb2@10.31.140.133@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:28.0:1616513095.479373:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ebf00 x1691160476587008/t0(0) o400->f44ba8ed-ba10-f6db-ec3e-c3c1dad4240f@10.31.140.133@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513095.479380:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691160476586880, offset 192 00000400:00000200:8.0:1616513095.479385:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.133@tcp1 00000100:00000200:28.0:1616513095.479387:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691160476587008, offset 192 00000400:00000200:8.0:1616513095.479390:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.133@tcp1 routed destination 00000400:00000200:28.0:1616513095.479393:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.133@tcp1 00000400:00000200:8.0:1616513095.479402:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.133@tcp1(10.31.140.133@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.479408:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513095.479410:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.133@tcp1 routed destination 00000800:00000200:8.0:1616513095.479416:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.479418:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513095.479420:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.479423:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513095.479424:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.133@tcp1(10.31.140.133@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:8.0:1616513095.479428:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513095.479429:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513095.479430:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513095.479448:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.479451:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513095.479453:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513095.479457:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.479465:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513095.479483:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.479490:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.479493:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513095.479494:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.479500:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.479505:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2cc0 00000800:00000200:16.0:1616513095.479509:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513095.479520:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.479527:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513095.479530:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513095.479536:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513095.479537:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513095.479541:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd210 00000800:00000200:26.0:1616513095.479545:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.487147:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513095.487203:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.487213:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513095.487221:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.157@tcp1 : PUT - for me 00000400:00000200:0.0:1616513095.487232:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.157@tcp1 of length 224 into portal 26 MB=0x60183d6127700 00000400:00000200:0.0:1616513095.487239:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.157@tcp1 of length 224/224 into md 0xf97cf3d9 [1] + 0 00000400:00000200:0.0:1616513095.487243:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c110 00000400:00000200:0.0:1616513095.487246:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.487249:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513095.487254:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c110 00000800:00000200:12.0:1616513095.487257:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.487257:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.487260:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513095.487261:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513095.487267:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515604141824 00010000:00000200:8.0:1616513095.487273:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1690515604141824/t0(0) o400->136f8dc8-1996-ce89-d5ef-eac6cd12050f@10.31.132.157@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.487284:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690515604141824, offset 192 00000400:00000200:8.0:1616513095.487289:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.157@tcp1 00000400:00000200:8.0:1616513095.487296:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.157@tcp1 routed destination 00000400:00000200:8.0:1616513095.487309:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.157@tcp1(10.31.132.157@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.487315:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513095.487319:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.487322:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513095.487323:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.487326:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.487331:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513095.487331:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513095.487390:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.487406:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513095.487408:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513095.487414:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513095.487420:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf34b8 00000800:00000200:20.0:1616513095.487423:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.487426:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.488157:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513095.488212:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.488221:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513095.488229:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.157@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.488238:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.157@tcp1 of length 224 into portal 12 MB=0x60183d61278c0 00000400:00000200:14.0:1616513095.488244:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.157@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 22064 00000400:00000200:14.0:1616513095.488248:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.488251:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513095.488258:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.488260:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.488262:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.488265:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513095.488276:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690515604142272 00010000:00000200:30.0:1616513095.488287:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf8d80 x1690515604142272/t0(0) o400->8042b538-35a2-7745-cfca-ec0361f3c70d@10.31.132.157@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513095.488299:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690515604142272, offset 192 00000400:00000200:30.0:1616513095.488305:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.157@tcp1 00000400:00000200:30.0:1616513095.488312:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.157@tcp1 routed destination 00000400:00000200:30.0:1616513095.488325:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.157@tcp1(10.31.132.157@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513095.488331:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513095.488336:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513095.488338:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513095.488340:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513095.488344:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.488349:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.488404:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.488411:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513095.488414:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513095.488420:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513095.488424:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eedd0 00000800:00000200:18.0:1616513095.488428:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.488431:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.491121:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513095.491174:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.491184:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513095.491192:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.164@tcp1 : PUT - for me 00000400:00000200:2.0:1616513095.491201:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.164@tcp1 of length 224 into portal 26 MB=0x5ffdc6cdc1a00 00000400:00000200:2.0:1616513095.491208:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.164@tcp1 of length 224/224 into md 0xf97cf409 [1] + 0 00000400:00000200:2.0:1616513095.491212:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d870 00000400:00000200:2.0:1616513095.491215:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513095.491218:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513095.491222:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d870 00000800:00000200:2.0:1616513095.491225:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.491228:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513095.491228:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513095.491235:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:8.0:1616513095.491236:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697067805184 00000800:00000200:2.0:1616513095.491238:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.491239:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:8.0:1616513095.491242:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688697067805184/t0(0) o400->287f5d4f-24c1-db14-6fd1-550fb92932e3@10.31.141.164@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:14.0:1616513095.491247:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.164@tcp1 : PUT - for me 00000100:00000200:8.0:1616513095.491253:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697067805184, offset 192 00000400:00000200:14.0:1616513095.491255:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.164@tcp1 of length 224 into portal 12 MB=0x5ffdc6cdc1d00 00000400:00000200:8.0:1616513095.491257:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.164@tcp1 00000400:00000200:14.0:1616513095.491262:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.164@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 22288 00000400:00000200:8.0:1616513095.491263:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.164@tcp1 routed destination 00000400:00000200:14.0:1616513095.491267:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.491270:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513095.491276:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.164@tcp1(10.31.141.164@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513095.491278:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.491281:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.491283:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513095.491285:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513095.491289:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.491292:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513095.491293:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.491296:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:2.0:1616513095.491298:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697067805952 00000100:00000200:8.0:1616513095.491301:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:2.0:1616513095.491308:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bff500 x1688697067805952/t0(0) o400->7d0e0437-0943-5ae3-4b94-06ba57c982ea@10.31.141.164@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:2.0:1616513095.491322:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697067805952, offset 192 00000400:00000200:2.0:1616513095.491327:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.164@tcp1 00000400:00000200:2.0:1616513095.491333:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.164@tcp1 routed destination 00000800:00000200:11.2:1616513095.491343:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513095.491348:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.164@tcp1(10.31.141.164@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513095.491356:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513095.491361:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513095.491364:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513095.491366:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.491370:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.491375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.491390:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.491397:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513095.491399:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513095.491400:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513095.491401:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513095.491406:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:12.0:1616513095.491407:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513095.491410:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d83c0eed7e8 00000400:00000200:12.0:1616513095.491410:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513095.491414:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513095.491416:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513095.491422:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:12.0:1616513095.491425:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513095.491428:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513095.498542:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929296, portal 4 00000100:00000200:30.0:1616513095.498547:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929296, offset 0 00000400:00000200:30.0:1616513095.498552:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513095.498557:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513095.498568:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513095.498573:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513095.498578:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513095.498580:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513095.498582:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513095.498586:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513095.498633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513095.498689:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513095.498696:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513095.498698:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513095.498705:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513095.498710:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d747c697500 x1693067725929296/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513095.498721:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eedd0 00000800:00000200:20.0:1616513095.498725:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513095.498728:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513095.498823:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:12.0:1616513095.498879:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:12.0:1616513095.498888:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:12.0:1616513095.498896:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:12.0:1616513095.498903:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627b50 00000400:00000200:12.0:1616513095.498909:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dc4c1 [1] + 192 00000400:00000200:12.0:1616513095.498914:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513095.498918:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d747c697500 x1693067725929296/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:0.0:1616513095.498930:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513095.498932:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:12.0:1616513095.498935:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:12.0:1616513095.498936:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513095.498937:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d747c697500 x1693067725929296/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513095.498948:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d747c697500 x1693067725929296/t0(0) o13->boslfs02-OST002d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513095.498956:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee330 00000100:00000200:10.0:1616513095.514477:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929312, portal 4 00000100:00000200:10.0:1616513095.514482:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929312, offset 0 00000400:00000200:10.0:1616513095.514488:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:10.0:1616513095.514493:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:10.0:1616513095.514503:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.514509:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:10.0:1616513095.514514:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.514516:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513095.514518:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513095.514522:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513095.514570:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513095.514624:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513095.514630:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513095.514632:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513095.514641:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513095.514646:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf87980 x1693067725929312/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513095.514656:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82770 00000800:00000200:14.0:1616513095.514660:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513095.514662:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513095.514697:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:2.0:1616513095.514751:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513095.514761:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:26.0:1616513095.514764:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513095.514770:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513095.514778:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627b60 00000400:00000200:2.0:1616513095.514784:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cfc79 [1] + 192 00000400:00000200:2.0:1616513095.514790:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513095.514795:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf87980 x1693067725929312/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513095.514809:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000100:00000200:10.0:1616513095.514813:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf87980 x1693067725929312/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513095.514813:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513095.514815:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:10.0:1616513095.514824:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf87980 x1693067725929312/t0(0) o13->boslfs02-OST0028-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513095.514832:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83d38 00000800:00000200:1.2:1616513095.516029:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513095.516047:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.516055:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513095.516064:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513095.516073:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff360 00000400:00000200:1.0:1616513095.516079:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 40032 00000400:00000200:1.0:1616513095.516085:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513095.516088:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:1.0:1616513095.516097:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:5.0:1616513095.516099:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513095.516100:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513095.516106:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513095.516109:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:5.0:1616513095.516110:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:1.0:1616513095.516116:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:5.0:1616513095.516119:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513095.516122:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:5.0:1616513095.516127:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff3e0 00000400:00000200:1.0:1616513095.516128:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 26 MB=0x5f931434ff320 00000400:00000200:5.0:1616513095.516134:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 40256 00000400:00000200:1.0:1616513095.516134:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0xf97cf419 [1] + 0 00000400:00000200:5.0:1616513095.516139:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:1.0:1616513095.516139:0:6468:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922cb28 00000400:00000200:1.0:1616513095.516141:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:5.0:1616513095.516142:0:6467:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:1.0:1616513095.516143:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:1.0:1616513095.516148:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cb28 00000800:00000200:5.0:1616513095.516150:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)++ 00000800:00000200:1.0:1616513095.516151:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)++ 00000800:00000200:1.0:1616513095.516153:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)-- 00000800:00000200:1.0:1616513095.516154:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)-- 00000800:00000200:5.0:1616513095.516155:0:6467:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (24)++ 00000800:00000200:1.0:1616513095.516156:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000800:00000200:5.0:1616513095.516159:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)-- 00000800:00000200:1.0:1616513095.516160:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.516161:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)-- 00000100:00000200:8.0:1616513095.516162:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580064 00000800:00000200:5.0:1616513095.516163:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000400:00000200:1.0:1616513095.516166:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00010000:00000200:8.0:1616513095.516168:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1681364861580064/t0(0) o400->023d13a7-5961-d0c8-eb71-c4c0fa9348a4@10.242.46.232@o2ib1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:5.0:1616513095.516168:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513095.516172:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff380 00000400:00000200:5.0:1616513095.516174:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000100:00000200:29.0:1616513095.516175:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580128 00000800:00000200:13.0:1616513095.516176:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000100:00000200:8.0:1616513095.516178:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681364861580064, offset 192 00000400:00000200:5.0:1616513095.516180:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff3a0 00000400:00000200:1.0:1616513095.516182:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 40480 00000400:00000200:8.0:1616513095.516183:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:1.0:1616513095.516186:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:13.0:1616513095.516187:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:5.0:1616513095.516188:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 40704 00010000:00000200:29.0:1616513095.516189:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d853e8cf500 x1681364861580128/t0(0) o400->boslfs02-MDT0001-lwp-OST001b_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:8.0:1616513095.516189:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:9.0:1616513095.516190:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)++ 00000400:00000200:5.0:1616513095.516192:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513095.516192:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:13.0:1616513095.516196:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:9.0:1616513095.516197:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000100:00000200:5.0:1616513095.516200:0:6467:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513095.516202:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.516207:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:1.0:1616513095.516208:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (26)++ 00000100:00000200:29.0:1616513095.516209:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580128, offset 192 00000800:00000200:5.0:1616513095.516209:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (27)++ 00000800:00000200:1.0:1616513095.516210:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (27)-- 00000400:00000200:13.0:1616513095.516211:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff400 00000800:00000200:5.0:1616513095.516211:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:1.0:1616513095.516211:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:8.0:1616513095.516212:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513095.516213:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:9.0:1616513095.516214:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:8.0:1616513095.516214:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:1.0:1616513095.516215:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000400:00000200:29.0:1616513095.516216:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:8.0:1616513095.516216:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:8.0:1616513095.516220:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (27)-- 00000400:00000200:13.0:1616513095.516221:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 40928 00000400:00000200:29.0:1616513095.516223:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000100:00000200:8.0:1616513095.516225:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:13.0:1616513095.516226:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:13.0:1616513095.516228:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:1.0:1616513095.516235:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (26)-- 00000400:00000200:29.0:1616513095.516237:0:170085:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:1.0:1616513095.516237:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000800:00000200:1.0:1616513095.516239:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000800:00000200:1.0:1616513095.516243:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:29.0:1616513095.516249:0:170085:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:9.0:1616513095.516249:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff420 00000400:00000200:1.0:1616513095.516251:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:13.0:1616513095.516252:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)++ 00000100:00000200:21.0:1616513095.516255:0:170083:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580256 00000800:00000200:13.0:1616513095.516255:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (26)-- 00000400:00000200:9.0:1616513095.516255:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 41152 00000800:00000200:29.0:1616513095.516256:0:170085:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000100:00000200:25.0:1616513095.516257:0:170518:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580160 00000400:00000200:1.0:1616513095.516257:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff480 00000800:00000200:13.0:1616513095.516258:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000100:00000200:5.0:1616513095.516258:0:170084:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580192 00000800:00000200:29.0:1616513095.516259:0:170085:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (24)++ 00000800:00000200:13.0:1616513095.516260:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000400:00000200:9.0:1616513095.516260:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:29.0:1616513095.516261:0:170085:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (26)++ 00000400:00000200:1.0:1616513095.516262:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 41376 00000100:00000200:17.0:1616513095.516264:0:175777:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580288 00000800:00000200:13.0:1616513095.516264:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513095.516265:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:29.0:1616513095.516266:0:170085:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (27)-- 00000100:00000200:9.0:1616513095.516266:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:5.0:1616513095.516269:0:170084:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843f7ba880 x1681364861580192/t0(0) o400->boslfs02-MDT0001-lwp-OST0017_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:25.0:1616513095.516270:0:170518:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d853e8cbf00 x1681364861580160/t0(0) o400->boslfs02-MDT0001-lwp-OST0023_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:21.0:1616513095.516270:0:170083:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843f7bf980 x1681364861580256/t0(0) o400->boslfs02-MDT0001-lwp-OST002b_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:13.0:1616513095.516271:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00010000:00000200:17.0:1616513095.516276:0:175777:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1394c80 x1681364861580288/t0(0) o400->boslfs02-MDT0001-lwp-OST002f_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:13.0:1616513095.516276:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff340 00000100:00000200:1.0:1616513095.516276:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:9.0:1616513095.516277:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (26)++ 00000800:00000200:9.0:1616513095.516281:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:1.0:1616513095.516281:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (27)++ 00000400:00000200:13.0:1616513095.516283:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 41600 00000800:00000200:9.0:1616513095.516283:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:1.0:1616513095.516283:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:1.0:1616513095.516285:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000800:00000200:9.0:1616513095.516286:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000400:00000200:13.0:1616513095.516287:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:5.0:1616513095.516287:0:170084:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580192, offset 192 00000100:00000200:21.0:1616513095.516288:0:170083:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580256, offset 192 00000100:00000200:25.0:1616513095.516289:0:170518:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580160, offset 192 00000100:00000200:13.0:1616513095.516289:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:29.0:1616513095.516290:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580320 00010000:00000200:29.0:1616513095.516295:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4558ad2880 x1681364861580320/t0(0) o400->boslfs02-MDT0001-lwp-OST0027_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:17.0:1616513095.516295:0:175777:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580288, offset 192 00000800:00000200:13.0:1616513095.516295:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:13.0:1616513095.516297:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (26)-- 00000400:00000200:5.0:1616513095.516297:0:170084:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513095.516301:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:21.0:1616513095.516303:0:170083:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513095.516303:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000800:00000200:9.0:1616513095.516303:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:5.0:1616513095.516304:0:170084:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:13.0:1616513095.516307:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:25.0:1616513095.516308:0:170518:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:17.0:1616513095.516314:0:175777:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000100:00000200:29.0:1616513095.516316:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580320, offset 192 00000400:00000200:5.0:1616513095.516317:0:170084:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000100:00000200:1.0:1616513095.516318:0:45715:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580416 00000400:00000200:29.0:1616513095.516322:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.516323:0:170084:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:21.0:1616513095.516326:0:170083:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:9.0:1616513095.516326:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00010000:00000200:1.0:1616513095.516328:0:45715:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d853e8c8480 x1681364861580416/t0(0) o400->boslfs02-MDT0001-lwp-OST0003_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:21.0:1616513095.516340:0:170083:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000100:00000200:1.0:1616513095.516342:0:45715:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580416, offset 192 00000800:00000200:21.0:1616513095.516346:0:170083:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:13.0:1616513095.516348:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513095.516348:0:45715:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:25.0:1616513095.516349:0:170518:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:25.0:1616513095.516362:0:170518:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:25.0:1616513095.516369:0:170518:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:17.0:1616513095.516371:0:175777:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:17.0:1616513095.516384:0:175777:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:17.0:1616513095.516391:0:175777:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:29.0:1616513095.516392:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:29.0:1616513095.516404:0:170085:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513095.516410:0:170085:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.516412:0:170084:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513095.516413:0:170083:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:1.0:1616513095.516414:0:45715:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:21.0:1616513095.516415:0:170083:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:21.0:1616513095.516418:0:170083:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (26)++ 00000800:00000200:5.0:1616513095.516421:0:170084:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (27)++ 00000800:00000200:21.0:1616513095.516422:0:170083:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (28)-- 00000800:00000200:5.0:1616513095.516422:0:170084:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (28)++ 00000800:00000200:5.0:1616513095.516426:0:170084:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (28)-- 00000400:00000200:1.0:1616513095.516427:0:45715:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000100:00000200:21.0:1616513095.516432:0:170083:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580096 00000800:00000200:1.0:1616513095.516433:0:45715:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:25.0:1616513095.516435:0:170518:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:25.0:1616513095.516438:0:170518:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (27)++ 00000800:00000200:29.0:1616513095.516455:0:170085:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:13.0:1616513095.516455:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff4a0 00000800:00000200:25.0:1616513095.516456:0:170518:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (28)++ 00000400:00000200:9.0:1616513095.516456:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff460 00000800:00000200:29.0:1616513095.516458:0:170085:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:1.0:1616513095.516459:0:45715:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513095.516460:0:170085:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:25.0:1616513095.516460:0:170518:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:5.0:1616513095.516461:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:1.0:1616513095.516461:0:45715:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00010000:00000200:21.0:1616513095.516462:0:170083:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1396c00 x1681364861580096/t0(0) o400->boslfs02-MDT0001-lwp-OST003f_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:9.0:1616513095.516462:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 41824 00000800:00000200:29.0:1616513095.516463:0:170085:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:1.0:1616513095.516464:0:45715:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (32)++ 00000400:00000200:9.0:1616513095.516465:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:13.0:1616513095.516468:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 42048 00000800:00000200:1.0:1616513095.516468:0:45715:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:5.0:1616513095.516469:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:13.0:1616513095.516471:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:9.0:1616513095.516471:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:5.0:1616513095.516471:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:5.0:1616513095.516476:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:13.0:1616513095.516477:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:9.0:1616513095.516477:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:9.0:1616513095.516480:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:9.0:1616513095.516481:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000400:00000200:5.0:1616513095.516481:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2198 00000100:00000200:21.0:1616513095.516483:0:170083:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580096, offset 192 00000800:00000200:9.0:1616513095.516483:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:13.0:1616513095.516484:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:5.0:1616513095.516485:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:13.0:1616513095.516486:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:9.0:1616513095.516487:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.516487:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:21.0:1616513095.516489:0:170083:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513095.516489:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000100:00000200:29.0:1616513095.516490:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580384 00000800:00000200:13.0:1616513095.516491:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:5.0:1616513095.516491:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:21.0:1616513095.516494:0:170083:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:13.0:1616513095.516494:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (30)-- 00000400:00000200:9.0:1616513095.516494:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000100:00000200:1.0:1616513095.516494:0:45715:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580448 00010000:00000200:29.0:1616513095.516496:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4558ad3180 x1681364861580384/t0(0) o400->boslfs02-MDT0001-lwp-OST0007_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:13.0:1616513095.516496:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:1.0:1616513095.516500:0:45715:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1392400 x1681364861580448/t0(0) o400->boslfs02-MDT0001-lwp-OST001f_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:13.0:1616513095.516501:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:13.0:1616513095.516504:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b3870 00000100:00000200:29.0:1616513095.516507:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580384, offset 192 00000400:00000200:21.0:1616513095.516507:0:170083:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000400:00000200:29.0:1616513095.516512:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:21.0:1616513095.516513:0:170083:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513095.516513:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)-- 00000100:00000200:1.0:1616513095.516514:0:45715:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580448, offset 192 00000800:00000200:13.0:1616513095.516515:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (28)++ 00000400:00000200:9.0:1616513095.516516:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff4e0 00000400:00000200:5.0:1616513095.516516:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:29.0:1616513095.516518:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:13.0:1616513095.516519:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513095.516519:0:45715:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:9.0:1616513095.516523:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 42272 00000400:00000200:9.0:1616513095.516526:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:9.0:1616513095.516528:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:29.0:1616513095.516530:0:170085:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513095.516536:0:170085:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:21.0:1616513095.516536:0:170083:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513095.516538:0:170083:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:1.0:1616513095.516538:0:45715:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:21.0:1616513095.516542:0:170083:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:9.0:1616513095.516543:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:21.0:1616513095.516545:0:170083:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:5.0:1616513095.516545:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff500 00000800:00000200:29.0:1616513095.516546:0:170085:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:9.0:1616513095.516546:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:13.0:1616513095.516547:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:9.0:1616513095.516547:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:29.0:1616513095.516548:0:170085:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:9.0:1616513095.516549:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:29.0:1616513095.516550:0:170085:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (30)++ 00000400:00000200:5.0:1616513095.516550:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 42496 00000100:00000200:1.0:1616513095.516551:0:45715:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580512 00000400:00000200:13.0:1616513095.516552:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff3c0 00000800:00000200:29.0:1616513095.516553:0:170085:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:9.0:1616513095.516553:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:5.0:1616513095.516554:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:9.0:1616513095.516555:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:13.0:1616513095.516557:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 42720 00010000:00000200:1.0:1616513095.516557:0:45715:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4558ad6780 x1681364861580512/t0(0) o400->boslfs02-MDT0001-lwp-OST000b_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:9.0:1616513095.516560:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:13.0:1616513095.516561:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:5.0:1616513095.516561:0:6467:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:13.0:1616513095.516566:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:5.0:1616513095.516567:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:5.0:1616513095.516569:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:5.0:1616513095.516570:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:13.0:1616513095.516572:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:9.0:1616513095.516572:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51298 00000800:00000200:5.0:1616513095.516572:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:13.0:1616513095.516574:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:5.0:1616513095.516575:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (31)-- 00000100:00000200:1.0:1616513095.516575:0:45715:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580512, offset 192 00000800:00000200:13.0:1616513095.516576:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000400:00000200:5.0:1616513095.516576:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:13.0:1616513095.516578:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (28)++ 00000800:00000200:9.0:1616513095.516578:0:6465:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:1.0:1616513095.516579:0:45715:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513095.516581:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:5.0:1616513095.516581:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:29.0:1616513095.516582:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580544 00000800:00000200:9.0:1616513095.516583:0:6465:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:5.0:1616513095.516584:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cc55e770 00000400:00000200:1.0:1616513095.516584:0:45715:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:9.0:1616513095.516586:0:6465:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (29)++ 00010000:00000200:29.0:1616513095.516587:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843f7bba80 x1681364861580544/t0(0) o400->boslfs02-MDT0001-lwp-OST003b_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:21.0:1616513095.516588:0:170083:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580224 00000800:00000200:9.0:1616513095.516588:0:6465:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:5.0:1616513095.516591:0:6467:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:13.0:1616513095.516592:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:9.0:1616513095.516592:0:6465:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (31)-- 00010000:00000200:21.0:1616513095.516593:0:170083:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1395100 x1681364861580224/t0(0) o400->boslfs02-MDT0001-lwp-OST0013_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:9.0:1616513095.516594:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:9.0:1616513095.516596:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:5.0:1616513095.516596:0:6467:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000100:00000200:29.0:1616513095.516597:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580544, offset 192 00000400:00000200:13.0:1616513095.516597:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff440 00000800:00000200:5.0:1616513095.516598:0:6467:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:5.0:1616513095.516600:0:6467:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (31)++ 00000400:00000200:13.0:1616513095.516602:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 42944 00000800:00000200:9.0:1616513095.516602:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.516603:0:6467:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:5.0:1616513095.516604:0:6467:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:13.0:1616513095.516605:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:1.0:1616513095.516605:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (32)++ 00000400:00000200:29.0:1616513095.516607:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.516607:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (32)-- 00000100:00000200:21.0:1616513095.516609:0:170083:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580224, offset 192 00000400:00000200:9.0:1616513095.516609:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:5.0:1616513095.516609:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:1.0:1616513095.516609:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000100:00000200:13.0:1616513095.516610:0:6466:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:29.0:1616513095.516613:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:5.0:1616513095.516613:0:6467:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:5.0:1616513095.516615:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:5.0:1616513095.516616:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:21.0:1616513095.516617:0:170083:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513095.516618:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)++ 00000400:00000200:9.0:1616513095.516620:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff520 00000400:00000200:5.0:1616513095.516621:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:1.0:1616513095.516621:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:21.0:1616513095.516622:0:170083:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:13.0:1616513095.516622:0:6466:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:13.0:1616513095.516623:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:13.0:1616513095.516625:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)-- 00000400:00000200:9.0:1616513095.516625:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 43168 00000100:00000200:25.0:1616513095.516626:0:109348:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580352 00000800:00000200:13.0:1616513095.516627:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)++ 00000400:00000200:9.0:1616513095.516628:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:5.0:1616513095.516629:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492672e58 00000400:00000200:1.0:1616513095.516629:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 224 into portal 12 MB=0x5f931434ff4c0 00000800:00000200:17.0:1616513095.516630:0:175777:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513095.516631:0:6466:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000100:00000200:9.0:1616513095.516631:0:6465:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:25.0:1616513095.516632:0:109348:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5de1390000 x1681364861580352/t0(0) o400->boslfs02-MDT0001-lwp-OST0037_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:13.0:1616513095.516632:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:17.0:1616513095.516633:0:175777:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (31)++ 00000400:00000200:13.0:1616513095.516634:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513095.516634:0:6467:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:17.0:1616513095.516635:0:175777:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:9.0:1616513095.516637:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)++ 00000800:00000200:5.0:1616513095.516638:0:6467:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:17.0:1616513095.516639:0:175777:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000400:00000200:13.0:1616513095.516639:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:1.0:1616513095.516639:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.232@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 43392 00000800:00000200:17.0:1616513095.516640:0:175777:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (33)-- 00000800:00000200:5.0:1616513095.516640:0:6467:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (33)++ 00000400:00000200:13.0:1616513095.516643:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2b28 00000800:00000200:9.0:1616513095.516643:0:6465:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000400:00000200:1.0:1616513095.516643:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:9.0:1616513095.516644:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (33)-- 00000800:00000200:5.0:1616513095.516644:0:6467:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (33)++ 00000100:00000200:25.0:1616513095.516645:0:109348:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580352, offset 192 00000800:00000200:9.0:1616513095.516646:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (33)-- 00000100:00000200:1.0:1616513095.516646:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:21.0:1616513095.516647:0:170083:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580576 00000800:00000200:9.0:1616513095.516648:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (32)++ 00000800:00000200:5.0:1616513095.516648:0:6467:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:13.0:1616513095.516649:0:6466:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.516649:0:6467:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)-- 00000800:00000200:9.0:1616513095.516651:0:6465:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:5.0:1616513095.516651:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (32)-- 00010000:00000200:21.0:1616513095.516652:0:170083:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4558ad5a00 x1681364861580576/t0(0) o400->boslfs02-MDT0001-lwp-OST000f_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:9.0:1616513095.516652:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:1.0:1616513095.516652:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)++ 00000800:00000200:5.0:1616513095.516653:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)++ 00000400:00000200:25.0:1616513095.516654:0:109348:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:9.0:1616513095.516654:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:13.0:1616513095.516655:0:6466:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:5.0:1616513095.516656:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.232@o2ib1 00000800:00000200:1.0:1616513095.516656:0:6468:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:13.0:1616513095.516657:0:6466:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (32)++ 00000800:00000200:1.0:1616513095.516657:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:25.0:1616513095.516659:0:109348:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:9.0:1616513095.516659:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:1.0:1616513095.516659:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (32)-- 00000100:00000200:29.0:1616513095.516660:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364861580480 00000800:00000200:13.0:1616513095.516661:0:6466:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:5.0:1616513095.516661:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:1.0:1616513095.516661:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)++ 00000400:00000200:9.0:1616513095.516662:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85430 00000800:00000200:5.0:1616513095.516662:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (33)-- 00000100:00000200:21.0:1616513095.516664:0:170083:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580576, offset 192 00000800:00000200:5.0:1616513095.516664:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (33)-- 00000800:00000200:13.0:1616513095.516665:0:6466:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (33)-- 00000800:00000200:1.0:1616513095.516665:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (33)-- 00010000:00000200:29.0:1616513095.516666:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d853e8ce300 x1681364861580480/t0(0) o400->boslfs02-MDT0001-lwp-OST0033_UUID@10.242.46.232@o2ib1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:9.0:1616513095.516666:0:6465:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.516666:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (32)++ 00000800:00000200:13.0:1616513095.516667:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)-- 00000400:00000200:1.0:1616513095.516667:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513095.516668:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (31)-- 00000400:00000200:21.0:1616513095.516669:0:170083:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:13.0:1616513095.516669:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:5.0:1616513095.516669:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:13.0:1616513095.516671:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:9.0:1616513095.516671:0:6465:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:1.0:1616513095.516672:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:13.0:1616513095.516673:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:9.0:1616513095.516673:0:6465:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (29)++ 00000400:00000200:21.0:1616513095.516674:0:170083:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:5.0:1616513095.516674:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:9.0:1616513095.516675:0:6465:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (30)++ 00000400:00000200:1.0:1616513095.516675:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50110 00000100:00000200:29.0:1616513095.516677:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364861580480, offset 192 00000400:00000200:13.0:1616513095.516678:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:9.0:1616513095.516678:0:6465:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000400:00000200:5.0:1616513095.516678:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2d48 00000800:00000200:9.0:1616513095.516679:0:6465:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:1.0:1616513095.516680:0:6468:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:5.0:1616513095.516681:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000400:00000200:29.0:1616513095.516682:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:9.0:1616513095.516682:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)-- 00000400:00000200:13.0:1616513095.516684:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84dd0 00000800:00000200:5.0:1616513095.516684:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (28)++ 00000800:00000200:1.0:1616513095.516685:0:6468:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:29.0:1616513095.516687:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:1.0:1616513095.516687:0:6468:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:1.0:1616513095.516690:0:6468:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (30)++ 00000800:00000200:5.0:1616513095.516691:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.232@o2ib1 00000800:00000200:1.0:1616513095.516693:0:6468:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.232@o2ib1: no credits 00000800:00000200:1.0:1616513095.516694:0:6468:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:17.0:1616513095.516695:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:17.0:1616513095.516697:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (31)-- 00000800:00000200:5.0:1616513095.516697:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (31)++ 00000800:00000200:5.0:1616513095.516698:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (32)-- 00000400:00000200:29.0:1616513095.516699:0:170085:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000400:00000200:17.0:1616513095.516699:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513095.516699:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:5.0:1616513095.516701:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (29)++ 00000800:00000200:13.0:1616513095.516703:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:5.0:1616513095.516703:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (30)-- 00000800:00000200:29.0:1616513095.516704:0:170085:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000400:00000200:17.0:1616513095.516705:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:5.0:1616513095.516705:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513095.516705:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (28)-- 00000800:00000200:13.0:1616513095.516707:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (27)++ 00000800:00000200:29.0:1616513095.516709:0:170085:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000400:00000200:17.0:1616513095.516709:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe85b18 00000400:00000200:5.0:1616513095.516709:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:29.0:1616513095.516711:0:170085:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (28)++ 00000800:00000200:13.0:1616513095.516711:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (28)-- 00000800:00000200:17.0:1616513095.516712:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (28)-- 00000400:00000200:5.0:1616513095.516712:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8326b3e660 00000800:00000200:29.0:1616513095.516713:0:170085:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (28)++ 00000400:00000200:13.0:1616513095.516713:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513095.516714:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:29.0:1616513095.516715:0:170085:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (28)-- 00000800:00000200:17.0:1616513095.516716:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (27)++ 00000400:00000200:13.0:1616513095.516717:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:13.0:1616513095.516720:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2770 00000800:00000200:17.0:1616513095.516722:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:13.0:1616513095.516722:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (27)-- 00000800:00000200:21.0:1616513095.516724:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)++ 00000400:00000200:17.0:1616513095.516724:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:17.0:1616513095.516729:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:17.0:1616513095.516731:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50aa0 00000800:00000200:21.0:1616513095.516741:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (26)-- 00000800:00000200:17.0:1616513095.516741:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (26)-- 00000400:00000200:21.0:1616513095.516743:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:9.0:1616513095.516743:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (24)++ 00000800:00000200:9.0:1616513095.516748:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:21.0:1616513095.516749:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:13.0:1616513095.516749:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)++ 00000400:00000200:9.0:1616513095.516750:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:21.0:1616513095.516752:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84926735c8 00000800:00000200:17.0:1616513095.516752:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)++ 00000800:00000200:13.0:1616513095.516753:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.232@o2ib1 00000800:00000200:17.0:1616513095.516754:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (26)-- 00000800:00000200:21.0:1616513095.516755:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (26)-- 00000800:00000200:13.0:1616513095.516755:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (26)++ 00000400:00000200:9.0:1616513095.516755:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:17.0:1616513095.516756:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:13.0:1616513095.516757:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:9.0:1616513095.516757:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51b18 00000800:00000200:21.0:1616513095.516758:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (25)-- 00000400:00000200:17.0:1616513095.516760:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000800:00000200:13.0:1616513095.516760:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:9.0:1616513095.516761:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:17.0:1616513095.516763:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b3760 00000800:00000200:17.0:1616513095.516765:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:7.2:1616513095.525945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513095.525996:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.526004:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513095.526011:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.58@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.526020:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.58@tcp1 of length 224 into portal 26 MB=0x5ffe2b9eefa00 00000400:00000200:14.0:1616513095.526026:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.58@tcp1 of length 224/224 into md 0xf97cf439 [1] + 0 00000400:00000200:14.0:1616513095.526030:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c3b8 00000400:00000200:14.0:1616513095.526032:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.526034:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513095.526038:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c3b8 00000800:00000200:14.0:1616513095.526040:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.526042:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.526044:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.526046:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513095.526051:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724130691584 00010000:00000200:8.0:1616513095.526057:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688724130691584/t0(0) o400->cfc96df0-7739-e3a7-d5f2-d6864a3f6e69@10.31.140.58@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.526067:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688724130691584, offset 192 00000800:00000200:7.2:1616513095.526071:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513095.526072:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.58@tcp1 00000400:00000200:8.0:1616513095.526077:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.58@tcp1 routed destination 00000400:00000200:8.0:1616513095.526089:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.58@tcp1(10.31.140.58@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.526094:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513095.526098:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.526100:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513095.526103:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.526106:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513095.526110:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513095.526125:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.526134:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513095.526143:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.58@tcp1 : PUT - for me 00000800:00000200:11.2:1616513095.526150:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513095.526151:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.58@tcp1 of length 224 into portal 12 MB=0x5ffe2b9eefa80 00000400:00000200:26.0:1616513095.526158:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.58@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 22512 00000400:00000200:26.0:1616513095.526163:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513095.526166:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513095.526174:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.526177:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513095.526177:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513095.526182:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513095.526190:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724130691712 00010000:00000200:10.0:1616513095.526200:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695a00 x1688724130691712/t0(0) o400->bb797345-a586-1a6c-0fe0-f01105acdb73@10.31.140.58@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513095.526207:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:10.0:1616513095.526212:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688724130691712, offset 192 00000800:00000200:20.0:1616513095.526214:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513095.526217:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513095.526217:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.58@tcp1 00000400:00000200:10.0:1616513095.526223:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.58@tcp1 routed destination 00000400:00000200:20.0:1616513095.526224:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513095.526230:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:20.0:1616513095.526233:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513095.526234:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.58@tcp1(10.31.140.58@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513095.526236:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513095.526240:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513095.526244:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.526246:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513095.526248:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513095.526252:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.526257:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.526264:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.526268:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.526270:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513095.526273:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513095.526274:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513095.526277:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83d38 00000800:00000200:14.0:1616513095.526279:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.526967:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513095.526978:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.526986:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513095.526987:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513095.526994:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.80@tcp1 : PUT - for me 00000400:00000200:2.0:1616513095.527003:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.80@tcp1 of length 224 into portal 26 MB=0x5ffdf21fd1d40 00000400:00000200:2.0:1616513095.527010:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.80@tcp1 of length 224/224 into md 0xf97cf449 [1] + 0 00000400:00000200:2.0:1616513095.527014:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922da90 00000400:00000200:2.0:1616513095.527017:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513095.527020:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513095.527024:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922da90 00000800:00000200:2.0:1616513095.527027:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.527030:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.527031:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513095.527035:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708696579392 00010000:00000200:8.0:1616513095.527040:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688708696579392/t0(0) o400->40fd6c7e-941b-5c94-869e-6a7816911eab@10.31.141.80@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513095.527059:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688708696579392, offset 192 00000400:00000200:8.0:1616513095.527063:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.80@tcp1 00000400:00000200:8.0:1616513095.527068:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.80@tcp1 routed destination 00000400:00000200:8.0:1616513095.527079:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.80@tcp1(10.31.141.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513095.527084:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513095.527088:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.527090:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513095.527091:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.527094:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513095.527098:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513095.527100:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.527151:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.527159:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513095.527161:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513095.527164:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513095.527167:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513095.527173:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:16.0:1616513095.527175:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513095.527176:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513095.527178:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.527179:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.527180:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.527182:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513095.527196:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513095.527204:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.527210:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513095.527213:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.527218:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.80@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.527227:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.80@tcp1 of length 224 into portal 12 MB=0x5ffdf21fd1f00 00000400:00000200:16.0:1616513095.527233:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.80@tcp1 of length 224/224 into md 0x865be291 [1] + 56320 00000400:00000200:16.0:1616513095.527239:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.527241:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513095.527248:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.527250:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.527252:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513095.527273:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708696579840 00010000:00000200:28.0:1616513095.527285:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6723180 x1688708696579840/t0(0) o400->2a5553da-e158-66cf-b6ee-acfafdc49263@10.31.141.80@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513095.527300:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688708696579840, offset 192 00000400:00000200:28.0:1616513095.527306:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.80@tcp1 00000400:00000200:28.0:1616513095.527313:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.80@tcp1 routed destination 00000400:00000200:28.0:1616513095.527328:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.80@tcp1(10.31.141.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.527337:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513095.527342:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.527345:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513095.527347:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513095.527352:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.527357:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.527368:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.527373:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513095.527376:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513095.527376:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513095.527382:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513095.527386:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcde48 00000800:00000200:18.0:1616513095.527390:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.544156:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513095.544212:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.544222:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.544230:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.191@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.544239:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.191@tcp1 of length 224 into portal 26 MB=0x5ffddbbda6e40 00000400:00000200:20.0:1616513095.544248:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.191@tcp1 of length 224/224 into md 0xf97cf479 [1] + 0 00000400:00000200:20.0:1616513095.544253:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922dcb0 00000400:00000200:20.0:1616513095.544256:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.544258:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513095.544263:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922dcb0 00000800:00000200:20.0:1616513095.544266:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.544266:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.544269:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.544274:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.544276:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.544277:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:8.0:1616513095.544278:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702688063040 00010000:00000200:8.0:1616513095.544285:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688702688063040/t0(0) o400->e18f364f-834f-a17f-17fe-d934da083ac3@10.31.132.191@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:16.0:1616513095.544286:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.191@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.544295:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.191@tcp1 of length 224 into portal 12 MB=0x5ffddbbda6ec0 00000100:00000200:8.0:1616513095.544296:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702688063040, offset 192 00000400:00000200:16.0:1616513095.544301:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.191@tcp1 of length 224/224 into md 0x865be291 [1] + 56544 00000400:00000200:16.0:1616513095.544311:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513095.544313:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.191@tcp1 00000100:00000200:16.0:1616513095.544315:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513095.544318:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.191@tcp1 routed destination 00000400:00000200:8.0:1616513095.544330:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.191@tcp1(10.31.132.191@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513095.544335:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.544336:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513095.544338:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.544340:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513095.544340:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513095.544342:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513095.544344:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513095.544348:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513095.544353:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513095.544354:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.544364:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.544368:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.544370:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513095.544372:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.544376:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.544380:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:16.0:1616513095.544384:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513095.544387:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702688063168 00010000:00000200:28.0:1616513095.544399:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6721b00 x1688702688063168/t0(0) o400->2cc49ba6-d7a6-eb04-eb38-2af5562337c3@10.31.132.191@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513095.544414:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702688063168, offset 192 00000400:00000200:28.0:1616513095.544421:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.191@tcp1 00000400:00000200:28.0:1616513095.544428:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.191@tcp1 routed destination 00000400:00000200:28.0:1616513095.544456:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.191@tcp1(10.31.132.191@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.544463:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513095.544469:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.544472:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513095.544474:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513095.544479:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.544491:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.544548:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.544554:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513095.544557:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513095.544565:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513095.544569:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcded0 00000800:00000200:26.0:1616513095.544573:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.544576:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.548234:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513095.548250:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.548258:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513095.548265:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.94@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.548273:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.94@tcp1 of length 224 into portal 12 MB=0x604a35e2d70c0 00000400:00000200:14.0:1616513095.548278:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.94@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 22736 00000400:00000200:14.0:1616513095.548283:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.548286:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513095.548292:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.548294:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.548296:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.548298:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.548303:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:10.0:1616513095.548308:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949566480576 00010000:00000200:10.0:1616513095.548317:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfda00 x1693949566480576/t0(0) o400->29ac1d90-2ab6-9c3e-3797-d158a42ffdb4@10.31.143.94@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513095.548329:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949566480576, offset 192 00000400:00000200:10.0:1616513095.548334:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.94@tcp1 00000400:00000200:10.0:1616513095.548341:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.94@tcp1 routed destination 00000400:00000200:10.0:1616513095.548353:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.94@tcp1(10.31.143.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513095.548357:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513095.548358:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513095.548362:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.548364:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513095.548366:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513095.548366:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:10.0:1616513095.548370:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:0.0:1616513095.548377:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.94@tcp1 : PUT - for me 00000400:00000200:0.0:1616513095.548386:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.94@tcp1 of length 224 into portal 26 MB=0x604a35e2d6bc0 00000400:00000200:0.0:1616513095.548394:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.94@tcp1 of length 224/224 into md 0xf97cf489 [1] + 0 00000400:00000200:0.0:1616513095.548399:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922cc38 00000400:00000200:0.0:1616513095.548402:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.548405:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:16.0:1616513095.548409:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:0.0:1616513095.548411:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cc38 00000800:00000200:0.0:1616513095.548414:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:16.0:1616513095.548416:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:0.0:1616513095.548417:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:0.0:1616513095.548418:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.548419:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513095.548421:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.548425:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.548429:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82bb0 00000800:00000200:16.0:1616513095.548432:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.548475:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949566479296 00010000:00000200:12.0:1616513095.548485:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1693949566479296/t0(0) o400->7e837180-ffbf-96ee-f1bb-35338d78598c@10.31.143.94@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.548499:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949566479296, offset 192 00000400:00000200:12.0:1616513095.548505:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.94@tcp1 00000400:00000200:12.0:1616513095.548513:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.94@tcp1 routed destination 00000400:00000200:12.0:1616513095.548528:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.94@tcp1(10.31.143.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.548534:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513095.548540:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.548543:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513095.548545:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513095.548551:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.548557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513095.548558:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513095.548612:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.548619:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513095.548622:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513095.548628:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513095.548634:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:26.0:1616513095.548638:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.548641:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:25.0:1616513095.578717:0:6494:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929328, portal 4 00000100:00000200:25.0:1616513095.578723:0:6494:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929328, offset 0 00000400:00000200:25.0:1616513095.578730:0:6494:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:25.0:1616513095.578736:0:6494:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:25.0:1616513095.578749:0:6494:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:25.0:1616513095.578756:0:6494:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:25.0:1616513095.578762:0:6494:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:25.0:1616513095.578764:0:6494:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:25.0:1616513095.578769:0:6494:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:25.0:1616513095.578773:0:6494:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513095.578837:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.578856:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513095.578861:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513095.578864:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513095.578871:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513095.578878:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cab103f00 x1693067725929328/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513095.578890:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8492672908 00000800:00000200:1.0:1616513095.578894:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513095.578897:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513095.578903:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.232@o2ib1 00000800:00000200:1.0:1616513095.578905:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513095.578907:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:13.0:1616513095.578909:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)++ 00000800:00000200:1.0:1616513095.578916:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:1.0:1616513095.578919:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:13.0:1616513095.578920:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.232@o2ib1 00000400:00000200:13.0:1616513095.578929:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:13.0:1616513095.578936:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627b70 00000400:00000200:13.0:1616513095.578943:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700235 [1] + 192 00000400:00000200:13.0:1616513095.578950:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:13.0:1616513095.578955:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cab103f00 x1693067725929328/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:13.0:1616513095.578970:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:13.0:1616513095.578973:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:13.0:1616513095.578975:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:25.0:1616513095.578977:0:6494:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cab103f00 x1693067725929328/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:25.0:1616513095.578990:0:6494:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cab103f00 x1693067725929328/t0(0) o13->boslfs02-OST001f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:25.0:1616513095.579000:0:6494:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84926727f8 00000800:00000200:7.2:1616513095.592076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513095.592131:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.592140:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513095.592148:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.208@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.592157:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.208@tcp1 of length 224 into portal 26 MB=0x5ffd4c5a249c0 00000400:00000200:14.0:1616513095.592163:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.208@tcp1 of length 224/224 into md 0xf97cf4a9 [1] + 0 00000400:00000200:14.0:1616513095.592167:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d650 00000400:00000200:14.0:1616513095.592170:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.592172:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513095.592176:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d650 00000800:00000200:14.0:1616513095.592179:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.592181:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.592183:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.592185:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513095.592241:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664197450176 00010000:00000200:12.0:1616513095.592252:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688664197450176/t0(0) o400->9e357f4e-cddc-4cad-bf78-5a8c44e99b1a@10.31.128.208@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.592266:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664197450176, offset 192 00000400:00000200:12.0:1616513095.592273:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.208@tcp1 00000400:00000200:12.0:1616513095.592281:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.208@tcp1 routed destination 00000400:00000200:12.0:1616513095.592295:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.208@tcp1(10.31.128.208@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.592301:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513095.592307:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.592310:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.592312:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.592317:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513095.592324:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513095.592325:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.592379:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.592386:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.592389:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513095.592395:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.592400:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa08 00000800:00000200:16.0:1616513095.592404:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.592406:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.592730:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513095.592785:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.592795:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513095.592804:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.208@tcp1 : PUT - for me 00000400:00000200:26.0:1616513095.592812:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.208@tcp1 of length 224 into portal 12 MB=0x5ffd4c5a24b80 00000400:00000200:26.0:1616513095.592819:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.208@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 22960 00000400:00000200:26.0:1616513095.592824:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513095.592827:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513095.592834:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.592838:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513095.592840:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.592846:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513095.592899:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664197450624 00010000:00000200:10.0:1616513095.592912:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c694380 x1688664197450624/t0(0) o400->1261186f-5d35-c065-7293-aa750db90564@10.31.128.208@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513095.592926:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664197450624, offset 192 00000400:00000200:10.0:1616513095.592933:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.208@tcp1 00000400:00000200:10.0:1616513095.592940:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.208@tcp1 routed destination 00000400:00000200:10.0:1616513095.592955:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.208@tcp1(10.31.128.208@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.592962:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513095.592968:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.592970:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513095.592973:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513095.592978:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.592983:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.592992:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.592996:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.592998:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513095.593000:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513095.593004:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513095.593007:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82550 00000800:00000200:14.0:1616513095.593010:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.614957:0:51124:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513095.615013:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.615031:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513095.615041:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.209@tcp1 : PUT - for me 00000400:00000200:2.0:1616513095.615050:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.209@tcp1 of length 224 into portal 26 MB=0x5ffdb525c1a00 00000400:00000200:2.0:1616513095.615056:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.209@tcp1 of length 224/224 into md 0xf97cf4c9 [1] + 0 00000400:00000200:2.0:1616513095.615062:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c990 00000400:00000200:2.0:1616513095.615064:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513095.615067:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513095.615071:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c990 00000800:00000200:2.0:1616513095.615075:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.615077:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513095.615079:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.615082:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513095.615136:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692328241664 00010000:00000200:12.0:1616513095.615147:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688692328241664/t0(0) o400->02778714-4e55-2c08-d74c-a21c739c571b@10.31.140.209@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.615162:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692328241664, offset 192 00000400:00000200:12.0:1616513095.615168:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.209@tcp1 00000400:00000200:12.0:1616513095.615176:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.209@tcp1 routed destination 00000400:00000200:12.0:1616513095.615190:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.209@tcp1(10.31.140.209@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.615197:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513095.615203:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.615206:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.615208:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.615213:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513095.615220:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513095.615221:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.615276:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.615282:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.615285:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513095.615292:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.615298:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f3a8 00000800:00000200:16.0:1616513095.615301:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.615304:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513095.615731:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513095.615743:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.615749:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513095.615757:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.209@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.615765:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.209@tcp1 of length 224 into portal 12 MB=0x5ffdb525c1d40 00000400:00000200:16.0:1616513095.615774:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.209@tcp1 of length 224/224 into md 0x865be291 [1] + 56768 00000400:00000200:16.0:1616513095.615779:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.615781:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513095.615788:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.615791:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513095.615793:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.615795:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513095.615852:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692328242496 00010000:00000200:28.0:1616513095.615864:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6722d00 x1688692328242496/t0(0) o400->72616914-918b-b8e7-284a-0647a7f5f214@10.31.140.209@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513095.615877:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692328242496, offset 192 00000400:00000200:28.0:1616513095.615883:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.209@tcp1 00000400:00000200:28.0:1616513095.615889:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.209@tcp1 routed destination 00000400:00000200:28.0:1616513095.615904:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.209@tcp1(10.31.140.209@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.615910:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513095.615915:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.615918:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513095.615920:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513095.615925:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.615932:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.615949:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.615956:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513095.615959:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513095.615966:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513095.615970:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcddc0 00000800:00000200:18.0:1616513095.615974:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.615977:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.676820:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513095.676828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513095.676880:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.676890:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:16.0:1616513095.676896:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513095.676899:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.46@tcp1 : PUT - for me 00000800:00000200:16.0:1616513095.676906:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513095.676911:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.46@tcp1 of length 224 into portal 12 MB=0x6036789870980 00000400:00000200:16.0:1616513095.676915:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.46@tcp1 : PUT - for me 00000400:00000200:2.0:1616513095.676918:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.46@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 23184 00000400:00000200:2.0:1616513095.676924:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513095.676925:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.46@tcp1 of length 224 into portal 26 MB=0x6036789870900 00000100:00000200:2.0:1616513095.676927:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513095.676934:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.676935:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513095.676936:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.46@tcp1 of length 224/224 into md 0xf97cf519 [1] + 0 00000800:00000200:2.0:1616513095.676938:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513095.676940:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513095.676941:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c220 00000400:00000200:16.0:1616513095.676944:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.676947:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513095.676951:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c220 00000800:00000200:20.0:1616513095.676953:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.676955:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.676958:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.676960:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513095.677001:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692593084107136 00010000:00000200:10.0:1616513095.677014:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d1680 x1692593084107136/t0(0) o400->48a4bc60-13cf-7791-093b-e0c2eef0d832@10.31.131.46@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513095.677018:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692593084107008 00000100:00000200:10.0:1616513095.677028:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692593084107136, offset 192 00010000:00000200:12.0:1616513095.677031:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1692593084107008/t0(0) o400->ed3ca774-7f9c-1ce6-8693-d8e23d82e597@10.31.131.46@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:10.0:1616513095.677034:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.46@tcp1 00000400:00000200:10.0:1616513095.677042:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.46@tcp1 routed destination 00000100:00000200:12.0:1616513095.677056:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692593084107008, offset 192 00000400:00000200:10.0:1616513095.677056:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.46@tcp1(10.31.131.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513095.677063:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.46@tcp1 00000800:00000200:10.0:1616513095.677063:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513095.677068:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513095.677071:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.46@tcp1 routed destination 00000800:00000200:10.0:1616513095.677071:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513095.677075:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513095.677080:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.677085:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513095.677086:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.46@tcp1(10.31.131.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.677093:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513095.677099:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.677102:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513095.677104:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513095.677109:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.677115:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513095.677116:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513095.677141:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.677149:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513095.677152:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513095.677153:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513095.677158:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513095.677162:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e837e8 00000800:00000200:0.0:1616513095.677166:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513095.677172:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.677179:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513095.677182:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513095.677184:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513095.677189:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513095.677195:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e550 00000800:00000200:18.0:1616513095.677198:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.683581:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513095.683633:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513095.683635:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.683643:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.683653:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.194@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.683663:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.194@tcp1 of length 224 into portal 26 MB=0x5ffdb4b45e640 00000400:00000200:20.0:1616513095.683669:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.194@tcp1 of length 224/224 into md 0xf97cf539 [1] + 0 00000400:00000200:20.0:1616513095.683674:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c6e8 00000400:00000200:20.0:1616513095.683677:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.683679:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513095.683684:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c6e8 00000800:00000200:20.0:1616513095.683687:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513095.683688:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513095.683689:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.683691:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.683694:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513095.683697:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513095.683706:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.194@tcp1 : PUT - for me 00000400:00000200:26.0:1616513095.683716:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.194@tcp1 of length 224 into portal 12 MB=0x5ffdb4b45e840 00000400:00000200:26.0:1616513095.683722:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.194@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 23408 00000400:00000200:26.0:1616513095.683727:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513095.683730:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513095.683737:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.683739:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513095.683741:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.683744:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513095.683755:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692209346624 00000100:00000200:12.0:1616513095.683767:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692209346112 00010000:00000200:10.0:1616513095.683768:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691b00 x1688692209346624/t0(0) o400->ba6818bb-82f5-79f7-8787-c2712b6aed8d@10.31.140.194@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:12.0:1616513095.683778:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688692209346112/t0(0) o400->9df8eccf-9bcd-cf7d-ba02-ff59d2c79ab5@10.31.140.194@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513095.683781:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692209346624, offset 192 00000400:00000200:10.0:1616513095.683787:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.194@tcp1 00000100:00000200:12.0:1616513095.683792:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692209346112, offset 192 00000400:00000200:10.0:1616513095.683794:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.194@tcp1 routed destination 00000400:00000200:12.0:1616513095.683798:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.194@tcp1 00000400:00000200:10.0:1616513095.683808:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.194@tcp1(10.31.140.194@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.683814:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513095.683817:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.194@tcp1 routed destination 00000800:00000200:10.0:1616513095.683819:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.683822:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513095.683824:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513095.683828:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513095.683833:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.194@tcp1(10.31.140.194@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513095.683833:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.683839:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513095.683845:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513095.683845:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.683848:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513095.683850:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513095.683852:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.683855:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513095.683855:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.683860:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513095.683862:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:12.0:1616513095.683862:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513095.683866:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82dd0 00000800:00000200:16.0:1616513095.683870:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513095.683871:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513095.683873:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513095.683878:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.683881:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513095.683887:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513095.683893:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f100 00000800:00000200:14.0:1616513095.683896:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.683899:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.695599:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513095.695655:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.695664:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513095.695666:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513095.695673:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.52@tcp1 : PUT - for me 00000400:00000200:26.0:1616513095.695683:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.52@tcp1 of length 224 into portal 26 MB=0x601990ba0a800 00000400:00000200:26.0:1616513095.695689:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.52@tcp1 of length 224/224 into md 0xf97cf559 [1] + 0 00000400:00000200:26.0:1616513095.695694:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d7e8 00000400:00000200:26.0:1616513095.695697:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513095.695700:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513095.695704:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d7e8 00000800:00000200:26.0:1616513095.695707:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.695710:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.695711:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513095.695767:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690606696966144 00010000:00000200:12.0:1616513095.695778:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1690606696966144/t0(0) o400->4f7a6f50-0f78-1ce4-8e61-1841072f2564@10.31.141.52@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.695793:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690606696966144, offset 192 00000400:00000200:12.0:1616513095.695799:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.52@tcp1 00000400:00000200:12.0:1616513095.695806:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.52@tcp1 routed destination 00000400:00000200:12.0:1616513095.695823:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.52@tcp1(10.31.141.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.695830:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513095.695835:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.695838:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.695840:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.695845:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513095.695852:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513095.695854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.695908:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.695915:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513095.695918:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513095.695924:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513095.695930:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ec38 00000800:00000200:20.0:1616513095.695933:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.695936:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.696133:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513095.696188:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.696197:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513095.696201:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513095.696206:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.52@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.696216:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.52@tcp1 of length 224 into portal 12 MB=0x601990ba0b400 00000400:00000200:14.0:1616513095.696223:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.52@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 23632 00000400:00000200:14.0:1616513095.696228:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.696231:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513095.696238:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.696241:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.696243:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513095.696299:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690606696969216 00010000:00000200:10.0:1616513095.696311:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf9f80 x1690606696969216/t0(0) o400->8873f302-b511-c29c-767c-93b1da968ff9@10.31.141.52@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513095.696325:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690606696969216, offset 192 00000400:00000200:10.0:1616513095.696330:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.52@tcp1 00000400:00000200:10.0:1616513095.696337:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.52@tcp1 routed destination 00000400:00000200:10.0:1616513095.696351:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.52@tcp1(10.31.141.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.696357:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513095.696362:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.696365:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513095.696367:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513095.696371:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.696377:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.696386:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.696391:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513095.696393:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513095.696395:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513095.696399:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513095.696403:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e826e8 00000800:00000200:2.0:1616513095.696406:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.716848:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513095.716905:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.716914:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513095.716923:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.108@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.716934:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.108@tcp1 of length 224 into portal 26 MB=0x60195bdd7ca80 00000400:00000200:16.0:1616513095.716941:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.108@tcp1 of length 224/224 into md 0xf97cf589 [1] + 0 00000400:00000200:16.0:1616513095.716950:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922ccc0 00000400:00000200:16.0:1616513095.716953:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.716956:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1616513095.716959:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.716960:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922ccc0 00000800:00000200:16.0:1616513095.716963:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.716966:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.716968:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.717026:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690592507054720 00010000:00000200:12.0:1616513095.717037:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1690592507054720/t0(0) o400->c2fa5898-efcc-ce62-6eaf-05b387e1e5b1@10.31.140.108@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.717051:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690592507054720, offset 192 00000400:00000200:12.0:1616513095.717057:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.108@tcp1 00000400:00000200:12.0:1616513095.717065:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.108@tcp1 routed destination 00000400:00000200:12.0:1616513095.717081:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.108@tcp1(10.31.140.108@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.717087:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513095.717093:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.717096:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.717098:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.717103:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.717108:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513095.717110:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513095.717162:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.717169:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513095.717172:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513095.717180:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513095.717181:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513095.717187:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:20.0:1616513095.717191:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.717525:0:51135:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513095.717581:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.717590:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513095.717599:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.108@tcp1 : PUT - for me 00000400:00000200:18.0:1616513095.717609:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.108@tcp1 of length 224 into portal 12 MB=0x60195bdd7cd00 00000400:00000200:18.0:1616513095.717615:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.108@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 23856 00000400:00000200:18.0:1616513095.717620:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513095.717623:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513095.717631:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.717634:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.717635:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513095.717639:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513095.717651:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690592507055360 00010000:00000200:10.0:1616513095.717662:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464624c80 x1690592507055360/t0(0) o400->6726f997-aac8-a05b-2f6c-9f720354a204@10.31.140.108@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513095.717675:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690592507055360, offset 192 00000400:00000200:10.0:1616513095.717682:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.108@tcp1 00000400:00000200:10.0:1616513095.717689:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.108@tcp1 routed destination 00000400:00000200:10.0:1616513095.717701:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.108@tcp1(10.31.140.108@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.717707:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513095.717712:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.717714:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513095.717716:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513095.717720:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.717725:0:51135:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.717779:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.717787:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513095.717789:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513095.717796:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513095.717800:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e825d8 00000800:00000200:26.0:1616513095.717804:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.717807:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.730650:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513095.730706:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.730715:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513095.730724:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.188@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.730733:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.188@tcp1 of length 224 into portal 12 MB=0x5ffdcbdc79c00 00000400:00000200:14.0:1616513095.730739:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.188@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 24080 00000400:00000200:14.0:1616513095.730744:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.730747:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513095.730753:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.730756:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.730758:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.730761:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513095.730815:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698425416704 00000800:00000200:11.2:1616513095.730826:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:10.0:1616513095.730827:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfe780 x1688698425416704/t0(0) o400->bee00828-b035-104d-41f7-782ac89b9bd8@10.31.141.188@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513095.730840:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698425416704, offset 192 00000400:00000200:10.0:1616513095.730846:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.188@tcp1 00000400:00000200:10.0:1616513095.730852:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.188@tcp1 routed destination 00000400:00000200:10.0:1616513095.730866:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.188@tcp1(10.31.141.188@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.730873:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513095.730878:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.730880:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.730881:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513095.730883:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513095.730887:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513095.730891:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513095.730900:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.188@tcp1 : PUT - for me 00000400:00000200:0.0:1616513095.730909:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.188@tcp1 of length 224 into portal 26 MB=0x5ffdcbdc79980 00000400:00000200:0.0:1616513095.730916:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.188@tcp1 of length 224/224 into md 0xf97cf599 [1] + 0 00000400:00000200:0.0:1616513095.730920:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d100 00000400:00000200:0.0:1616513095.730923:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.730926:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513095.730930:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d100 00000800:00000200:0.0:1616513095.730933:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513095.730935:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513095.730936:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513095.730943:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513095.730944:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513095.730946:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.730947:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513095.730954:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.730958:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83870 00000800:00000200:16.0:1616513095.730962:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.730995:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698425416064 00010000:00000200:12.0:1616513095.731006:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688698425416064/t0(0) o400->c4b8001a-32fb-1359-c9fb-c94cb1c2c4a1@10.31.141.188@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.731020:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698425416064, offset 192 00000400:00000200:12.0:1616513095.731026:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.188@tcp1 00000400:00000200:12.0:1616513095.731033:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.188@tcp1 routed destination 00000400:00000200:12.0:1616513095.731048:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.188@tcp1(10.31.141.188@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.731054:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513095.731060:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.731065:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513095.731067:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513095.731072:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.731076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513095.731078:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513095.731131:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.731138:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513095.731141:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513095.731147:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513095.731153:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fcb0 00000800:00000200:26.0:1616513095.731156:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.731159:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.778986:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513095.779044:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.779054:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513095.779060:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:20.0:1616513095.779064:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.27.196@tcp1 : PUT - for me 00000800:00000200:0.0:1616513095.779070:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.779073:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.27.196@tcp1 of length 224 into portal 26 MB=0x5ffe9c77c7bc0 00000400:00000200:0.0:1616513095.779079:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.27.196@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.779080:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.27.196@tcp1 of length 224/224 into md 0xf97cf5a9 [1] + 0 00000400:00000200:20.0:1616513095.779085:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d540 00000400:00000200:20.0:1616513095.779088:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:0.0:1616513095.779088:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.27.196@tcp1 of length 224 into portal 12 MB=0x5ffe9c77c7c40 00000100:00000200:20.0:1616513095.779091:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513095.779096:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d540 00000400:00000200:0.0:1616513095.779096:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.27.196@tcp1 of length 224/224 into md 0x865be291 [1] + 56992 00000800:00000200:20.0:1616513095.779100:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:0.0:1616513095.779101:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513095.779103:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:0.0:1616513095.779104:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513095.779105:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.779108:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.779111:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.779114:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.779116:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.779164:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688754422840256 00010000:00000200:12.0:1616513095.779176:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688754422840256/t0(0) o400->4675c3bc-b98b-a901-ad52-2d7c61ac2e3f@10.31.27.196@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513095.779186:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688754422840384 00000100:00000200:12.0:1616513095.779191:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688754422840256, offset 192 00000400:00000200:12.0:1616513095.779197:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.27.196@tcp1 00010000:00000200:28.0:1616513095.779199:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86de80 x1688754422840384/t0(0) o400->3459ec93-8524-14ad-74a5-97b5a008dba8@10.31.27.196@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513095.779205:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.27.196@tcp1 routed destination 00000100:00000200:28.0:1616513095.779214:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688754422840384, offset 192 00000400:00000200:12.0:1616513095.779220:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.27.196@tcp1(10.31.27.196@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513095.779221:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.27.196@tcp1 00000800:00000200:12.0:1616513095.779227:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513095.779229:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.27.196@tcp1 routed destination 00000800:00000200:12.0:1616513095.779237:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.779240:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.779243:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513095.779245:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.27.196@tcp1(10.31.27.196@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.779248:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513095.779251:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513095.779253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513095.779255:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513095.779258:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.779261:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513095.779262:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513095.779263:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513095.779267:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513095.779269:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513095.779270:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513095.779273:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513095.779275:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513095.779280:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f5c8 00000800:00000200:20.0:1616513095.779283:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.779285:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.779348:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.779355:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.779358:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513095.779365:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513095.779369:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdcb0 00000800:00000200:14.0:1616513095.779373:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.779376:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.795678:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513095.795734:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.795744:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513095.795753:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.120@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.795762:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.120@tcp1 of length 224 into portal 26 MB=0x602ad7217f3c0 00000400:00000200:16.0:1616513095.795769:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.120@tcp1 of length 224/224 into md 0xf97cf5b9 [1] + 0 00000400:00000200:16.0:1616513095.795773:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d430 00000400:00000200:16.0:1616513095.795776:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.795779:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513095.795784:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d430 00000800:00000200:16.0:1616513095.795787:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.795788:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.795789:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513095.795792:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.795863:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691793827034048 00010000:00000200:12.0:1616513095.795875:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1691793827034048/t0(0) o400->08027aae-22a8-3480-e713-f3a7fc005999@10.31.141.120@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.795889:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691793827034048, offset 192 00000400:00000200:12.0:1616513095.795895:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.120@tcp1 00000400:00000200:12.0:1616513095.795903:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.120@tcp1 routed destination 00000400:00000200:12.0:1616513095.795918:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.120@tcp1(10.31.141.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.795924:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513095.795928:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513095.795930:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.795933:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.795935:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.795940:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.795945:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513095.795946:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513095.795957:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513095.795964:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513095.795967:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513095.795974:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513095.795979:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ebb0 00000800:00000200:24.0:1616513095.795983:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513095.795986:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513095.795986:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513095.795996:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513095.796005:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.120@tcp1 : PUT - for me 00000400:00000200:26.0:1616513095.796014:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.120@tcp1 of length 224 into portal 12 MB=0x602ad7217f440 00000400:00000200:26.0:1616513095.796020:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.120@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 24304 00000400:00000200:26.0:1616513095.796026:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513095.796030:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513095.796038:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.796041:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513095.796043:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.796046:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513095.796058:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691793827034176 00010000:00000200:10.0:1616513095.796069:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c696c00 x1691793827034176/t0(0) o400->8c2a9a1f-8bbe-51e2-47d6-ed6fac19b664@10.31.141.120@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513095.796083:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691793827034176, offset 192 00000400:00000200:10.0:1616513095.796088:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.120@tcp1 00000400:00000200:10.0:1616513095.796096:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.120@tcp1 routed destination 00000400:00000200:10.0:1616513095.796110:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.120@tcp1(10.31.141.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.796116:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513095.796122:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.796125:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513095.796127:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513095.796132:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.796137:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.796148:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.796154:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.796157:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513095.796163:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513095.796167:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e836d8 00000800:00000200:14.0:1616513095.796171:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.796173:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.798199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513095.798236:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513095.798255:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.798264:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513095.798273:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.90@tcp1 : PUT - for me 00000400:00000200:26.0:1616513095.798284:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.90@tcp1 of length 224 into portal 26 MB=0x5ffde8a6326c0 00000400:00000200:26.0:1616513095.798290:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.90@tcp1 of length 224/224 into md 0xf97cf5d9 [1] + 0 00000800:00000200:0.0:1616513095.798292:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513095.798295:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c088 00000400:00000200:26.0:1616513095.798298:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513095.798300:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1616513095.798302:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513095.798304:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c088 00000800:00000200:26.0:1616513095.798308:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.798309:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.798310:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:0.0:1616513095.798311:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.90@tcp1 : PUT - for me 00000800:00000200:26.0:1616513095.798314:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513095.798320:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.90@tcp1 of length 224 into portal 12 MB=0x5ffde8a632740 00000400:00000200:0.0:1616513095.798326:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.90@tcp1 of length 224/224 into md 0x865be291 [1] + 57216 00000400:00000200:0.0:1616513095.798332:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.798334:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513095.798341:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.798344:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513095.798345:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513095.798349:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.798379:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706153129664 00010000:00000200:12.0:1616513095.798389:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688706153129664/t0(0) o400->8a86e990-792e-484c-8b9f-db20162c3bf6@10.31.142.90@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.798404:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688706153129664, offset 192 00000100:00000200:28.0:1616513095.798407:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688706153129792 00000400:00000200:12.0:1616513095.798410:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.90@tcp1 00000400:00000200:12.0:1616513095.798418:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.90@tcp1 routed destination 00010000:00000200:28.0:1616513095.798419:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86a400 x1688706153129792/t0(0) o400->346227fa-b28b-1238-4d67-e10621214188@10.31.142.90@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513095.798433:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.90@tcp1(10.31.142.90@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513095.798446:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688706153129792, offset 192 00000800:00000200:12.0:1616513095.798451:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513095.798453:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.90@tcp1 00000800:00000200:12.0:1616513095.798457:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.798460:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:28.0:1616513095.798461:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.90@tcp1 routed destination 00000800:00000200:12.0:1616513095.798462:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.798466:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.798472:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513095.798474:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513095.798476:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.90@tcp1(10.31.142.90@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.798482:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513095.798484:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513095.798488:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.798491:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513095.798491:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513095.798493:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513095.798493:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513095.798498:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513095.798502:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513095.798504:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513095.798508:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777edd0 00000800:00000200:24.0:1616513095.798511:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513095.798513:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513095.798514:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513095.798520:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.798522:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513095.798527:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513095.798527:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513095.798530:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdba0 00000800:00000200:14.0:1616513095.798533:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.801275:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513095.801331:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.801340:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513095.801349:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.136@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.801359:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.136@tcp1 of length 224 into portal 26 MB=0x604a54522c080 00000400:00000200:16.0:1616513095.801366:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.136@tcp1 of length 224/224 into md 0xf97cf5f9 [1] + 0 00000400:00000200:16.0:1616513095.801370:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922ded0 00000400:00000200:16.0:1616513095.801375:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.801378:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513095.801382:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922ded0 00000800:00000200:0.0:1616513095.801384:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.801385:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.801388:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513095.801391:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513095.801394:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.801394:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513095.801403:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.136@tcp1 : PUT - for me 00000400:00000200:0.0:1616513095.801412:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.136@tcp1 of length 224 into portal 12 MB=0x604a54522c180 00000400:00000200:0.0:1616513095.801418:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.136@tcp1 of length 224/224 into md 0x865be291 [1] + 57440 00000400:00000200:0.0:1616513095.801424:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.801426:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513095.801447:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.801450:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.801452:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.801458:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693957736284288 00000100:00000200:28.0:1616513095.801469:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693957736284544 00010000:00000200:12.0:1616513095.801469:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1693957736284288/t0(0) o400->77743738-d5a5-bf57-37cb-9e7bff522ca1@10.31.140.136@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:28.0:1616513095.801481:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86f080 x1693957736284544/t0(0) o400->445b2b62-97e4-74c9-24d1-2abc4acaf0c8@10.31.140.136@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513095.801483:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693957736284288, offset 192 00000400:00000200:12.0:1616513095.801489:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.136@tcp1 00000100:00000200:28.0:1616513095.801496:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693957736284544, offset 192 00000400:00000200:12.0:1616513095.801497:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.136@tcp1 routed destination 00000400:00000200:28.0:1616513095.801502:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.136@tcp1 00000400:00000200:12.0:1616513095.801512:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.136@tcp1(10.31.140.136@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.801518:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513095.801521:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.136@tcp1 routed destination 00000800:00000200:12.0:1616513095.801530:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.801533:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.801535:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513095.801538:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.136@tcp1(10.31.140.136@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.801540:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513095.801544:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513095.801546:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513095.801547:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513095.801550:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.801553:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513095.801555:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513095.801559:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.801565:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513095.801600:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.801608:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513095.801610:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513095.801612:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513095.801617:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513095.801621:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513095.801623:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f540 00000800:00000200:20.0:1616513095.801626:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513095.801630:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513095.801633:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513095.801639:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513095.801643:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcda90 00000800:00000200:2.0:1616513095.801647:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.801650:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.824971:0:51146:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513095.825026:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.825035:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513095.825044:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.30@tcp1 : PUT - for me 00000400:00000200:18.0:1616513095.825053:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.30@tcp1 of length 224 into portal 26 MB=0x5ffeb993eea80 00000400:00000200:18.0:1616513095.825060:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.30@tcp1 of length 224/224 into md 0xf97cf619 [1] + 0 00000400:00000200:18.0:1616513095.825065:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d8f8 00000400:00000200:18.0:1616513095.825068:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513095.825071:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513095.825075:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d8f8 00000800:00000200:18.0:1616513095.825078:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.825080:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.825080:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513095.825087:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513095.825090:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.825091:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513095.825099:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.30@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.825108:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.30@tcp1 of length 224 into portal 12 MB=0x5ffeb993eeb40 00000400:00000200:14.0:1616513095.825115:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.30@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 24528 00000400:00000200:14.0:1616513095.825120:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.825123:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513095.825130:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.825133:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.825135:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513095.825139:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688762236988032 00010000:00000200:12.0:1616513095.825150:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688762236988032/t0(0) o400->5b39c104-0cf6-304c-2304-7d883efb3bd3@10.31.140.30@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513095.825150:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688762236988224 00010000:00000200:10.0:1616513095.825162:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfc800 x1688762236988224/t0(0) o400->08644e19-f39f-09ea-650b-1ce5ad6fb05c@10.31.140.30@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513095.825165:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688762236988032, offset 192 00000400:00000200:12.0:1616513095.825173:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.30@tcp1 00000100:00000200:10.0:1616513095.825176:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688762236988224, offset 192 00000400:00000200:12.0:1616513095.825180:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.30@tcp1 routed destination 00000400:00000200:10.0:1616513095.825183:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.30@tcp1 00000400:00000200:10.0:1616513095.825191:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.30@tcp1 routed destination 00000400:00000200:12.0:1616513095.825196:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.30@tcp1(10.31.140.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.825203:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:10.0:1616513095.825206:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.30@tcp1(10.31.140.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.825208:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.825211:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.825213:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513095.825213:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513095.825218:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513095.825218:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.825221:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513095.825223:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513095.825224:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513095.825228:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.825234:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513095.825266:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513095.825304:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.825311:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513095.825314:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513095.825315:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513095.825320:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513095.825324:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83a90 00000800:00000200:24.0:1616513095.825324:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513095.825328:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513095.825331:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513095.825334:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513095.825340:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513095.825346:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ed48 00000800:00000200:24.0:1616513095.825349:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513095.825352:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.836733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513095.836786:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.836795:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513095.836804:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.10@tcp1 : PUT - for me 00000400:00000200:14.0:1616513095.836813:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.10@tcp1 of length 224 into portal 12 MB=0x603943b8ded00 00000400:00000200:14.0:1616513095.836819:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.10@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 24752 00000400:00000200:14.0:1616513095.836824:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513095.836827:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513095.836833:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.836836:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513095.836839:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.836843:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513095.836844:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692785049464064 00010000:00000200:10.0:1616513095.836851:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfa400 x1692785049464064/t0(0) o400->28791973-4d95-f33a-a0ad-065c0efa0990@10.31.140.10@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513095.836861:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692785049464064, offset 192 00000400:00000200:10.0:1616513095.836867:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.10@tcp1 00000400:00000200:10.0:1616513095.836873:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.10@tcp1 routed destination 00000400:00000200:10.0:1616513095.836886:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.10@tcp1(10.31.140.10@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.836892:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:11.2:1616513095.836894:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513095.836896:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.836899:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513095.836901:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:10.0:1616513095.836904:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513095.836949:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.836958:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513095.836968:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.10@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.836976:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.10@tcp1 of length 224 into portal 26 MB=0x603943b8dec80 00000400:00000200:16.0:1616513095.836983:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.10@tcp1 of length 224/224 into md 0xf97cf659 [1] + 0 00000400:00000200:16.0:1616513095.836987:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c7f8 00000400:00000200:16.0:1616513095.836990:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.836993:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513095.836997:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c7f8 00000800:00000200:16.0:1616513095.837000:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513095.837003:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:16.0:1616513095.837003:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513095.837010:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513095.837011:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513095.837013:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513095.837014:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513095.837020:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513095.837024:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83b18 00000800:00000200:20.0:1616513095.837028:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.837072:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692785049463936 00010000:00000200:12.0:1616513095.837083:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1692785049463936/t0(0) o400->6fd45521-bf6c-8818-b855-89119c43bf79@10.31.140.10@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.837097:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692785049463936, offset 192 00000400:00000200:12.0:1616513095.837103:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.10@tcp1 00000400:00000200:12.0:1616513095.837121:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.10@tcp1 routed destination 00000400:00000200:12.0:1616513095.837135:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.10@tcp1(10.31.140.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.837141:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513095.837146:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.837149:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513095.837151:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513095.837155:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.837160:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513095.837161:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513095.837171:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.837178:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513095.837181:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513095.837187:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513095.837193:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e990 00000800:00000200:18.0:1616513095.837197:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.837199:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:31.0:1616513095.871548:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513095.871556:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513095.899000:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513095.899057:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.899067:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513095.899076:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.94@tcp1 : PUT - for me 00000400:00000200:0.0:1616513095.899085:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.94@tcp1 of length 224 into portal 26 MB=0x6001d4b1ed680 00000400:00000200:0.0:1616513095.899092:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.94@tcp1 of length 224/224 into md 0xf97cf669 [1] + 0 00000400:00000200:0.0:1616513095.899099:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d078 00000400:00000200:0.0:1616513095.899102:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513095.899105:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513095.899109:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d078 00000800:00000200:16.0:1616513095.899111:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.899113:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.899116:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.899117:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.899176:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975674627712 00010000:00000200:12.0:1616513095.899187:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688975674627712/t0(0) o400->8cef55a2-de03-c4a4-4718-e4402b6270c3@10.31.142.94@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.899202:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688975674627712, offset 192 00000400:00000200:12.0:1616513095.899208:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.94@tcp1 00000400:00000200:12.0:1616513095.899215:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.94@tcp1 routed destination 00000400:00000200:12.0:1616513095.899230:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.94@tcp1(10.31.142.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.899236:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513095.899244:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.899247:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.899249:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.899255:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.899260:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513095.899261:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513095.899314:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.899321:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513095.899324:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513095.899331:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513095.899337:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e220 00000800:00000200:20.0:1616513095.899340:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.899343:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513095.899493:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513095.899504:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.899511:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513095.899513:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.899520:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.94@tcp1 : PUT - for me 00000400:00000200:16.0:1616513095.899529:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.94@tcp1 of length 224 into portal 12 MB=0x6001d4b1edbc0 00000400:00000200:16.0:1616513095.899536:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.94@tcp1 of length 224/224 into md 0x865be291 [1] + 57664 00000400:00000200:16.0:1616513095.899541:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513095.899544:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513095.899551:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513095.899554:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513095.899556:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513095.899615:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975674629056 00010000:00000200:28.0:1616513095.899628:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cd6724c80 x1688975674629056/t0(0) o400->ced8b216-6aca-6082-2072-f13fc9ec4b03@10.31.142.94@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513095.899642:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688975674629056, offset 192 00000400:00000200:28.0:1616513095.899648:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.94@tcp1 00000400:00000200:28.0:1616513095.899659:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.94@tcp1 routed destination 00000400:00000200:28.0:1616513095.899673:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.94@tcp1(10.31.142.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.899679:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513095.899685:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.899688:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513095.899690:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513095.899696:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.899702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.899758:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.899766:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513095.899769:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513095.899776:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513095.899780:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd980 00000800:00000200:2.0:1616513095.899784:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513095.899787:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513095.914533:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929344, portal 4 00000100:00000200:26.0:1616513095.914539:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 24, xid 1693067725929344, offset 0 00000400:00000200:26.0:1616513095.914545:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:26.0:1616513095.914551:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:26.0:1616513095.914564:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513095.914571:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:26.0:1616513095.914577:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513095.914580:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:26.0:1616513095.914582:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:26.0:1616513095.914587:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513095.914634:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:18.0:1616513095.914691:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:18.0:1616513095.914698:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:18.0:1616513095.914701:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513095.914702:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000400:00000200:18.0:1616513095.914707:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000100:00000200:18.0:1616513095.914713:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9642880 x1693067725929344/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513102 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513095.914725:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afba0 00000800:00000200:18.0:1616513095.914730:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:31.2:1616513095.914752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:2.0:1616513095.914807:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:2.0:1616513095.914816:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.234@o2ib1 00000400:00000200:2.0:1616513095.914825:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513095.914833:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 336 into portal 4 MB=0x603d60c627b80 00000400:00000200:2.0:1616513095.914839:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.234@o2ib1 of length 336/336 into md 0xf97cfe79 [1] + 192 00000400:00000200:2.0:1616513095.914845:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513095.914850:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9642880 x1693067725929344/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513095.914860:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:2.0:1616513095.914865:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:2.0:1616513095.914869:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)-- 00000100:00000200:26.0:1616513095.914871:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9642880 x1693067725929344/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513095.914871:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:26.0:1616513095.914885:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9642880 x1693067725929344/t0(0) o41->boslfs02-MDT0000-osp-MDT0001@10.242.46.234@o2ib1:24/4 lens 224/368 e 0 to 0 dl 1616513102 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513095.914895:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa90 00000800:00000200:11.2:1616513095.919236:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513095.919292:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513095.919297:0:51156:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513095.919301:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.919310:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.178@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.919319:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.178@tcp1 of length 224 into portal 26 MB=0x5ffdc89a6e300 00000400:00000200:20.0:1616513095.919326:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.178@tcp1 of length 224/224 into md 0xf97cf689 [1] + 0 00000400:00000200:20.0:1616513095.919330:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922cee0 00000400:00000200:20.0:1616513095.919333:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.919336:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513095.919341:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cee0 00000800:00000200:20.0:1616513095.919344:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513095.919346:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.919347:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.919350:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513095.919369:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.919379:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513095.919388:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.178@tcp1 : PUT - for me 00000400:00000200:26.0:1616513095.919397:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.178@tcp1 of length 224 into portal 12 MB=0x5ffdc89a6e500 00000400:00000200:26.0:1616513095.919403:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.178@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 24976 00000100:00000200:12.0:1616513095.919408:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697550856960 00000400:00000200:26.0:1616513095.919409:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513095.919412:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513095.919419:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00010000:00000200:12.0:1616513095.919419:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688697550856960/t0(0) o400->3f426e46-d924-8947-884a-ba9d6cfe333b@10.31.131.178@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513095.919422:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513095.919424:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.919426:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513095.919446:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697550856960, offset 192 00000400:00000200:12.0:1616513095.919453:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.178@tcp1 00000400:00000200:12.0:1616513095.919460:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.178@tcp1 routed destination 00000400:00000200:12.0:1616513095.919475:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.178@tcp1(10.31.131.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.919482:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513095.919487:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.919490:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.919492:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:10.0:1616513095.919496:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697550857472 00000800:00000200:12.0:1616513095.919497:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.919502:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513095.919505:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:10.0:1616513095.919509:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c696780 x1688697550857472/t0(0) o400->ea8b8e04-2558-5ea5-ca91-b2940fce99c6@10.31.131.178@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513095.919527:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:10.0:1616513095.919534:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697550857472, offset 192 00000800:00000200:0.0:1616513095.919534:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513095.919537:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513095.919540:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.178@tcp1 00000400:00000200:0.0:1616513095.919544:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:10.0:1616513095.919548:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.178@tcp1 routed destination 00000400:00000200:0.0:1616513095.919549:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e908 00000800:00000200:0.0:1616513095.919553:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.919556:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513095.919562:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.178@tcp1(10.31.131.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.919569:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513095.919575:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.919578:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513095.919580:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513095.919585:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.919589:0:51156:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513095.919643:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.919650:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.919653:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513095.919660:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513095.919664:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83c28 00000800:00000200:14.0:1616513095.919667:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.919670:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513095.920382:0:51156:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513095.920393:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.920398:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513095.920401:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:26.0:1616513095.920406:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.92@tcp1 : PUT - for me 00000800:00000200:18.0:1616513095.920410:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513095.920418:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513095.920419:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.92@tcp1 of length 224 into portal 26 MB=0x601ab57277740 00000400:00000200:18.0:1616513095.920419:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.92@tcp1 : PUT - for me 00000400:00000200:26.0:1616513095.920424:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.92@tcp1 of length 224/224 into md 0xf97cf6b9 [1] + 0 00000400:00000200:18.0:1616513095.920428:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.92@tcp1 of length 224 into portal 12 MB=0x601ab572777c0 00000400:00000200:26.0:1616513095.920429:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d210 00000400:00000200:26.0:1616513095.920431:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513095.920444:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.92@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 25200 00000100:00000200:26.0:1616513095.920449:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513095.920449:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513095.920453:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d210 00000100:00000200:18.0:1616513095.920455:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513095.920456:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.920458:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513095.920459:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.920463:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.920466:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513095.920466:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690685273503552 00000800:00000200:18.0:1616513095.920468:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:12.0:1616513095.920472:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1690685273503552/t0(0) o400->41d5a961-71b9-a8dd-1f83-0c78d27211c6@10.31.140.92@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513095.920475:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690685273503680 00010000:00000200:10.0:1616513095.920481:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464626c00 x1690685273503680/t0(0) o400->9d458c94-07d8-8efa-a81f-2e4ac17d883e@10.31.140.92@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513095.920484:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690685273503552, offset 192 00000400:00000200:12.0:1616513095.920489:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.92@tcp1 00000100:00000200:10.0:1616513095.920492:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690685273503680, offset 192 00000400:00000200:12.0:1616513095.920495:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.92@tcp1 routed destination 00000400:00000200:10.0:1616513095.920497:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.92@tcp1 00000400:00000200:10.0:1616513095.920503:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.92@tcp1 routed destination 00000400:00000200:12.0:1616513095.920508:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.92@tcp1(10.31.140.92@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.920514:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:10.0:1616513095.920516:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.92@tcp1(10.31.140.92@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.920519:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.920521:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513095.920522:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513095.920523:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.920526:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513095.920527:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.920529:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513095.920531:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513095.920531:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513095.920532:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513095.920534:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.920538:0:51156:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513095.920549:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.920556:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513095.920559:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513095.920559:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513095.920565:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513095.920569:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82088 00000800:00000200:2.0:1616513095.920573:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513095.920584:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513095.920590:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513095.920593:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513095.920600:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513095.920605:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f6d8 00000800:00000200:24.0:1616513095.920609:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513095.920612:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.939728:0:51158:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513095.939783:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.939793:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513095.939802:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.245.87.15@tcp1 : PUT - for me 00000400:00000200:26.0:1616513095.939813:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.245.87.15@tcp1 of length 224 into portal 12 MB=0x5e2b7cb993e00 00000400:00000200:26.0:1616513095.939821:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.245.87.15@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 25424 00000400:00000200:26.0:1616513095.939826:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513095.939829:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513095.939836:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513095.939839:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513095.939841:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.939843:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513095.939898:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1656653906263552 00010000:00000200:10.0:1616513095.939910:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c692d00 x1656653906263552/t0(0) o400->5761f818-f4e3-76be-12f6-5faa41aba7cc@10.245.87.15@tcp1:2/0 lens 224/224 e 0 to 0 dl 1616513137 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513095.939923:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1656653906263552, offset 192 00000400:00000200:10.0:1616513095.939929:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.245.87.15@tcp1 00000400:00000200:10.0:1616513095.939936:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.245.87.15@tcp1 routed destination 00000400:00000200:10.0:1616513095.939950:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.245.87.15@tcp1(10.245.87.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.939958:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513095.939963:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.939965:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513095.939967:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513095.939972:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513095.940019:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513095.940075:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513095.940082:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513095.940085:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513095.940091:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513095.940095:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82f68 00000800:00000200:0.0:1616513095.940099:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513095.940102:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513095.949070:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513095.949126:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513095.949136:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513095.949145:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.171@tcp1 : PUT - for me 00000400:00000200:24.0:1616513095.949154:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.171@tcp1 of length 224 into portal 26 MB=0x5ffdb9099f800 00000400:00000200:24.0:1616513095.949161:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.171@tcp1 of length 224/224 into md 0xf97cf6d9 [1] + 0 00000400:00000200:24.0:1616513095.949166:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c880 00000400:00000200:24.0:1616513095.949169:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513095.949172:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513095.949176:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c880 00000800:00000200:24.0:1616513095.949180:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.949180:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513095.949182:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513095.949187:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513095.949190:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.949190:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.949199:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.171@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.949208:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.171@tcp1 of length 224 into portal 12 MB=0x5ffdb9099fa80 00000400:00000200:20.0:1616513095.949214:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.171@tcp1 of length 224/224 into md 0x865be291 [1] + 57888 00000400:00000200:20.0:1616513095.949220:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.949222:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513095.949229:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.949232:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.949234:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.949241:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693372483584 00010000:00000200:12.0:1616513095.949252:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688693372483584/t0(0) o400->89ba2de4-4fa9-bfd6-462d-d0adba5313c1@10.31.140.171@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.949266:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693372483584, offset 192 00000400:00000200:12.0:1616513095.949272:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.171@tcp1 00000400:00000200:12.0:1616513095.949280:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.171@tcp1 routed destination 00000400:00000200:12.0:1616513095.949295:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.171@tcp1(10.31.140.171@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.949301:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513095.949307:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.949310:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:28.0:1616513095.949312:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693372484224 00000800:00000200:12.0:1616513095.949312:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513095.949317:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513095.949323:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513095.949323:0:51159:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:28.0:1616513095.949325:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eba80 x1688693372484224/t0(0) o400->a4226746-b337-7d89-908f-efabe9332e8a@10.31.140.171@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513095.949339:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693372484224, offset 192 00000400:00000200:28.0:1616513095.949345:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.171@tcp1 00000400:00000200:28.0:1616513095.949353:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.171@tcp1 routed destination 00000400:00000200:28.0:1616513095.949367:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.171@tcp1(10.31.140.171@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513095.949379:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513095.949383:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513095.949384:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513095.949387:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513095.949391:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513095.949391:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.949393:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513095.949396:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513095.949400:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513095.949401:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513095.949406:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f870 00000800:00000200:14.0:1616513095.949409:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.949412:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513095.949460:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513095.949469:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513095.949472:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513095.949473:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513095.949479:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513095.949483:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd870 00000800:00000200:16.0:1616513095.949486:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.950727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513095.950740:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.950749:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513095.950756:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513095.950758:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.226@tcp1 : PUT - for me 00000400:00000200:26.0:1616513095.950767:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.226@tcp1 of length 224 into portal 12 MB=0x60392fb7b9d40 00000400:00000200:26.0:1616513095.950774:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.226@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 25648 00000400:00000200:26.0:1616513095.950779:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513095.950782:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513095.950788:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513095.950791:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513095.950793:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513095.950852:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692779679554880 00010000:00000200:10.0:1616513095.950865:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c694800 x1692779679554880/t0(0) o400->c54275a3-0809-c099-3b7c-210d79721722@10.31.140.226@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513095.950879:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692779679554880, offset 192 00000400:00000200:10.0:1616513095.950885:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.226@tcp1 00000400:00000200:10.0:1616513095.950893:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.226@tcp1 routed destination 00000400:00000200:10.0:1616513095.950908:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.226@tcp1(10.31.140.226@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.950915:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513095.950921:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.950923:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513095.950926:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513095.950930:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.950935:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513095.950938:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513095.950988:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513095.950995:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513095.950997:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513095.951004:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513095.951006:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513095.951008:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83f58 00000800:00000200:14.0:1616513095.951012:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513095.951015:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513095.951016:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513095.951025:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.226@tcp1 : PUT - for me 00000400:00000200:20.0:1616513095.951033:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.226@tcp1 of length 224 into portal 26 MB=0x60392fb7b9ac0 00000400:00000200:20.0:1616513095.951042:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.226@tcp1 of length 224/224 into md 0xf97cf6f9 [1] + 0 00000400:00000200:20.0:1616513095.951047:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922ce58 00000400:00000200:20.0:1616513095.951050:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513095.951052:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513095.951056:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922ce58 00000800:00000200:20.0:1616513095.951059:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.951062:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513095.951064:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.951067:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513095.951119:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692779679554240 00010000:00000200:12.0:1616513095.951129:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1692779679554240/t0(0) o400->69cd725a-0eb6-9212-a530-effa5ed1978b@10.31.140.226@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.951142:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692779679554240, offset 192 00000400:00000200:12.0:1616513095.951148:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.226@tcp1 00000400:00000200:12.0:1616513095.951155:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.226@tcp1 routed destination 00000400:00000200:12.0:1616513095.951169:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.226@tcp1(10.31.140.226@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513095.951175:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513095.951180:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.951183:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513095.951185:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513095.951189:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513095.951195:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513095.951199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513095.951252:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513095.951260:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513095.951263:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513095.951269:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513095.951275:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eaa0 00000800:00000200:24.0:1616513095.951278:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513095.951281:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513095.995637:0:51164:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513095.995695:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.995705:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513095.995715:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.153@tcp1 : PUT - for me 00000400:00000200:18.0:1616513095.995726:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.153@tcp1 of length 224 into portal 26 MB=0x5ffdbf4c46940 00000400:00000200:18.0:1616513095.995733:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.153@tcp1 of length 224/224 into md 0xf97cf709 [1] + 0 00000400:00000200:18.0:1616513095.995738:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c000 00000400:00000200:18.0:1616513095.995741:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513095.995744:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513095.995749:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c000 00000800:00000200:18.0:1616513095.995753:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513095.995756:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513095.995757:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.995760:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.995767:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513095.995768:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513095.995774:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.153@tcp1 : PUT - for me 00000400:00000200:18.0:1616513095.995780:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.153@tcp1 of length 224 into portal 12 MB=0x5ffdbf4c46a00 00000400:00000200:18.0:1616513095.995785:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.153@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 25872 00000400:00000200:18.0:1616513095.995789:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513095.995791:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513095.995797:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513095.995799:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513095.995800:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513095.995819:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695052986688 00010000:00000200:12.0:1616513095.995831:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688695052986688/t0(0) o400->059e17b0-006a-d7a9-bc1d-970e823ff678@10.31.140.153@tcp1:52/0 lens 224/224 e 0 to 0 dl 1616513187 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513095.995846:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695052986688, offset 192 00000400:00000200:12.0:1616513095.995854:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.153@tcp1 00000400:00000200:12.0:1616513095.995862:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.153@tcp1 routed destination 00000400:00000200:12.0:1616513095.995879:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.153@tcp1(10.31.140.153@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513095.995885:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695052986880 00000800:00000200:12.0:1616513095.995886:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513095.995893:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513095.995896:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513095.995898:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:10.0:1616513095.995898:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623f00 x1688695052986880/t0(0) o400->948f6728-2420-a2cc-267f-ef8f9009aad7@10.31.140.153@tcp1:721/0 lens 224/224 e 0 to 0 dl 1616513101 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513095.995904:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513095.995910:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513095.995911:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:10.0:1616513095.995912:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695052986880, offset 192 00000400:00000200:10.0:1616513095.995918:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.153@tcp1 00000400:00000200:10.0:1616513095.995925:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.153@tcp1 routed destination 00000400:00000200:10.0:1616513095.995939:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.153@tcp1(10.31.140.153@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513095.995945:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513095.995951:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513095.995953:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513095.995955:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513095.995960:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513095.995965:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513095.995972:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513095.995975:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513095.995976:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513095.995984:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513095.995989:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ecc0 00000800:00000200:2.0:1616513095.995993:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513095.996007:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513095.996063:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513095.996070:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513095.996073:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513095.996080:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513095.996084:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82aa0 00000800:00000200:20.0:1616513095.996088:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513095.996091:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:21.0:1616513096.011629:0:6495:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929360, portal 4 00000100:00000200:21.0:1616513096.011635:0:6495:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929360, offset 0 00000400:00000200:21.0:1616513096.011642:0:6495:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:21.0:1616513096.011649:0:6495:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:21.0:1616513096.011663:0:6495:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:21.0:1616513096.011670:0:6495:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:21.0:1616513096.011678:0:6495:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513096.011681:0:6495:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:21.0:1616513096.011684:0:6495:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:21.0:1616513096.011690:0:6495:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513096.011698:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513096.011764:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:18.0:1616513096.011771:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:18.0:1616513096.011775:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.011782:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:18.0:1616513096.011789:0:6471:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d853e8c9f80 x1693067725929360/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:18.0:1616513096.011801:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf51f58 00000800:00000200:18.0:1616513096.011805:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513096.011808:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513096.011916:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513096.011988:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513096.011997:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:26.0:1616513096.012007:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513096.012015:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627b90 00000400:00000200:26.0:1616513096.012022:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cff39 [1] + 192 00000400:00000200:26.0:1616513096.012028:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.012033:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d853e8c9f80 x1693067725929360/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513096.012042:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513096.012049:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513096.012052:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513096.012054:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:21.0:1616513096.012059:0:6495:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d853e8c9f80 x1693067725929360/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:21.0:1616513096.012072:0:6495:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d853e8c9f80 x1693067725929360/t0(0) o13->boslfs02-OST002c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:21.0:1616513096.012083:0:6495:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50c38 00000800:00000200:11.2:1616513096.018524:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513096.018580:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.018590:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513096.018599:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:24.0:1616513096.018609:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d0322cb400 00000400:00000200:24.0:1616513096.018615:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0x865be291 [1] + 58112 00000400:00000200:24.0:1616513096.018621:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513096.018623:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513096.018630:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.018634:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513096.018634:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513096.018639:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513096.018698:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042590135296 00010000:00000200:28.0:1616513096.018786:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce9680 x1693042590135296/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:722/0 lens 440/368 e 0 to 0 dl 1616513102 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513096.018801:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042590135296, offset 192 00000400:00000200:28.0:1616513096.018808:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:28.0:1616513096.018816:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:28.0:1616513096.018832:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.018838:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.018844:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.018847:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.018849:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.018855:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.018863:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.018918:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.018925:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.018928:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.018934:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.018939:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd760 00000800:00000200:18.0:1616513096.018943:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.018945:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.030310:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513096.030367:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513096.030376:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513096.030385:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.34@tcp1 : PUT - for me 00000400:00000200:20.0:1616513096.030394:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.34@tcp1 of length 224 into portal 26 MB=0x5ffdb955fd880 00000400:00000200:20.0:1616513096.030401:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.34@tcp1 of length 224/224 into md 0xf97cf719 [1] + 0 00000400:00000200:20.0:1616513096.030406:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922cff0 00000400:00000200:20.0:1616513096.030409:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513096.030411:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513096.030416:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cff0 00000800:00000200:20.0:1616513096.030420:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.030421:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513096.030423:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513096.030428:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513096.030431:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.030431:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513096.030455:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.34@tcp1 : PUT - for me 00000400:00000200:0.0:1616513096.030464:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.34@tcp1 of length 224 into portal 12 MB=0x5ffdb955fd900 00000400:00000200:0.0:1616513096.030471:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.34@tcp1 of length 224/224 into md 0x865be291 [1] + 58552 00000400:00000200:0.0:1616513096.030476:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513096.030479:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513096.030486:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.030489:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.030491:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.030492:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693452560512 00010000:00000200:12.0:1616513096.030504:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688693452560512/t0(0) o400->a53203dd-a62b-3017-fb93-4aba0514f6b9@10.31.140.34@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.030518:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693452560512, offset 192 00000400:00000200:12.0:1616513096.030524:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.34@tcp1 00000400:00000200:12.0:1616513096.030531:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.34@tcp1 routed destination 00000400:00000200:12.0:1616513096.030546:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.34@tcp1(10.31.140.34@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513096.030550:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693452560640 00000800:00000200:12.0:1616513096.030553:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.030559:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.030562:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:28.0:1616513096.030564:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde869b00 x1688693452560640/t0(0) o400->a3cb54fa-a71f-d315-fbea-9684485f539a@10.31.140.34@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513096.030564:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.030569:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.030575:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.030576:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513096.030578:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693452560640, offset 192 00000400:00000200:28.0:1616513096.030585:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.34@tcp1 00000400:00000200:28.0:1616513096.030592:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.34@tcp1 routed destination 00000400:00000200:28.0:1616513096.030607:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.34@tcp1(10.31.140.34@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.030614:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.030620:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.030623:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.030625:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513096.030629:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513096.030630:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513096.030636:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513096.030636:0:51168:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513096.030639:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513096.030640:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513096.030645:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513096.030651:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e4c8 00000800:00000200:24.0:1616513096.030654:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513096.030695:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.030702:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513096.030705:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513096.030711:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513096.030716:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd650 00000800:00000200:2.0:1616513096.030719:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513096.030722:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.051375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513096.051432:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.051448:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513096.051456:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513096.051458:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513096.051466:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.117@tcp1 : PUT - for me 00000400:00000200:0.0:1616513096.051469:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.117@tcp1 : PUT - for me 00000400:00000200:16.0:1616513096.051476:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.117@tcp1 of length 224 into portal 26 MB=0x5f9153f62dac0 00000400:00000200:0.0:1616513096.051479:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.117@tcp1 of length 224 into portal 12 MB=0x5f9153f62db40 00000400:00000200:16.0:1616513096.051483:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.117@tcp1 of length 224/224 into md 0xf97cf729 [1] + 0 00000400:00000200:0.0:1616513096.051485:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.117@tcp1 of length 224/224 into md 0x865be291 [1] + 58776 00000400:00000200:16.0:1616513096.051488:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922ca18 00000400:00000200:16.0:1616513096.051491:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:0.0:1616513096.051491:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513096.051493:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:16.0:1616513096.051494:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513096.051498:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922ca18 00000800:00000200:0.0:1616513096.051500:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513096.051502:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513096.051502:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513096.051504:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:0.0:1616513096.051504:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:0.0:1616513096.051506:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.051509:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.051563:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681244536625856 00010000:00000200:12.0:1616513096.051575:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1681244536625856/t0(0) o400->4c1afb07-d0b6-9889-8c1f-ce80b499925b@10.31.143.117@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513096.051584:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681244536625984 00000100:00000200:12.0:1616513096.051589:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681244536625856, offset 192 00000400:00000200:12.0:1616513096.051596:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.117@tcp1 00010000:00000200:28.0:1616513096.051597:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86f980 x1681244536625984/t0(0) o400->922f3e09-2c84-ca46-0162-ce7e015a2d15@10.31.143.117@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513096.051604:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.117@tcp1 routed destination 00000100:00000200:28.0:1616513096.051611:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681244536625984, offset 192 00000400:00000200:28.0:1616513096.051619:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.117@tcp1 00000400:00000200:12.0:1616513096.051619:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.117@tcp1(10.31.143.117@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.051630:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513096.051632:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.117@tcp1 routed destination 00000800:00000200:12.0:1616513096.051640:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.051643:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.051645:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513096.051647:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.117@tcp1(10.31.143.117@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.051650:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513096.051654:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513096.051655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.051657:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513096.051660:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.051663:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.051665:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.051670:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.051676:0:51170:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513096.051719:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.051726:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513096.051729:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513096.051730:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513096.051736:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513096.051739:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513096.051741:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f320 00000800:00000200:24.0:1616513096.051744:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513096.051746:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.051748:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.051755:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.051759:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd540 00000800:00000200:18.0:1616513096.051763:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.051766:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.051772:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513096.051774:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.051778:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.051779:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513096.051782:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.055273:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513096.055326:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.055336:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.055346:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.155@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.055356:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.155@tcp1 of length 224 into portal 26 MB=0x603e13d6e7ec0 00000400:00000200:26.0:1616513096.055362:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.155@tcp1 of length 224/224 into md 0xf97cf739 [1] + 0 00000400:00000200:26.0:1616513096.055367:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922ddc0 00000400:00000200:26.0:1616513096.055369:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.055372:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513096.055376:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922ddc0 00000800:00000200:26.0:1616513096.055379:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.055379:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.055382:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.055386:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.055389:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.055390:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.055399:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.155@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.055407:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.155@tcp1 of length 224 into portal 12 MB=0x603e13d6e7f80 00000400:00000200:14.0:1616513096.055414:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.155@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 26096 00000400:00000200:14.0:1616513096.055419:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.055422:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513096.055429:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.055432:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.055455:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.055457:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115793440448 00010000:00000200:12.0:1616513096.055468:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1693115793440448/t0(0) o400->290345ad-4f3d-852e-939a-5ea5613bea39@10.31.136.155@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513096.055469:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115793440640 00010000:00000200:10.0:1616513096.055480:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfb180 x1693115793440640/t0(0) o400->69b84329-3ff4-eaaf-8b93-15559330dff7@10.31.136.155@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513096.055482:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693115793440448, offset 192 00000400:00000200:12.0:1616513096.055488:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.155@tcp1 00000100:00000200:10.0:1616513096.055494:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693115793440640, offset 192 00000400:00000200:12.0:1616513096.055496:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.155@tcp1 routed destination 00000400:00000200:10.0:1616513096.055499:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.155@tcp1 00000400:00000200:10.0:1616513096.055507:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.155@tcp1 routed destination 00000400:00000200:12.0:1616513096.055511:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.155@tcp1(10.31.136.155@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.055518:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:10.0:1616513096.055520:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.155@tcp1(10.31.136.155@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.055524:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.055526:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513096.055527:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513096.055528:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513096.055532:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.055533:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513096.055535:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513096.055537:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513096.055540:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513096.055541:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.055546:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513096.055579:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513096.055600:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.055607:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513096.055610:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513096.055612:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513096.055617:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513096.055621:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82e58 00000800:00000200:2.0:1616513096.055625:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513096.055638:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.055645:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513096.055647:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513096.055654:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513096.055659:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:0.0:1616513096.055663:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.055666:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513096.069643:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513096.069700:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513096.069709:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513096.069720:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.32@tcp1 : PUT - for me 00000400:00000200:20.0:1616513096.069729:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.32@tcp1 of length 224 into portal 26 MB=0x5faa0912017c0 00000400:00000200:20.0:1616513096.069736:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.32@tcp1 of length 224/224 into md 0xf97cf769 [1] + 0 00000400:00000200:20.0:1616513096.069741:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c770 00000400:00000200:20.0:1616513096.069744:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513096.069747:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513096.069751:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c770 00000800:00000200:20.0:1616513096.069754:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.069755:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513096.069757:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513096.069762:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513096.069765:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513096.069766:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513096.069774:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.32@tcp1 : PUT - for me 00000400:00000200:24.0:1616513096.069783:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.32@tcp1 of length 224 into portal 12 MB=0x5faa091201840 00000400:00000200:24.0:1616513096.069789:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.32@tcp1 of length 224/224 into md 0x865be291 [1] + 59000 00000400:00000200:24.0:1616513096.069796:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513096.069799:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513096.069805:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.069809:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513096.069810:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.069815:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682942420064192 00010000:00000200:12.0:1616513096.069827:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1682942420064192/t0(0) o400->2ad44288-9f89-1246-cc51-f3aab955947a@10.31.132.32@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.069841:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1682942420064192, offset 192 00000400:00000200:12.0:1616513096.069847:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.32@tcp1 00000400:00000200:12.0:1616513096.069854:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.32@tcp1 routed destination 00000400:00000200:12.0:1616513096.069869:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.32@tcp1(10.31.132.32@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513096.069870:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1682942420064320 00000800:00000200:12.0:1616513096.069876:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.069882:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:28.0:1616513096.069883:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cecc80 x1682942420064320/t0(0) o400->b903c161-7a2e-553a-e1ce-b1de11261aac@10.31.132.32@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513096.069884:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.069888:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.069893:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513096.069897:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1682942420064320, offset 192 00000800:00000200:11.2:1616513096.069898:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.069900:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513096.069903:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.32@tcp1 00000400:00000200:28.0:1616513096.069911:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.32@tcp1 routed destination 00000400:00000200:28.0:1616513096.069925:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.32@tcp1(10.31.132.32@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.069932:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.069938:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.069941:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.069943:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.069948:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513096.069954:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513096.069954:0:51172:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513096.069961:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513096.069964:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513096.069965:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513096.069970:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513096.069976:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fd38 00000800:00000200:16.0:1616513096.069980:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513096.070015:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.070023:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513096.070025:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513096.070032:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513096.070036:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd430 00000800:00000200:14.0:1616513096.070040:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.070043:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:29.0:1616513096.075026:0:6488:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929376, portal 4 00000100:00000200:29.0:1616513096.075032:0:6488:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929376, offset 0 00000400:00000200:29.0:1616513096.075039:0:6488:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:29.0:1616513096.075045:0:6488:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:29.0:1616513096.075058:0:6488:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513096.075065:0:6488:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:29.0:1616513096.075071:0:6488:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513096.075075:0:6488:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:29.0:1616513096.075077:0:6488:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:29.0:1616513096.075083:0:6488:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513096.075130:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513096.075187:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513096.075195:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513096.075198:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513096.075205:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513096.075212:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4558ad2d00 x1693067725929376/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513096.075224:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2770 00000800:00000200:26.0:1616513096.075228:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513096.075231:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513096.075307:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513096.075360:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513096.075369:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513096.075378:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513096.075387:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627ba0 00000400:00000200:14.0:1616513096.075394:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cff99 [1] + 192 00000400:00000200:14.0:1616513096.075399:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.075404:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4558ad2d00 x1693067725929376/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513096.075413:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513096.075419:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513096.075423:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513096.075424:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:29.0:1616513096.075429:0:6488:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4558ad2d00 x1693067725929376/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:29.0:1616513096.075470:0:6488:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4558ad2d00 x1693067725929376/t0(0) o13->boslfs02-OST003c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:29.0:1616513096.075480:0:6488:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b3760 00000100:00000200:9.0:1616513096.122674:0:6489:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929392, portal 4 00000100:00000200:9.0:1616513096.122681:0:6489:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929392, offset 0 00000400:00000200:9.0:1616513096.122687:0:6489:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:9.0:1616513096.122693:0:6489:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:9.0:1616513096.122707:0:6489:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000100:00000200:21.0:1616513096.122711:0:6490:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929408, portal 4 00000800:00000200:9.0:1616513096.122714:0:6489:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000100:00000200:21.0:1616513096.122717:0:6490:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929408, offset 0 00000800:00000200:9.0:1616513096.122720:0:6489:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:9.0:1616513096.122723:0:6489:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000400:00000200:21.0:1616513096.122724:0:6490:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000800:00000200:9.0:1616513096.122726:0:6489:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:21.0:1616513096.122730:0:6490:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:9.0:1616513096.122732:0:6489:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513096.122740:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:21.0:1616513096.122743:0:6490:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:21.0:1616513096.122749:0:6490:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:21.0:1616513096.122755:0:6490:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513096.122758:0:6490:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:21.0:1616513096.122761:0:6490:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:21.0:1616513096.122765:0:6490:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513096.122774:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:13.0:1616513096.122790:0:6491:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929424, portal 4 00000800:00000200:1.0:1616513096.122790:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000100:00000200:13.0:1616513096.122795:0:6491:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929424, offset 0 00000800:00000200:1.0:1616513096.122796:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000800:00000200:24.0:1616513096.122798:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:1.0:1616513096.122799:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:9.0:1616513096.122800:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:13.0:1616513096.122802:0:6491:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000800:00000200:24.0:1616513096.122806:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:1.0:1616513096.122806:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000400:00000200:13.0:1616513096.122808:0:6491:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:24.0:1616513096.122810:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:1.0:1616513096.122813:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843f7bde80 x1693067725929408/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513096.122818:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000400:00000200:13.0:1616513096.122822:0:6491:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000400:00000200:1.0:1616513096.122825:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf506e8 00000800:00000200:13.0:1616513096.122828:0:6491:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:1.0:1616513096.122830:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:24.0:1616513096.122832:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cab102400 x1693067725929392/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:13.0:1616513096.122835:0:6491:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:13.0:1616513096.122838:0:6491:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:13.0:1616513096.122840:0:6491:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000400:00000200:24.0:1616513096.122845:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca34aa0 00000800:00000200:13.0:1616513096.122845:0:6491:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:24.0:1616513096.122850:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:25.2:1616513096.122853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:24.0:1616513096.122853:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:26.0:1616513096.122911:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513096.122919:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513096.122922:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513096.122929:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513096.122936:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d843a2ede80 x1693067725929424/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513096.122948:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81783442a8 00000800:00000200:26.0:1616513096.122953:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513096.122956:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:5.2:1616513096.122968:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:1.2:1616513096.122981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513096.122990:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513096.122996:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:13.0:1616513096.122999:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513096.123005:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513096.123014:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627bc0 00000400:00000200:1.0:1616513096.123020:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700285 [1] + 192 00000800:00000200:20.0:1616513096.123022:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:1.0:1616513096.123026:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513096.123030:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843f7bde80 x1693067725929408/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513096.123032:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:20.0:1616513096.123042:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000800:00000200:1.0:1616513096.123047:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513096.123049:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:20.0:1616513096.123050:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627bb0 00000800:00000200:1.0:1616513096.123051:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:21.0:1616513096.123052:0:6490:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843f7bde80 x1693067725929408/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513096.123058:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dc751 [1] + 192 00000400:00000200:20.0:1616513096.123063:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:25.2:1616513096.123065:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000100:00000200:21.0:1616513096.123065:0:6490:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843f7bde80 x1693067725929408/t0(0) o13->boslfs02-OST0007-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:20.0:1616513096.123069:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cab102400 x1693067725929392/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:21.0:1616513096.123075:0:6490:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50c38 00000800:00000200:16.0:1616513096.123077:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513096.123084:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513096.123087:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513096.123089:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:9.0:1616513096.123093:0:6489:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cab102400 x1693067725929392/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:9.0:1616513096.123106:0:6489:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cab102400 x1693067725929392/t0(0) o13->boslfs02-OST0001-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:9.0:1616513096.123117:0:6489:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d848ca35430 00000800:00000200:18.0:1616513096.123120:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513096.123130:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:18.0:1616513096.123139:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513096.123148:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627bd0 00000400:00000200:18.0:1616513096.123155:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97cffa9 [1] + 192 00000400:00000200:18.0:1616513096.123160:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513096.123165:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d843a2ede80 x1693067725929424/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513096.123174:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513096.123181:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513096.123184:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513096.123186:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:13.0:1616513096.123189:0:6491:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d843a2ede80 x1693067725929424/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:13.0:1616513096.123202:0:6491:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d843a2ede80 x1693067725929424/t0(0) o13->boslfs02-OST0014-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:13.0:1616513096.123214:0:6491:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8178345078 00000100:00000200:29.0:1616513096.123482:0:6492:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929440, portal 4 00000100:00000200:29.0:1616513096.123487:0:6492:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929440, offset 0 00000400:00000200:29.0:1616513096.123494:0:6492:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:29.0:1616513096.123500:0:6492:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:29.0:1616513096.123513:0:6492:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:29.0:1616513096.123520:0:6492:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:29.0:1616513096.123526:0:6492:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513096.123529:0:6492:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:29.0:1616513096.123531:0:6492:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:29.0:1616513096.123537:0:6492:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513096.123583:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513096.123604:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513096.123611:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513096.123613:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513096.123620:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513096.123626:0:6476:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8302325100 x1693067725929440/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513096.123638:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b2d48 00000800:00000200:27.0:1616513096.123642:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513096.123647:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513096.123755:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513096.123764:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513096.123770:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513096.123773:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513096.123778:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513096.123785:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627be0 00000400:00000200:27.0:1616513096.123792:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716732d [1] + 192 00000400:00000200:27.0:1616513096.123797:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513096.123801:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8302325100 x1693067725929440/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513096.123814:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513096.123817:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513096.123819:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:29.0:1616513096.123822:0:6492:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8302325100 x1693067725929440/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:29.0:1616513096.123835:0:6492:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8302325100 x1693067725929440/t0(0) o13->boslfs02-OST0012-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:29.0:1616513096.123844:0:6492:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b3760 00000800:00000200:11.2:1616513096.129497:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513096.129553:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.129563:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513096.129572:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.77@tcp1 : PUT - for me 00000400:00000200:0.0:1616513096.129581:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.77@tcp1 of length 224 into portal 26 MB=0x5f90935c4c100 00000400:00000200:0.0:1616513096.129589:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.77@tcp1 of length 224/224 into md 0xf97cf7a9 [1] + 0 00000400:00000200:0.0:1616513096.129593:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922dba0 00000400:00000200:0.0:1616513096.129596:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513096.129599:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513096.129604:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922dba0 00000800:00000200:24.0:1616513096.129606:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.129607:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.129610:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513096.129613:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513096.129616:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513096.129617:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513096.129626:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.77@tcp1 : PUT - for me 00000400:00000200:24.0:1616513096.129634:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.77@tcp1 of length 224 into portal 12 MB=0x5f90935c4c180 00000400:00000200:24.0:1616513096.129641:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.77@tcp1 of length 224/224 into md 0x865be291 [1] + 59224 00000400:00000200:24.0:1616513096.129646:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513096.129649:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513096.129657:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.129660:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513096.129662:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.129669:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681192835662080 00010000:00000200:12.0:1616513096.129681:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1681192835662080/t0(0) o400->b401a576-56c2-732b-8c42-d29989cfe911@10.31.131.77@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.129695:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681192835662080, offset 192 00000400:00000200:12.0:1616513096.129700:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.77@tcp1 00000400:00000200:12.0:1616513096.129707:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.77@tcp1 routed destination 00000400:00000200:12.0:1616513096.129722:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.77@tcp1(10.31.131.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513096.129723:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681192835662208 00000800:00000200:12.0:1616513096.129728:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.129734:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.129737:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:28.0:1616513096.129738:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ced100 x1681192835662208/t0(0) o400->7f8693ee-07df-8e3a-39f9-226c6baf5ae4@10.31.131.77@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513096.129739:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.129744:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.129749:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.129751:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513096.129753:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681192835662208, offset 192 00000400:00000200:28.0:1616513096.129759:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.77@tcp1 00000400:00000200:28.0:1616513096.129766:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.77@tcp1 routed destination 00000400:00000200:28.0:1616513096.129782:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.77@tcp1(10.31.131.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.129789:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.129795:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.129800:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.129802:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513096.129805:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513096.129807:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513096.129813:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513096.129814:0:51178:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513096.129815:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513096.129816:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513096.129822:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513096.129828:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:20.0:1616513096.129831:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513096.129871:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.129878:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513096.129881:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513096.129890:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513096.129894:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd320 00000800:00000200:26.0:1616513096.129898:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.129901:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.206199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513096.206260:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.206269:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513096.206279:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.55@tcp1 : PUT - for me 00000400:00000200:24.0:1616513096.206288:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.55@tcp1 of length 224 into portal 26 MB=0x603e2aad70ac0 00000400:00000200:24.0:1616513096.206295:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.55@tcp1 of length 224/224 into md 0xf97cf7b9 [1] + 0 00000400:00000200:24.0:1616513096.206300:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bab28 00000400:00000200:24.0:1616513096.206303:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513096.206306:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:16.0:1616513096.206314:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513096.206316:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bab28 00000800:00000200:24.0:1616513096.206320:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513096.206323:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513096.206324:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513096.206325:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513096.206328:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513096.206333:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.55@tcp1 : PUT - for me 00000400:00000200:16.0:1616513096.206342:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.55@tcp1 of length 224 into portal 12 MB=0x603e2aad70c00 00000400:00000200:16.0:1616513096.206348:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.55@tcp1 of length 224/224 into md 0x865be291 [1] + 59448 00000400:00000200:16.0:1616513096.206354:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513096.206357:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513096.206365:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513096.206370:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.206372:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.206383:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693121923975872 00010000:00000200:12.0:1616513096.206395:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1693121923975872/t0(0) o400->68ca986a-2294-f0cb-6fd9-5d2c15b2f673@10.31.136.55@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.206410:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693121923975872, offset 192 00000400:00000200:12.0:1616513096.206416:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.55@tcp1 00000400:00000200:12.0:1616513096.206423:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.55@tcp1 routed destination 00000100:00000200:28.0:1616513096.206446:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693121923976192 00000400:00000200:12.0:1616513096.206453:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.55@tcp1(10.31.136.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:28.0:1616513096.206460:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b105a00 x1693121923976192/t0(0) o400->e39b0cba-f814-015c-7e3d-76bcb0cb22b3@10.31.136.55@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513096.206460:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.206466:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.206468:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.206471:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:28.0:1616513096.206474:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693121923976192, offset 192 00000800:00000200:12.0:1616513096.206476:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513096.206481:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.55@tcp1 00000800:00000200:11.2:1616513096.206481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.206484:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513096.206488:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.55@tcp1 routed destination 00000400:00000200:28.0:1616513096.206503:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.55@tcp1(10.31.136.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.206510:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.206516:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.206519:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.206521:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.206526:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.206533:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513096.206535:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513096.206543:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513096.206545:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513096.206547:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513096.206552:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513096.206559:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:20.0:1616513096.206563:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513096.206588:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.206596:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513096.206598:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513096.206605:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513096.206609:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd100 00000800:00000200:14.0:1616513096.206613:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.206616:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.206622:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513096.206624:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.206627:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.206628:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.206630:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.208847:0:51186:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513096.208902:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513096.208912:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513096.208920:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.173@tcp1 : PUT - for me 00000400:00000200:2.0:1616513096.208929:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.173@tcp1 of length 224 into portal 26 MB=0x60198a41b7a40 00000400:00000200:2.0:1616513096.208935:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.173@tcp1 of length 224/224 into md 0xf97cf7e9 [1] + 0 00000400:00000200:2.0:1616513096.208940:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba000 00000400:00000200:2.0:1616513096.208943:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513096.208945:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513096.208949:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba000 00000800:00000200:2.0:1616513096.208952:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.208955:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513096.208957:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.208960:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.208970:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.208979:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.173@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.208987:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.173@tcp1 of length 224 into portal 12 MB=0x60198a41b7b80 00000400:00000200:14.0:1616513096.208994:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.173@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 26320 00000400:00000200:14.0:1616513096.208999:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.209002:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513096.209010:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:12.0:1616513096.209012:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690604960184896 00000800:00000200:14.0:1616513096.209015:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.209017:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.209019:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:12.0:1616513096.209023:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1690604960184896/t0(0) o400->cfbbb5b6-7d69-7b6b-4834-c68789b3c395@10.31.141.173@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513096.209032:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690604960185216 00000100:00000200:12.0:1616513096.209039:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690604960184896, offset 192 00010000:00000200:10.0:1616513096.209044:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bff080 x1690604960185216/t0(0) o400->725147e0-6daf-f1f7-77e6-e8a417a1f595@10.31.141.173@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513096.209045:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.173@tcp1 00000400:00000200:12.0:1616513096.209052:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.173@tcp1 routed destination 00000100:00000200:10.0:1616513096.209059:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690604960185216, offset 192 00000400:00000200:10.0:1616513096.209065:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.173@tcp1 00000400:00000200:12.0:1616513096.209067:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.173@tcp1(10.31.141.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513096.209073:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.173@tcp1 routed destination 00000800:00000200:12.0:1616513096.209074:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.209079:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.209082:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.209084:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:10.0:1616513096.209088:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.173@tcp1(10.31.141.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.209089:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513096.209094:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:12.0:1616513096.209095:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513096.209100:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513096.209103:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513096.209105:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513096.209110:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.209117:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.209130:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513096.209135:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513096.209137:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513096.209139:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513096.209141:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513096.209146:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513096.209150:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82908 00000800:00000200:26.0:1616513096.209154:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513096.209192:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513096.209199:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513096.209202:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513096.209209:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513096.209214:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:16.0:1616513096.209217:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.209220:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513096.231114:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513096.231171:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.231181:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.231190:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.105@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.231199:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.105@tcp1 of length 224 into portal 26 MB=0x5ffdbffed9f00 00000400:00000200:14.0:1616513096.231206:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.105@tcp1 of length 224/224 into md 0xf97cf7f9 [1] + 0 00000400:00000200:14.0:1616513096.231211:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bae58 00000400:00000200:14.0:1616513096.231214:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.231217:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513096.231221:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bae58 00000800:00000200:14.0:1616513096.231225:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.231227:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.231227:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.231234:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.231237:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.231238:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513096.231246:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.105@tcp1 : PUT - for me 00000400:00000200:18.0:1616513096.231255:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.105@tcp1 of length 224 into portal 12 MB=0x5ffdbffeda200 00000400:00000200:18.0:1616513096.231262:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.105@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 26544 00000400:00000200:18.0:1616513096.231267:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513096.231270:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513096.231277:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.231280:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.231282:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.231286:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695240236800 00000100:00000200:10.0:1616513096.231288:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695240237568 00010000:00000200:10.0:1616513096.231294:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464626300 x1688695240237568/t0(0) o400->1f5924ef-6390-c244-eef9-f03b9f8c771e@10.31.141.105@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:12.0:1616513096.231297:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688695240236800/t0(0) o400->0bac2642-f8bf-405f-4785-66c46b1bc422@10.31.141.105@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513096.231304:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695240237568, offset 192 00000400:00000200:10.0:1616513096.231308:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.105@tcp1 00000100:00000200:12.0:1616513096.231311:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695240236800, offset 192 00000400:00000200:10.0:1616513096.231314:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.105@tcp1 routed destination 00000400:00000200:12.0:1616513096.231317:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.105@tcp1 00000400:00000200:10.0:1616513096.231326:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.105@tcp1(10.31.141.105@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513096.231331:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513096.231335:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.105@tcp1 routed destination 00000800:00000200:10.0:1616513096.231335:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513096.231338:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513096.231339:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513096.231342:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513096.231350:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.105@tcp1(10.31.141.105@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513096.231350:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.231356:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513096.231362:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.231365:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513096.231367:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513096.231372:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.231377:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513096.231379:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513096.231407:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.231415:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513096.231418:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513096.231424:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513096.231428:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82908 00000800:00000200:24.0:1616513096.231432:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513096.231448:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.231455:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513096.231458:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513096.231459:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513096.231460:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513096.231465:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513096.231470:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e4c8 00000800:00000200:26.0:1616513096.231474:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.244632:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513096.244688:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513096.244698:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513096.244707:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.120@tcp1 : PUT - for me 00000400:00000200:16.0:1616513096.244716:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.120@tcp1 of length 224 into portal 26 MB=0x6001dc4b22700 00000400:00000200:16.0:1616513096.244723:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.120@tcp1 of length 224/224 into md 0xf97cf819 [1] + 0 00000400:00000200:16.0:1616513096.244728:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6babb0 00000400:00000200:16.0:1616513096.244730:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513096.244733:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513096.244738:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6babb0 00000800:00000200:16.0:1616513096.244741:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.244742:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.244743:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513096.244747:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.244800:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688977714325248 00010000:00000200:12.0:1616513096.244810:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688977714325248/t0(0) o400->2d429ab3-9275-f1c9-6273-12b2168f8c5a@10.31.142.120@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.244823:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688977714325248, offset 192 00000400:00000200:12.0:1616513096.244829:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.120@tcp1 00000400:00000200:12.0:1616513096.244836:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.120@tcp1 routed destination 00000400:00000200:12.0:1616513096.244851:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.120@tcp1(10.31.142.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.244857:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.244863:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.244865:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.244867:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.244872:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.244877:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.244878:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513096.244930:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513096.244937:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513096.244940:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513096.244946:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513096.244952:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eaa0 00000800:00000200:20.0:1616513096.244955:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513096.244958:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513096.245375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513096.245386:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.245395:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513096.245396:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513096.245404:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.120@tcp1 : PUT - for me 00000400:00000200:0.0:1616513096.245414:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.120@tcp1 of length 224 into portal 12 MB=0x6001dc4b22c40 00000400:00000200:0.0:1616513096.245421:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.120@tcp1 of length 224/224 into md 0x865be291 [1] + 59672 00000400:00000200:0.0:1616513096.245426:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513096.245429:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513096.245450:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.245453:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.245455:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513096.245503:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688977714326592 00010000:00000200:28.0:1616513096.245516:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde869200 x1688977714326592/t0(0) o400->0fafc198-91c3-a745-9311-7221f9c04cb2@10.31.142.120@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513096.245530:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688977714326592, offset 192 00000400:00000200:28.0:1616513096.245536:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.120@tcp1 00000400:00000200:28.0:1616513096.245543:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.120@tcp1 routed destination 00000400:00000200:28.0:1616513096.245558:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.120@tcp1(10.31.142.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.245564:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.245570:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.245573:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.245575:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.245580:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.245587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.245643:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.245650:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.245653:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.245660:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.245664:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccff0 00000800:00000200:18.0:1616513096.245668:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.245671:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.270684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513096.270741:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.270751:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.270760:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.104@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.270770:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.104@tcp1 of length 224 into portal 26 MB=0x6001d2f8256c0 00000400:00000200:14.0:1616513096.270777:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.104@tcp1 of length 224/224 into md 0xf97cf829 [1] + 0 00000400:00000200:14.0:1616513096.270781:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba7f8 00000400:00000200:14.0:1616513096.270784:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.270787:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513096.270791:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba7f8 00000800:00000200:14.0:1616513096.270794:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.270796:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.270797:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.270803:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.270806:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.270807:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.270815:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.104@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.270824:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.104@tcp1 of length 224 into portal 12 MB=0x6001d2f825740 00000400:00000200:26.0:1616513096.270831:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.104@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 26768 00000400:00000200:26.0:1616513096.270836:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.270839:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513096.270846:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.270862:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.270864:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.270868:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975211386560 00010000:00000200:12.0:1616513096.270879:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688975211386560/t0(0) o400->ad487c92-73cb-631d-30c2-8c04eaf0c89f@10.31.142.104@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513096.270881:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975211386688 00010000:00000200:10.0:1616513096.270891:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c696300 x1688975211386688/t0(0) o400->38e4c04f-51d2-5473-c668-b9d794022cc1@10.31.142.104@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513096.270893:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688975211386560, offset 192 00000400:00000200:12.0:1616513096.270899:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.104@tcp1 00000100:00000200:10.0:1616513096.270903:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688975211386688, offset 192 00000400:00000200:12.0:1616513096.270907:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.104@tcp1 routed destination 00000400:00000200:10.0:1616513096.270908:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.104@tcp1 00000400:00000200:10.0:1616513096.270914:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.104@tcp1 routed destination 00000400:00000200:12.0:1616513096.270922:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.104@tcp1(10.31.142.104@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513096.270926:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.104@tcp1(10.31.142.104@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.270929:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513096.270932:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513096.270935:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513096.270937:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.270938:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513096.270939:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513096.270940:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513096.270941:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513096.270944:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513096.270945:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513096.270951:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513096.270952:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513096.270954:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513096.271005:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.271012:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.271015:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513096.271015:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513096.271022:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:20.0:1616513096.271023:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513096.271028:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82e58 00000800:00000200:20.0:1616513096.271030:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513096.271032:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513096.271033:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513096.271040:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513096.271045:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f6d8 00000800:00000200:20.0:1616513096.271049:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513096.271052:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513096.284268:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513096.284324:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.284334:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.284343:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.16@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.284352:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.16@tcp1 of length 224 into portal 26 MB=0x60393e050ec80 00000400:00000200:26.0:1616513096.284359:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.16@tcp1 of length 224/224 into md 0xf97cf839 [1] + 0 00000400:00000200:26.0:1616513096.284363:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb980 00000400:00000200:26.0:1616513096.284366:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.284371:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513096.284376:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb980 00000800:00000200:18.0:1616513096.284378:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.284379:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.284382:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.284384:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.284451:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:12.0:1616513096.284454:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783518739584 00010000:00000200:12.0:1616513096.284465:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1692783518739584/t0(0) o400->0496c335-08df-09df-7c45-3456b406c5cf@10.31.140.16@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.284479:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783518739584, offset 192 00000400:00000200:12.0:1616513096.284485:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.16@tcp1 00000400:00000200:12.0:1616513096.284493:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.16@tcp1 routed destination 00000400:00000200:12.0:1616513096.284508:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.16@tcp1(10.31.140.16@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.284515:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513096.284519:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.284521:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.284525:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513096.284527:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513096.284529:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.284532:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:16.0:1616513096.284539:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.16@tcp1 : PUT - for me 00000100:00000200:12.0:1616513096.284539:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513096.284549:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.16@tcp1 of length 224 into portal 12 MB=0x60393e050ed00 00000400:00000200:16.0:1616513096.284555:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.16@tcp1 of length 224/224 into md 0x865be291 [1] + 59896 00000400:00000200:16.0:1616513096.284560:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513096.284563:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513096.284570:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513096.284573:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513096.284574:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:16.0:1616513096.284581:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513096.284582:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513096.284584:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513096.284585:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513096.284592:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513096.284598:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e220 00000800:00000200:0.0:1616513096.284601:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513096.284662:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783518739712 00010000:00000200:28.0:1616513096.284674:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b104380 x1692783518739712/t0(0) o400->cac226bd-6922-7933-5c39-d83a53f5f2f5@10.31.140.16@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513096.284699:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783518739712, offset 192 00000400:00000200:28.0:1616513096.284706:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.16@tcp1 00000400:00000200:28.0:1616513096.284713:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.16@tcp1 routed destination 00000400:00000200:28.0:1616513096.284729:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.16@tcp1(10.31.140.16@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.284735:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.284741:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.284744:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.284746:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.284751:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.284757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.284810:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.284817:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513096.284820:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513096.284829:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513096.284833:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccee0 00000800:00000200:14.0:1616513096.284837:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.284839:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.345581:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513096.345627:0:51199:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513096.345637:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.345647:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513096.345657:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.11@tcp1 : PUT - for me 00000400:00000200:24.0:1616513096.345669:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.11@tcp1 of length 224 into portal 26 MB=0x603d0eaf21d40 00000400:00000200:24.0:1616513096.345676:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.21.11@tcp1 of length 224/224 into md 0xf97cf859 [1] + 0 00000400:00000200:24.0:1616513096.345681:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb650 00000800:00000200:18.0:1616513096.345683:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513096.345684:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513096.345687:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513096.345692:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb650 00000800:00000200:18.0:1616513096.345693:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513096.345695:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.345698:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513096.345699:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513096.345703:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513096.345703:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.11@tcp1 : PUT - for me 00000400:00000200:18.0:1616513096.345714:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.11@tcp1 of length 224 into portal 12 MB=0x603d0eaf221c0 00000400:00000200:18.0:1616513096.345720:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.21.11@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 26992 00000400:00000200:18.0:1616513096.345726:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513096.345729:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513096.345736:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.345738:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.345739:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513096.345742:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.345780:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693045690080576 00010000:00000200:12.0:1616513096.345793:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1693045690080576/t0(0) o400->ec40b684-b04b-5b2b-6eb1-78e53f711d90@10.31.21.11@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513096.345805:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693045690081728 00000100:00000200:12.0:1616513096.345808:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693045690080576, offset 192 00000400:00000200:12.0:1616513096.345814:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.11@tcp1 00010000:00000200:10.0:1616513096.345819:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464624800 x1693045690081728/t0(0) o400->818c9703-1177-c410-c401-4735e4ab1220@10.31.21.11@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513096.345822:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.11@tcp1 routed destination 00000100:00000200:10.0:1616513096.345835:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693045690081728, offset 192 00000400:00000200:12.0:1616513096.345837:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.11@tcp1(10.31.21.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.345846:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:10.0:1616513096.345846:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.11@tcp1 00000800:00000200:12.0:1616513096.345852:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.345854:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:10.0:1616513096.345854:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.11@tcp1 routed destination 00000800:00000200:12.0:1616513096.345857:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.345862:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.345868:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.345869:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:10.0:1616513096.345869:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.11@tcp1(10.31.21.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513096.345875:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513096.345882:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513096.345884:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513096.345889:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513096.345894:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.345899:0:51199:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.345911:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.345919:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513096.345922:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513096.345923:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513096.345928:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513096.345931:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513096.345932:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82aa0 00000400:00000200:0.0:1616513096.345933:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513096.345937:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.345940:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513096.345940:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513096.345946:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ed48 00000800:00000200:0.0:1616513096.345949:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.345952:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:3.0:1616513096.346660:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929456, portal 4 00000100:00000200:3.0:1616513096.346667:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929456, offset 0 00000400:00000200:3.0:1616513096.346675:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:3.0:1616513096.346681:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:3.0:1616513096.346695:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513096.346702:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:3.0:1616513096.346709:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513096.346712:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:3.0:1616513096.346715:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:3.0:1616513096.346721:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513096.346769:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513096.346791:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513096.346797:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513096.346800:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513096.346807:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513096.346814:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456e9200 x1693067725929456/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513096.346826:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd3078 00000800:00000200:1.0:1616513096.346831:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513096.346833:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513096.346949:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513096.346958:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513096.346964:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513096.346967:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513096.346972:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513096.346981:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627bf0 00000400:00000200:1.0:1616513096.346987:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x437002c5 [1] + 192 00000400:00000200:1.0:1616513096.346993:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513096.346997:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456e9200 x1693067725929456/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513096.347011:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513096.347014:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513096.347016:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:3.0:1616513096.347020:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456e9200 x1693067725929456/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513096.347033:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456e9200 x1693067725929456/t0(0) o13->boslfs02-OST003f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513096.347043:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000800:00000200:7.2:1616513096.348785:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513096.348840:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.348850:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.348859:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.56@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.348868:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.56@tcp1 of length 224 into portal 26 MB=0x5f90c85b4cf40 00000400:00000200:26.0:1616513096.348875:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.56@tcp1 of length 224/224 into md 0xf97cf879 [1] + 0 00000400:00000200:26.0:1616513096.348879:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba990 00000400:00000200:26.0:1616513096.348882:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.348885:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513096.348888:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba990 00000800:00000200:26.0:1616513096.348892:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.348894:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513096.348894:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.348901:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.348903:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.348904:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513096.348913:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.56@tcp1 : PUT - for me 00000400:00000200:18.0:1616513096.348922:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.56@tcp1 of length 224 into portal 12 MB=0x5f90c85b4cfc0 00000400:00000200:18.0:1616513096.348928:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.56@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 27216 00000400:00000200:18.0:1616513096.348933:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513096.348936:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513096.348942:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.348945:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.348947:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.348951:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681207061696320 00010000:00000200:12.0:1616513096.348962:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1681207061696320/t0(0) o400->06decc27-24fd-7d8b-17c1-049473294dce@10.31.131.56@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.348976:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681207061696320, offset 192 00000400:00000200:12.0:1616513096.348982:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.56@tcp1 00000400:00000200:12.0:1616513096.348990:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.56@tcp1 routed destination 00000400:00000200:12.0:1616513096.349005:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.56@tcp1(10.31.131.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513096.349005:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681207061696448 00000800:00000200:12.0:1616513096.349011:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.349017:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:10.0:1616513096.349017:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623a80 x1681207061696448/t0(0) o400->725553bd-5325-7b93-62c8-c1c3d6180426@10.31.131.56@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513096.349020:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.349022:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.349026:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:10.0:1616513096.349031:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681207061696448, offset 192 00000800:00000200:11.2:1616513096.349032:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.349033:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:10.0:1616513096.349038:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.56@tcp1 00000400:00000200:10.0:1616513096.349045:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.56@tcp1 routed destination 00000400:00000200:10.0:1616513096.349060:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.56@tcp1(10.31.131.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513096.349066:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513096.349072:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513096.349075:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513096.349077:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513096.349082:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513096.349086:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513096.349087:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513096.349093:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513096.349095:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513096.349102:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513096.349108:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777edd0 00000800:00000200:16.0:1616513096.349111:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.349114:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513096.349141:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.349148:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513096.349151:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513096.349152:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513096.349158:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513096.349162:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83f58 00000800:00000200:2.0:1616513096.349166:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.349895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513096.349904:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.349910:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513096.349914:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513096.349917:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.246@tcp1 : PUT - for me 00000400:00000200:18.0:1616513096.349924:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.246@tcp1 of length 224 into portal 26 MB=0x605288a2f4580 00000800:00000200:14.0:1616513096.349924:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513096.349925:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.349930:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.246@tcp1 of length 224/224 into md 0xf97cf8c9 [1] + 0 00000400:00000200:14.0:1616513096.349933:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.246@tcp1 : PUT - for me 00000400:00000200:18.0:1616513096.349934:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba4c8 00000400:00000200:18.0:1616513096.349936:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513096.349939:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513096.349942:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba4c8 00000400:00000200:14.0:1616513096.349942:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.246@tcp1 of length 224 into portal 12 MB=0x605288a2f4640 00000800:00000200:18.0:1616513096.349945:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.349947:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513096.349948:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.246@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 27440 00000800:00000200:18.0:1616513096.349949:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513096.349953:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513096.349955:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694521535448448 00000100:00000200:14.0:1616513096.349956:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:12.0:1616513096.349961:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1694521535448448/t0(0) o400->dc8f1cc4-099b-ac64-3a5d-b02e7f8b08c7@10.31.140.246@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513096.349963:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.349966:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.349968:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.349972:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694521535448448, offset 192 00000100:00000200:10.0:1616513096.349975:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694521535448640 00000400:00000200:12.0:1616513096.349977:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.246@tcp1 00010000:00000200:10.0:1616513096.349982:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf8900 x1694521535448640/t0(0) o400->34d8b736-f9cf-2485-95e3-a5c469b0b736@10.31.140.246@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513096.349983:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.246@tcp1 routed destination 00000100:00000200:10.0:1616513096.349993:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694521535448640, offset 192 00000400:00000200:12.0:1616513096.349996:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.246@tcp1(10.31.140.246@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:10.0:1616513096.349998:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.246@tcp1 00000800:00000200:12.0:1616513096.350002:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:10.0:1616513096.350004:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.246@tcp1 routed destination 00000800:00000200:12.0:1616513096.350007:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.350009:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.350011:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.350014:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513096.350017:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.246@tcp1(10.31.140.246@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:12.0:1616513096.350019:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513096.350020:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513096.350023:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513096.350028:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513096.350030:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513096.350032:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513096.350035:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.350039:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513096.350049:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.350052:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513096.350054:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513096.350056:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513096.350060:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513096.350062:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82f68 00000800:00000200:2.0:1616513096.350065:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513096.350076:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.350083:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513096.350086:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513096.350093:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513096.350098:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f5c8 00000800:00000200:0.0:1616513096.350102:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.350105:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513096.359384:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513096.359398:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.359407:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513096.359416:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.210@tcp1 : PUT - for me 00000400:00000200:18.0:1616513096.359425:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.210@tcp1 of length 224 into portal 26 MB=0x5ffe12d4c23c0 00000400:00000200:18.0:1616513096.359432:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.210@tcp1 of length 224/224 into md 0xf97cf999 [1] + 0 00000400:00000200:18.0:1616513096.359443:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba440 00000400:00000200:18.0:1616513096.359446:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513096.359449:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513096.359451:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513096.359453:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba440 00000800:00000200:18.0:1616513096.359457:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513096.359459:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513096.359461:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.359462:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513096.359464:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513096.359471:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.210@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.359479:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.210@tcp1 of length 224 into portal 12 MB=0x5ffe12d4c2740 00000400:00000200:14.0:1616513096.359485:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.210@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 27664 00000400:00000200:14.0:1616513096.359491:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.359493:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513096.359502:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.359505:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.359507:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:18.0:1616513096.359522:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717476243264 00000100:00000200:12.0:1616513096.359526:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688717476242368 00010000:00000200:18.0:1616513096.359533:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfde80 x1688717476243264/t0(0) o400->a01b37fc-8be2-bd78-a327-0e8d54fdde93@10.31.142.210@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:12.0:1616513096.359536:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688717476242368/t0(0) o400->ac37609c-27a7-e22d-84f1-51628cc4c640@10.31.142.210@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:18.0:1616513096.359546:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688717476243264, offset 192 00000100:00000200:12.0:1616513096.359550:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688717476242368, offset 192 00000400:00000200:18.0:1616513096.359552:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.210@tcp1 00000400:00000200:12.0:1616513096.359557:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.210@tcp1 00000400:00000200:18.0:1616513096.359558:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.210@tcp1 routed destination 00000400:00000200:12.0:1616513096.359564:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.210@tcp1 routed destination 00000400:00000200:18.0:1616513096.359580:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.210@tcp1(10.31.142.210@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513096.359580:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.210@tcp1(10.31.142.210@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513096.359586:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.359587:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513096.359592:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.359592:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513096.359594:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.359595:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513096.359596:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.359597:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.359600:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513096.359602:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513096.359607:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513096.359607:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513096.359609:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513096.359660:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.359667:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513096.359670:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513096.359671:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513096.359676:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513096.359679:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513096.359682:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:16.0:1616513096.359686:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513096.359686:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513096.359688:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513096.359695:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513096.359699:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc550 00000800:00000200:16.0:1616513096.359703:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.359706:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513096.359712:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513096.359714:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513096.359716:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513096.359717:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.359720:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513096.361998:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513096.362053:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513096.362062:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513096.362071:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.222@tcp1 : PUT - for me 00000400:00000200:20.0:1616513096.362080:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.222@tcp1 of length 224 into portal 12 MB=0x5ffe5f00abc40 00000400:00000200:20.0:1616513096.362086:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.222@tcp1 of length 224/224 into md 0x865be291 [1] + 60120 00000400:00000200:20.0:1616513096.362091:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513096.362094:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513096.362101:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513096.362104:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513096.362106:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513096.362110:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513096.362165:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737923382336 00010000:00000200:28.0:1616513096.362178:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eb600 x1688737923382336/t0(0) o400->3fd0be8b-9b7d-face-051c-22095d31e46b@10.31.141.222@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513096.362192:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688737923382336, offset 192 00000400:00000200:28.0:1616513096.362199:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.222@tcp1 00000400:00000200:28.0:1616513096.362206:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.222@tcp1 routed destination 00000400:00000200:28.0:1616513096.362221:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.222@tcp1(10.31.141.222@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.362227:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513096.362233:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.362235:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513096.362237:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513096.362242:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.362247:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.362300:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.362307:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513096.362310:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513096.362312:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513096.362316:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513096.362320:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccdd0 00000800:00000200:24.0:1616513096.362324:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513096.364081:0:51201:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513096.364135:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.364145:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.364156:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.222@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.364165:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.222@tcp1 of length 224 into portal 26 MB=0x5ffe5f00aba80 00000400:00000200:14.0:1616513096.364171:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.222@tcp1 of length 224/224 into md 0xf97cf9a9 [1] + 0 00000400:00000200:14.0:1616513096.364176:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb188 00000400:00000200:14.0:1616513096.364179:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.364181:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513096.364185:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb188 00000800:00000200:26.0:1616513096.364189:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.364189:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.364191:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.364193:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.364249:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737923381888 00010000:00000200:12.0:1616513096.364260:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688737923381888/t0(0) o400->9d6a268d-70f0-9ec2-2861-fe50fdf38fd3@10.31.141.222@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.364274:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688737923381888, offset 192 00000400:00000200:12.0:1616513096.364283:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.222@tcp1 00000400:00000200:12.0:1616513096.364290:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.222@tcp1 routed destination 00000400:00000200:12.0:1616513096.364305:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.222@tcp1(10.31.141.222@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.364311:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513096.364317:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.364320:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513096.364322:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513096.364337:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.364341:0:51201:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513096.364343:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513096.364394:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.364400:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.364403:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.364409:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.364414:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f100 00000800:00000200:18.0:1616513096.364417:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.364422:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.369425:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513096.369482:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.369491:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.369500:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.129@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.369509:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.129@tcp1 of length 224 into portal 26 MB=0x603fc0bf29480 00000400:00000200:26.0:1616513096.369516:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.129@tcp1 of length 224/224 into md 0xf97cf9c9 [1] + 0 00000400:00000200:26.0:1616513096.369520:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb5c8 00000400:00000200:26.0:1616513096.369523:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.369526:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513096.369530:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb5c8 00000800:00000200:26.0:1616513096.369533:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.369535:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.369537:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.369541:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.369545:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.369545:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.369554:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.129@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.369563:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.129@tcp1 of length 224 into portal 12 MB=0x603fc0bf29500 00000400:00000200:14.0:1616513096.369569:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.129@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 27888 00000400:00000200:14.0:1616513096.369574:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.369576:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513096.369584:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.369587:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.369589:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.369593:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230927352960 00010000:00000200:12.0:1616513096.369604:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1693230927352960/t0(0) o400->bf7ee353-6557-b743-23fa-219e8e8a2b62@10.31.138.129@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.369618:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693230927352960, offset 192 00000400:00000200:12.0:1616513096.369624:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.129@tcp1 00000400:00000200:12.0:1616513096.369631:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.129@tcp1 routed destination 00000400:00000200:12.0:1616513096.369646:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.129@tcp1(10.31.138.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513096.369650:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693230927353088 00000800:00000200:12.0:1616513096.369652:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.369657:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.369660:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:22.0:1616513096.369662:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfbf00 x1693230927353088/t0(0) o400->9ef2b2b6-1937-131f-c978-edecb69a4fdc@10.31.138.129@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513096.369662:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.369667:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513096.369673:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513096.369673:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513096.369677:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693230927353088, offset 192 00000400:00000200:22.0:1616513096.369683:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.129@tcp1 00000400:00000200:22.0:1616513096.369691:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.129@tcp1 routed destination 00000400:00000200:22.0:1616513096.369706:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.129@tcp1(10.31.138.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513096.369712:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513096.369718:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513096.369721:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513096.369723:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513096.369727:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513096.369728:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513096.369732:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513096.369735:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513096.369738:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513096.369744:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513096.369750:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f3a8 00000800:00000200:16.0:1616513096.369753:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.369756:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513096.369802:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.369819:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.369820:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513096.369822:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513096.369828:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513096.369832:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aee0 00000800:00000200:2.0:1616513096.369835:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.414921:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513096.414999:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.415009:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513096.415019:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.91@tcp1 : PUT - for me 00000400:00000200:0.0:1616513096.415030:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.91@tcp1 of length 224 into portal 26 MB=0x602d30b4484c0 00000400:00000200:0.0:1616513096.415037:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.91@tcp1 of length 224/224 into md 0xf97cf9d9 [1] + 0 00000400:00000200:0.0:1616513096.415044:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2d48 00000400:00000200:0.0:1616513096.415047:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513096.415050:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513096.415054:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513096.415055:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2d48 00000800:00000200:0.0:1616513096.415059:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513096.415062:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513096.415064:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513096.415066:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513096.415067:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513096.415075:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.91@tcp1 : PUT - for me 00000400:00000200:24.0:1616513096.415084:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.91@tcp1 of length 224 into portal 12 MB=0x602d30b448540 00000400:00000200:24.0:1616513096.415093:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.91@tcp1 of length 224/224 into md 0x865be291 [1] + 60344 00000400:00000200:24.0:1616513096.415099:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513096.415101:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513096.415109:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.415112:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513096.415114:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.415134:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691955310658752 00010000:00000200:12.0:1616513096.415146:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1691955310658752/t0(0) o400->2c4321b6-f028-873d-c5c5-6cc493f0f475@10.31.140.91@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.415161:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691955310658752, offset 192 00000400:00000200:12.0:1616513096.415168:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.91@tcp1 00000400:00000200:12.0:1616513096.415175:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.91@tcp1 routed destination 00000100:00000200:28.0:1616513096.415176:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691955310658880 00010000:00000200:28.0:1616513096.415190:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cea400 x1691955310658880/t0(0) o400->35de64a1-b486-0b4f-4367-e2502302382c@10.31.140.91@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513096.415191:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.91@tcp1(10.31.140.91@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513096.415205:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691955310658880, offset 192 00000800:00000200:12.0:1616513096.415206:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513096.415212:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.91@tcp1 00000800:00000200:12.0:1616513096.415213:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.415215:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.415218:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513096.415220:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.91@tcp1 routed destination 00000800:00000200:12.0:1616513096.415223:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.415228:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.415230:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513096.415235:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.91@tcp1(10.31.140.91@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.415243:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.415250:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.415253:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.415255:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.415260:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.415267:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513096.415283:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513096.415284:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513096.415291:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513096.415291:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513096.415293:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513096.415294:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513096.415294:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513096.415300:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:14.0:1616513096.415300:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513096.415306:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:20.0:1616513096.415310:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513096.415310:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcccc0 00000800:00000200:14.0:1616513096.415313:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.415316:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.415788:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513096.415798:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.415803:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513096.415807:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513096.415811:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.188@tcp1 : PUT - for me 00000400:00000200:24.0:1616513096.415818:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.188@tcp1 of length 224 into portal 26 MB=0x6058f2cbf95c0 00000400:00000200:24.0:1616513096.415824:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.188@tcp1 of length 224/224 into md 0xf97cf9f9 [1] + 0 00000400:00000200:24.0:1616513096.415828:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2a18 00000400:00000200:24.0:1616513096.415830:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513096.415833:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513096.415837:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2a18 00000800:00000200:24.0:1616513096.415840:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.415842:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513096.415843:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.415850:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694962349479360 00010000:00000200:12.0:1616513096.415856:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1694962349479360/t0(0) o400->4cd7a0c0-535b-5dbe-9fba-28790020b0b5@10.31.129.188@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.415867:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694962349479360, offset 192 00000400:00000200:12.0:1616513096.415873:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.188@tcp1 00000400:00000200:12.0:1616513096.415879:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.188@tcp1 routed destination 00000400:00000200:12.0:1616513096.415892:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.188@tcp1(10.31.129.188@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.415898:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.415902:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.415905:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.415906:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.415909:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513096.415913:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513096.415914:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.415924:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.415928:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513096.415930:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513096.415933:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513096.415936:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513096.415939:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513096.415940:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513096.415941:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ee58 00000800:00000200:0.0:1616513096.415943:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513096.415948:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.188@tcp1 : PUT - for me 00000400:00000200:20.0:1616513096.415957:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.188@tcp1 of length 224 into portal 12 MB=0x6058f2cbf9640 00000400:00000200:20.0:1616513096.415963:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.188@tcp1 of length 224/224 into md 0x865be291 [1] + 60568 00000400:00000200:20.0:1616513096.415968:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513096.415970:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513096.415977:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513096.415980:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513096.415981:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513096.415990:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694962349479488 00010000:00000200:28.0:1616513096.415997:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0eda00 x1694962349479488/t0(0) o400->e7446ee0-1789-6315-b20c-cca96b84d484@10.31.129.188@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513096.416008:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694962349479488, offset 192 00000400:00000200:28.0:1616513096.416014:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.188@tcp1 00000400:00000200:28.0:1616513096.416020:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.188@tcp1 routed destination 00000400:00000200:28.0:1616513096.416033:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.188@tcp1(10.31.129.188@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.416039:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.416043:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.416046:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.416047:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.416050:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.416056:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.416110:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.416117:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513096.416119:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513096.416126:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513096.416130:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccbb0 00000800:00000200:26.0:1616513096.416134:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.416137:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.509406:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513096.509464:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.509474:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.509484:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.199@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.509495:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.199@tcp1 of length 224 into portal 26 MB=0x60182d9bbe540 00000400:00000200:14.0:1616513096.509502:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.199@tcp1 of length 224/224 into md 0xf97cfa19 [1] + 0 00000400:00000200:14.0:1616513096.509509:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3ba0 00000400:00000200:14.0:1616513096.509512:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.509515:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513096.509519:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ba0 00000800:00000200:18.0:1616513096.509520:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.509523:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.509527:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.509528:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.509589:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511370609984 00010000:00000200:12.0:1616513096.509602:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1690511370609984/t0(0) o400->03d9fb55-c574-8f08-be7a-df83b0425227@10.31.140.199@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.509617:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511370609984, offset 192 00000400:00000200:12.0:1616513096.509623:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.199@tcp1 00000400:00000200:12.0:1616513096.509631:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.199@tcp1 routed destination 00000400:00000200:12.0:1616513096.509647:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.199@tcp1(10.31.140.199@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.509654:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.509660:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.509663:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.509667:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.509672:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513096.509680:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513096.509681:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513096.509697:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513096.509738:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.509745:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513096.509748:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513096.509754:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513096.509757:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513096.509761:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e5d8 00000800:00000200:24.0:1616513096.509764:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513096.509767:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513096.509767:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513096.509776:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.199@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.509785:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.199@tcp1 of length 224 into portal 12 MB=0x60182d9bbe780 00000400:00000200:26.0:1616513096.509791:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.199@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 28112 00000400:00000200:26.0:1616513096.509797:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.509799:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513096.509807:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.509810:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.509812:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.509815:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513096.509875:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511370610560 00010000:00000200:22.0:1616513096.509888:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693f00 x1690511370610560/t0(0) o400->98c87c69-437a-d7b0-37e5-006ff1d03d16@10.31.140.199@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513096.509903:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511370610560, offset 192 00000400:00000200:22.0:1616513096.509909:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.199@tcp1 00000400:00000200:22.0:1616513096.509917:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.199@tcp1 routed destination 00000400:00000200:22.0:1616513096.509931:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.199@tcp1(10.31.140.199@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513096.509938:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513096.509945:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513096.509948:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513096.509950:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513096.509956:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.509961:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.509971:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.509976:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.509978:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513096.509980:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513096.509985:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.509988:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba90 00000800:00000200:18.0:1616513096.509992:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.519017:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513096.519092:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.519102:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513096.519112:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.184@tcp1 : PUT - for me 00000400:00000200:0.0:1616513096.519121:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.184@tcp1 of length 224 into portal 26 MB=0x60194c7434940 00000400:00000200:0.0:1616513096.519128:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.184@tcp1 of length 224/224 into md 0xf97cfa39 [1] + 0 00000400:00000200:0.0:1616513096.519133:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf37e8 00000400:00000200:0.0:1616513096.519136:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513096.519139:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513096.519143:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf37e8 00000800:00000200:20.0:1616513096.519147:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.519147:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.519149:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513096.519154:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513096.519157:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513096.519158:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513096.519167:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.184@tcp1 : PUT - for me 00000400:00000200:20.0:1616513096.519176:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.184@tcp1 of length 224 into portal 12 MB=0x60194c7434a80 00000400:00000200:20.0:1616513096.519182:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.184@tcp1 of length 224/224 into md 0x865be291 [1] + 60792 00000400:00000200:20.0:1616513096.519188:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513096.519190:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513096.519198:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513096.519201:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513096.519203:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.519207:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588370127168 00010000:00000200:12.0:1616513096.519218:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1690588370127168/t0(0) o400->eca2550f-268d-62d5-3523-62fcd6e6a292@10.31.132.184@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.519232:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588370127168, offset 192 00000400:00000200:12.0:1616513096.519238:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.184@tcp1 00000400:00000200:12.0:1616513096.519246:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.184@tcp1 routed destination 00000400:00000200:12.0:1616513096.519261:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.184@tcp1(10.31.132.184@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513096.519263:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588370127488 00000800:00000200:12.0:1616513096.519267:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.519273:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:28.0:1616513096.519276:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0ee300 x1690588370127488/t0(0) o400->053d684e-eb3a-5478-208b-a5ec58780d17@10.31.132.184@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513096.519276:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.519278:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.519283:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.519288:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:28.0:1616513096.519290:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588370127488, offset 192 00000100:00000200:12.0:1616513096.519290:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513096.519297:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.184@tcp1 00000400:00000200:28.0:1616513096.519304:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.184@tcp1 routed destination 00000400:00000200:28.0:1616513096.519319:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.184@tcp1(10.31.132.184@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.519326:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.519332:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.519334:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.519337:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.519342:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513096.519342:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513096.519349:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513096.519350:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513096.519353:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513096.519353:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513096.519359:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513096.519365:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f430 00000800:00000200:16.0:1616513096.519370:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513096.519406:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.519413:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513096.519415:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513096.519422:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513096.519426:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccaa0 00000800:00000200:2.0:1616513096.519430:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513096.519454:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.523442:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513096.523497:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.523507:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.523516:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.212@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.523524:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.212@tcp1 of length 224 into portal 12 MB=0x6000e7c35f7c0 00000400:00000200:14.0:1616513096.523531:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.212@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 28336 00000400:00000200:14.0:1616513096.523536:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.523539:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513096.523546:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.523549:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513096.523551:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.523555:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.523579:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:22.0:1616513096.523610:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688912073717696 00010000:00000200:22.0:1616513096.523623:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfe300 x1688912073717696/t0(0) o400->d04eb00b-774f-61ec-949e-e75a93d78b67@10.31.136.212@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513096.523632:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513096.523637:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688912073717696, offset 192 00000800:00000200:2.0:1616513096.523641:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513096.523645:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.212@tcp1 00000400:00000200:22.0:1616513096.523652:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.212@tcp1 routed destination 00000400:00000200:2.0:1616513096.523652:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.212@tcp1 : PUT - for me 00000400:00000200:2.0:1616513096.523661:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.212@tcp1 of length 224 into portal 26 MB=0x6000e7c35f740 00000400:00000200:22.0:1616513096.523668:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.212@tcp1(10.31.136.212@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513096.523668:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.212@tcp1 of length 224/224 into md 0xf97cfa89 [1] + 0 00000400:00000200:2.0:1616513096.523673:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2ff0 00000800:00000200:22.0:1616513096.523674:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513096.523675:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513096.523678:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513096.523680:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513096.523682:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:2.0:1616513096.523682:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:22.0:1616513096.523685:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513096.523685:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.523686:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513096.523688:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513096.523689:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513096.523690:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.523696:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.523747:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688912073717568 00000800:00000200:20.0:1616513096.523752:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:12.0:1616513096.523757:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688912073717568/t0(0) o400->5c997e49-4f83-bfcc-c22e-8a6e42e5416e@10.31.136.212@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513096.523759:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513096.523761:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513096.523768:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513096.523772:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a000 00000100:00000200:12.0:1616513096.523772:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688912073717568, offset 192 00000800:00000200:20.0:1616513096.523776:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:12.0:1616513096.523778:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.212@tcp1 00000800:00000200:20.0:1616513096.523779:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:12.0:1616513096.523785:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.212@tcp1 routed destination 00000400:00000200:12.0:1616513096.523800:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.212@tcp1(10.31.136.212@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.523807:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513096.523812:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.523815:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513096.523817:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513096.523822:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.523826:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513096.523828:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513096.523837:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.523842:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.523845:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513096.523846:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513096.523851:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.523857:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa90 00000800:00000200:18.0:1616513096.523860:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.530622:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513096.530675:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.530684:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.530693:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.160@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.530702:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.160@tcp1 of length 224 into portal 26 MB=0x5ffd51cc03cc0 00000400:00000200:26.0:1616513096.530709:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.160@tcp1 of length 224/224 into md 0xf97cfab9 [1] + 0 00000400:00000200:26.0:1616513096.530714:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3ed0 00000400:00000200:26.0:1616513096.530716:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.530719:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513096.530723:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ed0 00000800:00000200:26.0:1616513096.530726:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.530731:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513096.530731:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.530734:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.530783:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665659030720 00010000:00000200:12.0:1616513096.530794:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688665659030720/t0(0) o400->957105a4-c94b-06fd-6c43-b840e72adc42@10.31.132.160@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.530809:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665659030720, offset 192 00000400:00000200:12.0:1616513096.530814:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.160@tcp1 00000400:00000200:12.0:1616513096.530821:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.160@tcp1 routed destination 00000400:00000200:12.0:1616513096.530835:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.160@tcp1(10.31.132.160@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.530841:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.530846:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.530849:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.530851:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.530855:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.530860:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.530861:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513096.530915:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.530922:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513096.530924:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513096.530931:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513096.530937:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e5d8 00000800:00000200:0.0:1616513096.530942:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.530945:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513096.531324:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513096.531377:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513096.531386:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513096.531395:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.160@tcp1 : PUT - for me 00000400:00000200:20.0:1616513096.531404:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.160@tcp1 of length 224 into portal 12 MB=0x5ffd51cc03ec0 00000400:00000200:20.0:1616513096.531411:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.160@tcp1 of length 224/224 into md 0x865be291 [1] + 61016 00000400:00000200:20.0:1616513096.531416:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513096.531421:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513096.531427:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513096.531430:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513096.531431:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513096.531435:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513096.531491:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665659031232 00010000:00000200:28.0:1616513096.531503:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d576f0e8000 x1688665659031232/t0(0) o400->8009775a-f2f8-f5fb-c096-70a16ab8a8e6@10.31.132.160@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513096.531518:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665659031232, offset 192 00000400:00000200:28.0:1616513096.531524:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.160@tcp1 00000400:00000200:28.0:1616513096.531531:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.160@tcp1 routed destination 00000400:00000200:28.0:1616513096.531546:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.160@tcp1(10.31.132.160@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.531553:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.531558:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.531561:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.531563:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.531568:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.531573:0:51219:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.531627:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.531634:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513096.531637:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513096.531643:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513096.531647:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc990 00000800:00000200:14.0:1616513096.531651:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.531654:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513096.602715:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929472, portal 4 00000100:00000200:14.0:1616513096.602722:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929472, offset 0 00000400:00000200:14.0:1616513096.602728:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:14.0:1616513096.602734:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:14.0:1616513096.602747:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513096.602753:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:14.0:1616513096.602762:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513096.602765:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:14.0:1616513096.602767:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:14.0:1616513096.602773:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513096.602822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513096.602842:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513096.602854:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513096.602857:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513096.602865:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513096.602872:0:6476:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdf3d1b00 x1693067725929472/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513096.602884:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756dd0 00000800:00000200:27.0:1616513096.602890:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513096.602892:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513096.602900:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513096.602904:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513096.602908:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513096.602916:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627c00 00000400:00000200:27.0:1616513096.602924:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716734d [1] + 192 00000400:00000200:27.0:1616513096.602929:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513096.602933:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdf3d1b00 x1693067725929472/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513096.602948:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513096.602951:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513096.602953:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:14.0:1616513096.602956:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdf3d1b00 x1693067725929472/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513096.602968:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdf3d1b00 x1693067725929472/t0(0) o13->boslfs02-OST0016-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513103 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513096.602978:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756440 00000800:00000200:7.2:1616513096.621059:0:51227:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513096.621117:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513096.621134:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513096.621144:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.116@tcp1 : PUT - for me 00000400:00000200:2.0:1616513096.621153:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.116@tcp1 of length 224 into portal 26 MB=0x60215e01e3800 00000400:00000200:2.0:1616513096.621160:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.116@tcp1 of length 224/224 into md 0xf97cfac9 [1] + 0 00000400:00000200:2.0:1616513096.621165:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2550 00000400:00000200:2.0:1616513096.621168:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513096.621171:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513096.621176:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2550 00000800:00000200:2.0:1616513096.621180:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.621182:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.621183:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513096.621189:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513096.621192:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.621193:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.621202:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.116@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.621211:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.116@tcp1 of length 224 into portal 12 MB=0x60215e01e3880 00000400:00000200:26.0:1616513096.621218:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.116@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 28560 00000400:00000200:26.0:1616513096.621223:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.621226:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513096.621234:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.621237:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.621239:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.621242:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691142837909504 00010000:00000200:12.0:1616513096.621253:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1691142837909504/t0(0) o400->baac9af8-c008-178e-e088-580f63ec7aac@10.31.141.116@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.621267:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691142837909504, offset 192 00000400:00000200:12.0:1616513096.621273:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.116@tcp1 00000400:00000200:12.0:1616513096.621280:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.116@tcp1 routed destination 00000400:00000200:12.0:1616513096.621294:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.116@tcp1(10.31.141.116@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513096.621300:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691142837909632 00000800:00000200:12.0:1616513096.621300:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.621305:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.621308:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.621310:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513096.621314:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c692880 x1691142837909632/t0(0) o400->4fbca977-13bd-5b78-8284-76cf7a7ac775@10.31.141.116@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513096.621315:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513096.621322:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513096.621323:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513096.621328:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691142837909632, offset 192 00000400:00000200:22.0:1616513096.621334:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.116@tcp1 00000400:00000200:22.0:1616513096.621341:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.116@tcp1 routed destination 00000400:00000200:22.0:1616513096.621357:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.116@tcp1(10.31.141.116@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513096.621363:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513096.621369:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513096.621372:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513096.621374:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513096.621378:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513096.621379:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513096.621385:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513096.621386:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513096.621387:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513096.621394:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513096.621399:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:0.0:1616513096.621402:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.621405:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513096.621453:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.621461:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.621463:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513096.621465:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513096.621470:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.621474:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a908 00000800:00000200:18.0:1616513096.621478:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.689013:0:51234:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513096.689071:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.689081:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.689091:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.31@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.689102:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.31@tcp1 of length 224 into portal 26 MB=0x5ffea2e36a580 00000400:00000200:14.0:1616513096.689109:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.31@tcp1 of length 224/224 into md 0xf97cfae9 [1] + 0 00000400:00000200:14.0:1616513096.689114:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf27f8 00000400:00000200:14.0:1616513096.689117:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.689120:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513096.689125:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf27f8 00000800:00000200:26.0:1616513096.689127:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.689129:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.689132:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.689133:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.689194:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688756146316672 00010000:00000200:12.0:1616513096.689208:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688756146316672/t0(0) o400->8d94cfda-efb8-4622-bc9f-cb499f0a0bb0@10.31.142.31@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.689223:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688756146316672, offset 192 00000400:00000200:12.0:1616513096.689229:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.31@tcp1 00000400:00000200:12.0:1616513096.689237:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.31@tcp1 routed destination 00000400:00000200:12.0:1616513096.689255:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.31@tcp1(10.31.142.31@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.689263:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.689269:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.689272:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.689274:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.689281:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513096.689288:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513096.689289:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.689346:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.689353:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513096.689356:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513096.689363:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513096.689369:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f100 00000800:00000200:24.0:1616513096.689372:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513096.689376:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513096.689831:0:51234:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513096.689895:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513096.689906:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513096.689914:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.31@tcp1 : PUT - for me 00000400:00000200:2.0:1616513096.689923:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.31@tcp1 of length 224 into portal 12 MB=0x5ffea2e36b380 00000400:00000200:2.0:1616513096.689930:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.31@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 28784 00000400:00000200:2.0:1616513096.689935:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513096.689938:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513096.689947:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.689950:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513096.689951:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513096.689954:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513096.690015:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688756146320256 00010000:00000200:22.0:1616513096.690029:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d4800 x1688756146320256/t0(0) o400->a7c56294-829d-329a-d2f3-8233c7acaff2@10.31.142.31@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513096.690044:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688756146320256, offset 192 00000400:00000200:22.0:1616513096.690050:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.31@tcp1 00000400:00000200:22.0:1616513096.690058:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.31@tcp1 routed destination 00000400:00000200:22.0:1616513096.690072:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.31@tcp1(10.31.142.31@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513096.690079:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513096.690086:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513096.690090:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513096.690092:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513096.690098:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.690103:0:51234:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.690114:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.690120:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513096.690122:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513096.690123:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513096.690129:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513096.690133:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a4c8 00000800:00000200:26.0:1616513096.690137:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.694060:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513096.694116:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.694125:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513096.694134:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.53@tcp1 : PUT - for me 00000400:00000200:0.0:1616513096.694143:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.53@tcp1 of length 224 into portal 12 MB=0x6052127b50800 00000400:00000200:0.0:1616513096.694150:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.53@tcp1 of length 224/224 into md 0x865be291 [1] + 61240 00000400:00000200:0.0:1616513096.694155:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513096.694158:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513096.694165:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.694168:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513096.694170:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513096.694174:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513096.694238:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694489818499072 00010000:00000200:28.0:1616513096.694250:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86ec00 x1694489818499072/t0(0) o400->81bcc19b-a047-67b1-e23d-90d37a9a289c@10.31.141.53@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513096.694261:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:28.0:1616513096.694263:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694489818499072, offset 192 00000400:00000200:28.0:1616513096.694269:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.53@tcp1 00000400:00000200:28.0:1616513096.694276:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.53@tcp1 routed destination 00000400:00000200:28.0:1616513096.694290:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.53@tcp1(10.31.141.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.694296:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513096.694301:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.694303:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513096.694306:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513096.694310:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.694314:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513096.694316:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.694326:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513096.694335:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.53@tcp1 : PUT - for me 00000400:00000200:18.0:1616513096.694343:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.53@tcp1 of length 224 into portal 26 MB=0x6052127b50780 00000400:00000200:18.0:1616513096.694350:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.53@tcp1 of length 224/224 into md 0xf97cfb29 [1] + 0 00000400:00000200:18.0:1616513096.694355:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf24c8 00000400:00000200:18.0:1616513096.694357:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513096.694360:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513096.694364:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf24c8 00000800:00000200:18.0:1616513096.694367:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513096.694368:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513096.694370:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513096.694372:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513096.694375:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513096.694376:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513096.694378:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513096.694385:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513096.694389:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc880 00000800:00000200:24.0:1616513096.694392:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513096.694395:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.694459:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694489818498944 00010000:00000200:12.0:1616513096.694469:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1694489818498944/t0(0) o400->5ce708f6-d1e2-2639-07a5-23c884be4152@10.31.141.53@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.694484:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694489818498944, offset 192 00000400:00000200:12.0:1616513096.694490:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.53@tcp1 00000400:00000200:12.0:1616513096.694498:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.53@tcp1 routed destination 00000400:00000200:12.0:1616513096.694513:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.53@tcp1(10.31.141.53@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.694520:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513096.694525:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.694528:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513096.694530:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513096.694535:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.694540:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513096.694542:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513096.694592:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.694599:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513096.694601:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513096.694608:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513096.694613:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f5c8 00000800:00000200:14.0:1616513096.694617:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.694620:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.702077:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513096.702134:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513096.702143:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513096.702149:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513096.702152:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.57@tcp1 : PUT - for me 00000800:00000200:0.0:1616513096.702160:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513096.702162:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.57@tcp1 of length 224 into portal 26 MB=0x6018358a70900 00000400:00000200:0.0:1616513096.702168:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.57@tcp1 : PUT - for me 00000400:00000200:16.0:1616513096.702169:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.57@tcp1 of length 224/224 into md 0xf97cfb39 [1] + 0 00000400:00000200:16.0:1616513096.702174:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2aa0 00000400:00000200:16.0:1616513096.702177:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:0.0:1616513096.702178:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.57@tcp1 of length 224 into portal 12 MB=0x6018358a70a00 00000100:00000200:16.0:1616513096.702179:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513096.702184:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2aa0 00000400:00000200:0.0:1616513096.702184:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.57@tcp1 of length 224/224 into md 0x865be291 [1] + 61464 00000800:00000200:16.0:1616513096.702187:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:0.0:1616513096.702189:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:16.0:1616513096.702190:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513096.702191:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:0.0:1616513096.702192:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513096.702194:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.702199:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.702202:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.702204:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.702263:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513499949312 00000100:00000200:28.0:1616513096.702265:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513499949568 00010000:00000200:12.0:1616513096.702274:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1690513499949312/t0(0) o400->a0f67f03-2543-fbb7-db4a-4dfa67d9ef68@10.31.131.57@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:28.0:1616513096.702277:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86bf00 x1690513499949568/t0(0) o400->d28db30c-f3a8-d209-8417-b804f04f15bc@10.31.131.57@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513096.702288:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513499949312, offset 192 00000100:00000200:28.0:1616513096.702292:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513499949568, offset 192 00000400:00000200:12.0:1616513096.702295:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.57@tcp1 00000400:00000200:28.0:1616513096.702300:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.57@tcp1 00000400:00000200:12.0:1616513096.702302:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.57@tcp1 routed destination 00000400:00000200:12.0:1616513096.702318:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.57@tcp1(10.31.131.57@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.702324:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513096.702327:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.57@tcp1 routed destination 00000800:00000200:12.0:1616513096.702339:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.702344:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.702346:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513096.702347:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.57@tcp1(10.31.131.57@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.702350:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513096.702353:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513096.702356:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.702357:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513096.702359:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.702362:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.702364:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.702369:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.702375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513096.702411:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513096.702419:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513096.702421:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513096.702422:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513096.702428:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513096.702430:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513096.702450:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ed48 00000800:00000200:18.0:1616513096.702451:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513096.702453:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513096.702454:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.702460:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.702464:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc770 00000800:00000200:18.0:1616513096.702468:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.702471:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.800324:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513096.800382:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.800393:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.800403:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.181@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.800414:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.181@tcp1 of length 224 into portal 26 MB=0x5ffde71aa29c0 00000400:00000200:14.0:1616513096.800423:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.181@tcp1 of length 224/224 into md 0xf97cfb59 [1] + 0 00000400:00000200:14.0:1616513096.800453:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2e58 00000800:00000200:26.0:1616513096.800456:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513096.800457:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.800460:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513096.800465:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2e58 00000800:00000200:14.0:1616513096.800469:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.800472:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.800473:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.800539:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705738353088 00010000:00000200:12.0:1616513096.800551:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688705738353088/t0(0) o400->69e0a1be-289f-96f3-09a5-2904c5fb2e44@10.31.132.181@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.800566:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705738353088, offset 192 00000400:00000200:12.0:1616513096.800572:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.181@tcp1 00000400:00000200:12.0:1616513096.800580:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.181@tcp1 routed destination 00000400:00000200:12.0:1616513096.800595:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.181@tcp1(10.31.132.181@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.800602:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.800608:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.800611:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.800613:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.800619:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513096.800626:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513096.800664:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.800721:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.800729:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513096.800732:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513096.800738:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513096.800738:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513096.800745:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f6d8 00000800:00000200:24.0:1616513096.800748:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513096.800881:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513096.800934:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513096.800943:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513096.800952:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.181@tcp1 : PUT - for me 00000400:00000200:16.0:1616513096.800960:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.181@tcp1 of length 224 into portal 12 MB=0x5ffde71aa2ac0 00000400:00000200:16.0:1616513096.800966:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.181@tcp1 of length 224/224 into md 0x865be291 [1] + 61688 00000400:00000200:16.0:1616513096.800971:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513096.800974:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513096.800981:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513096.800984:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513096.800986:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513096.800988:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513096.801047:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705738353344 00010000:00000200:28.0:1616513096.801061:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b101680 x1688705738353344/t0(0) o400->7cb89c0f-1f69-a03e-13ee-d2da71f4f158@10.31.132.181@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513096.801075:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705738353344, offset 192 00000400:00000200:28.0:1616513096.801082:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.181@tcp1 00000400:00000200:28.0:1616513096.801089:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.181@tcp1 routed destination 00000400:00000200:28.0:1616513096.801104:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.181@tcp1(10.31.132.181@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.801115:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.801122:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.801125:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.801127:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.801133:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.801139:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.801151:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.801158:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.801161:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.801168:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.801172:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc660 00000800:00000200:18.0:1616513096.801176:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.801179:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.835728:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513096.835786:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.835796:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513096.835805:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.69@tcp1 : PUT - for me 00000400:00000200:0.0:1616513096.835817:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.69@tcp1 of length 224 into portal 12 MB=0x603ff8ee4f840 00000400:00000200:0.0:1616513096.835823:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.69@tcp1 of length 224/224 into md 0x865be291 [1] + 61912 00000400:00000200:0.0:1616513096.835829:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513096.835832:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513096.835839:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.835840:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.835842:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513096.835844:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513096.835905:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693246009178176 00000800:00000200:7.2:1616513096.835907:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:28.0:1616513096.835920:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86ba80 x1693246009178176/t0(0) o400->dd9d825e-1cd0-85db-b258-396400dceb6c@10.31.132.69@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513096.835934:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693246009178176, offset 192 00000400:00000200:28.0:1616513096.835941:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.69@tcp1 00000400:00000200:28.0:1616513096.835949:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.69@tcp1 routed destination 00000400:00000200:28.0:1616513096.835964:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.69@tcp1(10.31.132.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513096.835966:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.835971:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513096.835975:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.835977:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.835980:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513096.835984:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513096.835985:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.69@tcp1 : PUT - for me 00000800:00000200:28.0:1616513096.835989:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513096.835994:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.69@tcp1 of length 224 into portal 26 MB=0x603ff8ee4f780 00000800:00000200:11.2:1616513096.835995:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513096.836000:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.69@tcp1 of length 224/224 into md 0xf97cfb79 [1] + 0 00000400:00000200:26.0:1616513096.836005:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3430 00000400:00000200:26.0:1616513096.836008:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.836011:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513096.836015:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3430 00000800:00000200:26.0:1616513096.836018:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.836020:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.836021:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.836024:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513096.836058:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513096.836065:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513096.836068:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513096.836074:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513096.836079:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc440 00000800:00000200:16.0:1616513096.836082:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.836085:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.836095:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693246009177984 00010000:00000200:12.0:1616513096.836106:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1693246009177984/t0(0) o400->08738ba8-00d3-350d-16db-1f3f3badb4a6@10.31.132.69@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.836121:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693246009177984, offset 192 00000400:00000200:12.0:1616513096.836127:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.69@tcp1 00000400:00000200:12.0:1616513096.836135:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.69@tcp1 routed destination 00000400:00000200:12.0:1616513096.836150:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.69@tcp1(10.31.132.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.836157:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513096.836163:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.836166:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513096.836168:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513096.836173:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.836178:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513096.836180:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513096.836232:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.836249:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.836251:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.836258:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.836264:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e4c8 00000800:00000200:18.0:1616513096.836267:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.836270:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.840157:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513096.840212:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.840221:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513096.840230:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.97@tcp1 : PUT - for me 00000400:00000200:24.0:1616513096.840240:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.97@tcp1 of length 224 into portal 26 MB=0x60467b95d7e00 00000400:00000200:24.0:1616513096.840246:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.97@tcp1 of length 224/224 into md 0xf97cfb89 [1] + 0 00000400:00000200:24.0:1616513096.840251:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2880 00000400:00000200:24.0:1616513096.840254:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513096.840257:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513096.840261:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2880 00000800:00000200:24.0:1616513096.840264:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.840266:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513096.840267:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513096.840270:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.840324:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693693398318592 00010000:00000200:12.0:1616513096.840335:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1693693398318592/t0(0) o400->734d68a0-8a91-6824-76ce-168803b87303@10.31.128.97@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.840349:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693693398318592, offset 192 00000400:00000200:12.0:1616513096.840355:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.97@tcp1 00000400:00000200:12.0:1616513096.840363:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.97@tcp1 routed destination 00000400:00000200:12.0:1616513096.840378:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.97@tcp1(10.31.128.97@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.840385:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.840390:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.840393:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.840395:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.840400:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.840405:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.840407:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513096.840458:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513096.840465:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513096.840468:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513096.840475:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513096.840480:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:16.0:1616513096.840484:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.840487:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513096.840618:0:51249:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513096.840674:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.840683:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.840693:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.97@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.840701:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.97@tcp1 of length 224 into portal 12 MB=0x60467b95d8280 00000400:00000200:14.0:1616513096.840708:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.97@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 29008 00000400:00000200:14.0:1616513096.840713:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.840716:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513096.840723:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.840726:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.840727:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.840737:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513096.840789:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693693398319744 00010000:00000200:22.0:1616513096.840802:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfa880 x1693693398319744/t0(0) o400->a338b4d5-df5e-cc81-e859-ab382a1dd185@10.31.128.97@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513096.840816:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693693398319744, offset 192 00000400:00000200:22.0:1616513096.840822:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.97@tcp1 00000400:00000200:22.0:1616513096.840830:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.97@tcp1 routed destination 00000400:00000200:22.0:1616513096.840845:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.97@tcp1(10.31.128.97@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513096.840851:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513096.840857:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513096.840860:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513096.840862:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513096.840867:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.840871:0:51249:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.840883:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.840888:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.840891:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.840897:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.840901:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a110 00000800:00000200:18.0:1616513096.840904:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.840907:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:31.0:1616513096.871542:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513096.871550:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513096.899036:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513096.899092:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513096.899095:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513096.899103:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513096.899113:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.81@tcp1 : PUT - for me 00000400:00000200:0.0:1616513096.899123:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.81@tcp1 of length 224 into portal 26 MB=0x5f90c0df92600 00000400:00000200:0.0:1616513096.899130:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.81@tcp1 of length 224/224 into md 0xf97cfb99 [1] + 0 00000400:00000200:0.0:1616513096.899135:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf22a8 00000400:00000200:0.0:1616513096.899139:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513096.899142:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513096.899147:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf22a8 00000800:00000200:24.0:1616513096.899149:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.899151:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.899154:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.899156:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513096.899171:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.899183:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.899193:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.81@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.899203:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.81@tcp1 of length 224 into portal 12 MB=0x5f90c0df92680 00000400:00000200:26.0:1616513096.899209:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.81@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 29232 00000400:00000200:26.0:1616513096.899215:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513096.899218:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681205052909056 00000100:00000200:26.0:1616513096.899219:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513096.899227:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.899227:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.899231:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00010000:00000200:12.0:1616513096.899231:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1681205052909056/t0(0) o400->68bfc844-9c81-d401-2247-290519a043d3@10.31.131.81@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513096.899232:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.899247:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681205052909056, offset 192 00000400:00000200:12.0:1616513096.899253:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.81@tcp1 00000400:00000200:12.0:1616513096.899261:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.81@tcp1 routed destination 00000400:00000200:12.0:1616513096.899278:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.81@tcp1(10.31.131.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.899284:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.899291:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.899293:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513096.899296:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681205052909184 00000800:00000200:12.0:1616513096.899296:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.899301:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.899306:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.899309:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:22.0:1616513096.899311:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c697980 x1681205052909184/t0(0) o400->98d41b88-0bd4-35a4-f097-c35b19675f59@10.31.131.81@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513096.899326:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681205052909184, offset 192 00000400:00000200:22.0:1616513096.899332:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.81@tcp1 00000400:00000200:22.0:1616513096.899339:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.81@tcp1 routed destination 00000400:00000200:22.0:1616513096.899354:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.81@tcp1(10.31.131.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513096.899361:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513096.899361:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513096.899368:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513096.899368:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513096.899371:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513096.899371:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513096.899373:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513096.899378:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513096.899379:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513096.899384:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f6d8 00000800:00000200:7.2:1616513096.899384:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513096.899387:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.899390:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513096.899452:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.899459:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.899462:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.899468:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.899471:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba08 00000800:00000200:18.0:1616513096.899475:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.899481:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.910800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513096.910843:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513096.910856:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513096.910866:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513096.910875:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.74@tcp1 : PUT - for me 00000400:00000200:24.0:1616513096.910885:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.74@tcp1 of length 224 into portal 26 MB=0x5f90a44741000 00000400:00000200:24.0:1616513096.910892:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.74@tcp1 of length 224/224 into md 0xf97cfba9 [1] + 0 00000400:00000200:24.0:1616513096.910896:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3760 00000800:00000200:14.0:1616513096.910899:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513096.910901:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513096.910904:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513096.910908:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3760 00000800:00000200:14.0:1616513096.910908:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513096.910912:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513096.910914:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513096.910914:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513096.910917:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.74@tcp1 : PUT - for me 00000800:00000200:24.0:1616513096.910918:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513096.910926:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.74@tcp1 of length 224 into portal 12 MB=0x5f90a44741080 00000400:00000200:14.0:1616513096.910933:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.74@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 29456 00000400:00000200:14.0:1616513096.910938:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.910941:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513096.910948:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.910951:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.910952:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.910956:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.910989:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681197376999424 00010000:00000200:12.0:1616513096.911001:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1681197376999424/t0(0) o400->c0d3215a-467f-04d9-083b-76e8d6b134b6@10.31.131.74@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.911015:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681197376999424, offset 192 00000100:00000200:22.0:1616513096.911016:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681197376999552 00000400:00000200:12.0:1616513096.911023:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.74@tcp1 00010000:00000200:22.0:1616513096.911029:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfd580 x1681197376999552/t0(0) o400->bbc9e879-be0d-ef64-f453-0b6d247be607@10.31.131.74@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513096.911031:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.74@tcp1 routed destination 00000400:00000200:12.0:1616513096.911045:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.74@tcp1(10.31.131.74@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.911052:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:22.0:1616513096.911055:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681197376999552, offset 192 00000800:00000200:12.0:1616513096.911057:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.911060:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513096.911061:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.74@tcp1 00000800:00000200:12.0:1616513096.911063:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.911068:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513096.911069:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.74@tcp1 routed destination 00000800:00000200:11.2:1616513096.911073:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.911075:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513096.911083:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.74@tcp1(10.31.131.74@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513096.911090:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513096.911096:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513096.911099:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513096.911101:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513096.911106:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.911111:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513096.911125:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513096.911132:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513096.911135:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513096.911142:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513096.911148:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f5c8 00000800:00000200:16.0:1616513096.911151:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.911154:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513096.911164:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.911171:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.911174:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513096.911181:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.911185:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b320 00000800:00000200:18.0:1616513096.911189:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513096.911192:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.916819:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513096.916875:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.916884:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.916892:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.37@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.916901:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.37@tcp1 of length 224 into portal 26 MB=0x603cf018b4fc0 00000400:00000200:26.0:1616513096.916908:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.37@tcp1 of length 224/224 into md 0xf97cfbc9 [1] + 0 00000400:00000200:26.0:1616513096.916913:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3cb0 00000400:00000200:26.0:1616513096.916915:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.916918:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513096.916922:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3cb0 00000800:00000200:26.0:1616513096.916925:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.916928:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.916929:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.916932:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.916937:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.916943:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.37@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.916949:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.37@tcp1 of length 224 into portal 12 MB=0x603cf018b5040 00000400:00000200:26.0:1616513096.916954:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.37@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 29680 00000400:00000200:26.0:1616513096.916957:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.916959:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513096.916964:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513096.916966:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.916969:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.916972:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.916984:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693037479284672 00010000:00000200:12.0:1616513096.916994:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1693037479284672/t0(0) o400->8cb93be6-29a8-6abf-a359-33552ee1be3c@10.31.132.37@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.917008:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693037479284672, offset 192 00000400:00000200:12.0:1616513096.917013:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.37@tcp1 00000400:00000200:12.0:1616513096.917021:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.37@tcp1 routed destination 00000100:00000200:22.0:1616513096.917030:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693037479284800 00000400:00000200:12.0:1616513096.917034:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.37@tcp1(10.31.132.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.917040:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:22.0:1616513096.917042:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c692400 x1693037479284800/t0(0) o400->cfd407a6-d984-94ab-f12d-68042dfc55ed@10.31.132.37@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513096.917045:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.917048:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.917050:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.917054:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513096.917055:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693037479284800, offset 192 00000800:00000200:11.2:1616513096.917060:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513096.917061:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.37@tcp1 00000100:00000200:12.0:1616513096.917061:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513096.917068:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.37@tcp1 routed destination 00000400:00000200:22.0:1616513096.917081:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.37@tcp1(10.31.132.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513096.917087:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513096.917093:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513096.917095:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513096.917097:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513096.917102:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.917106:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513096.917112:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.917119:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513096.917122:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513096.917129:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513096.917135:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:0.0:1616513096.917138:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.917141:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513096.917158:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513096.917165:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513096.917168:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513096.917170:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513096.917174:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513096.917178:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aaa0 00000800:00000200:2.0:1616513096.917182:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.929101:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513096.929155:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.929164:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513096.929174:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.218@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.929184:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.218@tcp1 of length 224 into portal 26 MB=0x5ffd4f5acb780 00000400:00000200:26.0:1616513096.929190:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.218@tcp1 of length 224/224 into md 0xf97cfbd9 [1] + 0 00000400:00000200:26.0:1616513096.929195:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2330 00000400:00000200:26.0:1616513096.929198:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.929201:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513096.929205:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2330 00000800:00000200:26.0:1616513096.929208:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.929209:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.929211:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513096.929213:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.929267:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665003440000 00010000:00000200:12.0:1616513096.929277:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688665003440000/t0(0) o400->f8900430-2cff-33a4-3f33-7ed5ba0d941b@10.31.132.218@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.929290:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665003440000, offset 192 00000400:00000200:12.0:1616513096.929296:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.218@tcp1 00000400:00000200:12.0:1616513096.929303:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.218@tcp1 routed destination 00000400:00000200:12.0:1616513096.929317:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.218@tcp1(10.31.132.218@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.929323:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.929330:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.929333:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.929335:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.929340:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513096.929346:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513096.929346:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513096.929368:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513096.929400:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513096.929407:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513096.929409:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513096.929416:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513096.929422:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa90 00000800:00000200:14.0:1616513096.929422:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513096.929425:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513096.929428:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513096.929447:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.929456:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.218@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.929464:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.218@tcp1 of length 224 into portal 12 MB=0x5ffd4f5acb880 00000400:00000200:14.0:1616513096.929470:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.218@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 29904 00000400:00000200:14.0:1616513096.929475:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.929477:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513096.929483:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.929487:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.929488:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.929491:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513096.929545:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665003440256 00010000:00000200:22.0:1616513096.929557:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfb600 x1688665003440256/t0(0) o400->332ada9b-a836-e098-bfa4-0347399b57ca@10.31.132.218@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513096.929574:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665003440256, offset 192 00000400:00000200:22.0:1616513096.929580:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.218@tcp1 00000400:00000200:22.0:1616513096.929586:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.218@tcp1 routed destination 00000400:00000200:22.0:1616513096.929600:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.218@tcp1(10.31.132.218@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513096.929607:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513096.929613:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513096.929616:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513096.929618:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513096.929622:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.929626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.929636:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.929641:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.929644:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513096.929645:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513096.929651:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.929654:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b650 00000800:00000200:18.0:1616513096.929658:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513096.940419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513096.940474:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513096.940483:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513096.940493:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.75@tcp1 : PUT - for me 00000400:00000200:0.0:1616513096.940502:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.75@tcp1 of length 224 into portal 26 MB=0x5ffd67b2dbcc0 00000400:00000200:0.0:1616513096.940509:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.75@tcp1 of length 224/224 into md 0xf97cfbe9 [1] + 0 00000400:00000200:0.0:1616513096.940515:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3980 00000400:00000200:0.0:1616513096.940518:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513096.940521:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513096.940525:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3980 00000800:00000200:24.0:1616513096.940528:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513096.940528:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513096.940531:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513096.940533:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513096.940589:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688671538232512 00000800:00000200:11.2:1616513096.940594:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:12.0:1616513096.940599:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688671538232512/t0(0) o400->6bf922de-1c55-1ad2-2a2c-512c3c500f65@10.31.129.75@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.940614:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688671538232512, offset 192 00000400:00000200:12.0:1616513096.940620:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.75@tcp1 00000400:00000200:12.0:1616513096.940628:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.75@tcp1 routed destination 00000400:00000200:12.0:1616513096.940643:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.75@tcp1(10.31.129.75@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513096.940647:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.940649:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.940655:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513096.940656:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.940658:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513096.940660:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:16.0:1616513096.940665:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.75@tcp1 : PUT - for me 00000800:00000200:12.0:1616513096.940665:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:12.0:1616513096.940671:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513096.940674:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.75@tcp1 of length 224 into portal 12 MB=0x5ffd67b2dc040 00000400:00000200:16.0:1616513096.940681:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.75@tcp1 of length 224/224 into md 0x865be291 [1] + 62136 00000400:00000200:16.0:1616513096.940686:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513096.940691:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513096.940698:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513096.940700:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513096.940701:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513096.940707:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513096.940717:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513096.940717:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513096.940719:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513096.940726:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513096.940732:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f5c8 00000800:00000200:20.0:1616513096.940735:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513096.940778:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688671538233408 00010000:00000200:28.0:1616513096.940790:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b100000 x1688671538233408/t0(0) o400->bd99cbf3-cf4c-3498-7b52-195324dc6469@10.31.129.75@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513096.940804:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688671538233408, offset 192 00000400:00000200:28.0:1616513096.940810:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.75@tcp1 00000400:00000200:28.0:1616513096.940818:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.75@tcp1 routed destination 00000400:00000200:28.0:1616513096.940833:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.75@tcp1(10.31.129.75@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513096.940839:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513096.940845:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513096.940848:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513096.940850:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513096.940854:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.940860:0:51259:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.940917:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.940923:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513096.940926:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513096.940933:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513096.940936:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc330 00000800:00000200:14.0:1616513096.940940:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513096.940943:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.950560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513096.950615:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.950624:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513096.950631:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513096.950633:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.83@tcp1 : PUT - for me 00000400:00000200:26.0:1616513096.950642:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.83@tcp1 of length 224 into portal 26 MB=0x604a3111f03c0 00000400:00000200:26.0:1616513096.950649:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.83@tcp1 of length 224/224 into md 0xf97cfbf9 [1] + 0 00000400:00000200:26.0:1616513096.950654:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3320 00000400:00000200:26.0:1616513096.950656:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513096.950659:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513096.950663:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:26.0:1616513096.950666:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513096.950669:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513096.950671:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.950738:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693948273689536 00010000:00000200:12.0:1616513096.950749:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1693948273689536/t0(0) o400->fd25278e-3676-4d3a-0391-a7fd88301aed@10.31.143.83@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.950763:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693948273689536, offset 192 00000800:00000200:7.2:1616513096.950764:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513096.950769:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.83@tcp1 00000400:00000200:12.0:1616513096.950777:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.83@tcp1 routed destination 00000400:00000200:12.0:1616513096.950792:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.83@tcp1(10.31.143.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.950798:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.950804:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.950807:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.950809:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.950814:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513096.950820:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513096.950822:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513096.950832:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.950841:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.950849:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.83@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.950857:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.83@tcp1 of length 224 into portal 12 MB=0x604a3111f0500 00000400:00000200:14.0:1616513096.950863:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.83@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 30128 00000400:00000200:14.0:1616513096.950868:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.950870:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513096.950873:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513096.950877:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.950879:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513096.950880:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513096.950881:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513096.950883:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513096.950884:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513096.950885:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513096.950889:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513096.950894:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:0.0:1616513096.950897:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513096.950937:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693948273689856 00010000:00000200:22.0:1616513096.950948:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfd100 x1693948273689856/t0(0) o400->d0a28e26-69bb-8579-694f-2f71df3dba35@10.31.143.83@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513096.950960:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693948273689856, offset 192 00000400:00000200:22.0:1616513096.950965:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.83@tcp1 00000400:00000200:22.0:1616513096.950972:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.83@tcp1 routed destination 00000400:00000200:22.0:1616513096.950984:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.83@tcp1(10.31.143.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513096.950990:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513096.950995:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513096.950998:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513096.950999:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513096.951003:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.951008:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513096.951019:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.951024:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.951027:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513096.951029:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513096.951033:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.951037:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ac38 00000800:00000200:18.0:1616513096.951041:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513096.962490:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513096.962542:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513096.962551:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513096.962559:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.252@tcp1 : PUT - for me 00000400:00000200:14.0:1616513096.962567:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.252@tcp1 of length 224 into portal 26 MB=0x5fffcb379e740 00000400:00000200:14.0:1616513096.962573:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.252@tcp1 of length 224/224 into md 0xf97cfc09 [1] + 0 00000400:00000200:14.0:1616513096.962579:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2088 00000400:00000200:14.0:1616513096.962582:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513096.962585:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513096.962589:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:14.0:1616513096.962591:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513096.962594:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513096.962594:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513096.962601:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513096.962604:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513096.962605:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513096.962613:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.252@tcp1 : PUT - for me 00000400:00000200:2.0:1616513096.962621:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.252@tcp1 of length 224 into portal 12 MB=0x5fffcb379e7c0 00000400:00000200:2.0:1616513096.962626:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.252@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 30352 00000400:00000200:2.0:1616513096.962631:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513096.962633:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513096.962641:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513096.962644:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513096.962646:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513096.962650:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688835691505472 00010000:00000200:12.0:1616513096.962660:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688835691505472/t0(0) o400->130ed0d0-26ba-006b-b1cb-a2a1cc86a230@10.31.141.252@tcp1:53/0 lens 224/224 e 0 to 0 dl 1616513188 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513096.962673:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688835691505472, offset 192 00000400:00000200:12.0:1616513096.962679:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.252@tcp1 00000400:00000200:12.0:1616513096.962686:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.252@tcp1 routed destination 00000400:00000200:12.0:1616513096.962699:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.252@tcp1(10.31.141.252@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513096.962706:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513096.962711:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513096.962713:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513096.962715:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513096.962720:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513096.962723:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688835691505600 00000100:00000200:12.0:1616513096.962726:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513096.962726:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513096.962736:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d1b00 x1688835691505600/t0(0) o400->daaa1c64-1f9e-03df-9d4c-448574f347b2@10.31.141.252@tcp1:722/0 lens 224/224 e 0 to 0 dl 1616513102 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513096.962750:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688835691505600, offset 192 00000400:00000200:22.0:1616513096.962756:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.252@tcp1 00000400:00000200:22.0:1616513096.962763:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.252@tcp1 routed destination 00000400:00000200:22.0:1616513096.962778:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.252@tcp1(10.31.141.252@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513096.962780:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513096.962785:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513096.962793:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513096.962793:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513096.962796:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513096.962796:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513096.962798:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513096.962802:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513096.962803:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513096.962807:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513096.962808:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa90 00000800:00000200:24.0:1616513096.962811:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513096.962814:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513096.962865:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513096.962872:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513096.962875:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513096.962876:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513096.962882:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513096.962886:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bed0 00000800:00000200:18.0:1616513096.962890:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.009013:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.009068:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.009078:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.009088:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.80@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.009099:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.80@tcp1 of length 224 into portal 26 MB=0x601887486fc80 00000400:00000200:16.0:1616513097.009106:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.80@tcp1 of length 224/224 into md 0xf97cfc19 [1] + 0 00000400:00000200:16.0:1616513097.009111:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3b18 00000400:00000200:16.0:1616513097.009114:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.009117:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.009123:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:0.0:1616513097.009124:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.009126:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.009129:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.009131:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.009188:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690535442447488 00000800:00000200:11.2:1616513097.009200:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:12.0:1616513097.009201:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1690535442447488/t0(0) o400->5ea6261a-f8bd-371b-9779-0513c4e57538@10.31.142.80@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.009215:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690535442447488, offset 192 00000400:00000200:12.0:1616513097.009221:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.80@tcp1 00000400:00000200:12.0:1616513097.009229:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.80@tcp1 routed destination 00000400:00000200:12.0:1616513097.009244:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.80@tcp1(10.31.142.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.009250:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513097.009256:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.009256:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.009258:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.009261:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513097.009265:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.009266:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:12.0:1616513097.009273:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513097.009274:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.80@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.009283:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.80@tcp1 of length 224 into portal 12 MB=0x601887486ff00 00000400:00000200:24.0:1616513097.009290:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.80@tcp1 of length 224/224 into md 0x865be291 [1] + 62360 00000400:00000200:24.0:1616513097.009295:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.009298:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513097.009305:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513097.009308:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513097.009309:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:24.0:1616513097.009316:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513097.009317:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513097.009319:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.009320:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.009327:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.009333:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa90 00000800:00000200:20.0:1616513097.009336:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.009379:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690535442448128 00010000:00000200:28.0:1616513097.009393:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cef980 x1690535442448128/t0(0) o400->790ed2e6-ccd9-9c5a-74fc-0c4c172bf42a@10.31.142.80@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.009408:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690535442448128, offset 192 00000400:00000200:28.0:1616513097.009414:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.80@tcp1 00000400:00000200:28.0:1616513097.009422:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.80@tcp1 routed destination 00000400:00000200:28.0:1616513097.009455:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.80@tcp1(10.31.142.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.009462:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.009469:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.009472:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.009474:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.009480:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.009486:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.009542:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.009549:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.009552:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.009558:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.009562:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc220 00000800:00000200:2.0:1616513097.009566:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.009569:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.018254:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.018308:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.018318:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513097.018318:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513097.018327:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.185@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.018336:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.185@tcp1 of length 224 into portal 26 MB=0x5ffd5f1452400 00000400:00000200:16.0:1616513097.018343:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.185@tcp1 of length 224/224 into md 0xf97cfc29 [1] + 0 00000400:00000200:16.0:1616513097.018348:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3c28 00000400:00000200:16.0:1616513097.018350:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.018353:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.018357:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3c28 00000800:00000200:16.0:1616513097.018360:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.018362:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.018363:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.018367:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513097.018386:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.018396:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.018405:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.185@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.018415:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.185@tcp1 of length 224 into portal 12 MB=0x5ffd5f1452500 00000400:00000200:14.0:1616513097.018421:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.185@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 30576 00000400:00000200:14.0:1616513097.018426:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513097.018428:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688669224510464 00000100:00000200:14.0:1616513097.018429:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513097.018446:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.018450:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00010000:00000200:12.0:1616513097.018450:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688669224510464/t0(0) o400->9ba29459-7fe8-eaf3-7c14-f16fd2af03c1@10.31.138.185@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513097.018453:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.018455:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.018464:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688669224510464, offset 192 00000400:00000200:12.0:1616513097.018472:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.185@tcp1 00000400:00000200:12.0:1616513097.018479:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.185@tcp1 routed destination 00000400:00000200:12.0:1616513097.018494:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.185@tcp1(10.31.138.185@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.018501:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:22.0:1616513097.018506:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688669224510720 00000800:00000200:12.0:1616513097.018506:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.018509:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.018511:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.018516:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513097.018519:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfad00 x1688669224510720/t0(0) o400->41d40723-389d-042a-3d13-6ffd3a572c49@10.31.138.185@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513097.018520:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.018523:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513097.018533:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688669224510720, offset 192 00000800:00000200:0.0:1616513097.018534:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513097.018540:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.185@tcp1 00000800:00000200:0.0:1616513097.018541:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.018543:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513097.018547:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.185@tcp1 routed destination 00000400:00000200:0.0:1616513097.018550:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.018556:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bea18 00000800:00000200:0.0:1616513097.018559:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513097.018562:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.185@tcp1(10.31.138.185@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513097.018562:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513097.018568:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.018574:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.018577:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.018579:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.018584:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.018588:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.018640:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.018648:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.018650:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.018657:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.018661:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aff0 00000800:00000200:2.0:1616513097.018665:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.018668:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.021681:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.021733:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.021742:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513097.021751:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.65@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.021760:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.65@tcp1 of length 224 into portal 26 MB=0x5ffd6ca929640 00000400:00000200:24.0:1616513097.021767:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.65@tcp1 of length 224/224 into md 0xf97cfc49 [1] + 0 00000400:00000200:24.0:1616513097.021772:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf38f8 00000400:00000200:24.0:1616513097.021774:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.021779:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513097.021783:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000800:00000200:24.0:1616513097.021786:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.021787:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.021789:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513097.021791:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.021846:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672870241856 00010000:00000200:12.0:1616513097.021857:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688672870241856/t0(0) o400->e14207ca-646d-bd4c-b6fe-a6f195d53a07@10.31.129.65@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.021871:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688672870241856, offset 192 00000400:00000200:12.0:1616513097.021877:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.65@tcp1 00000400:00000200:12.0:1616513097.021885:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.65@tcp1 routed destination 00000400:00000200:12.0:1616513097.021900:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.65@tcp1(10.31.129.65@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.021907:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.021912:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.021915:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.021917:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.021922:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.021926:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.021928:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513097.021979:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.021986:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.021995:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513097.022002:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.022008:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf980 00000800:00000200:0.0:1616513097.022012:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.022014:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.022021:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513097.022022:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.022028:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.65@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.022037:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.65@tcp1 of length 224 into portal 12 MB=0x5ffd6ca929980 00000400:00000200:0.0:1616513097.022043:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.65@tcp1 of length 224/224 into md 0x865be291 [1] + 62584 00000400:00000200:0.0:1616513097.022048:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.022050:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513097.022058:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.022060:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.022062:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.022084:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688672870242688 00010000:00000200:28.0:1616513097.022096:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86e780 x1688672870242688/t0(0) o400->14bb0b9d-d1b1-9e74-4048-5d9aabd0b98e@10.31.129.65@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.022110:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688672870242688, offset 192 00000400:00000200:28.0:1616513097.022118:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.65@tcp1 00000400:00000200:28.0:1616513097.022126:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.65@tcp1 routed destination 00000400:00000200:28.0:1616513097.022141:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.65@tcp1(10.31.129.65@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.022147:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.022153:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.022156:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.022158:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.022163:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.022167:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.022220:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.022227:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.022229:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513097.022236:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.022240:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc110 00000800:00000200:18.0:1616513097.022244:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.022248:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.022469:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513097.022473:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513097.022478:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.022483:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513097.022486:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513097.022491:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.148@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.022498:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.148@tcp1 of length 224 into portal 12 MB=0x603fcd2a6bd80 00000400:00000200:24.0:1616513097.022504:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.148@tcp1 of length 224/224 into md 0x865be291 [1] + 62808 00000400:00000200:24.0:1616513097.022508:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.022510:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513097.022516:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.022518:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.022519:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513097.022528:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513097.022529:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234261048704 00010000:00000200:28.0:1616513097.022535:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceda00 x1693234261048704/t0(0) o400->74869c6e-502a-b4ff-3529-60c3bea231e2@10.31.138.148@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513097.022540:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000100:00000200:28.0:1616513097.022546:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234261048704, offset 192 00000400:00000200:2.0:1616513097.022549:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.148@tcp1 : PUT - for me 00000400:00000200:28.0:1616513097.022552:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.148@tcp1 00000400:00000200:28.0:1616513097.022558:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.148@tcp1 routed destination 00000400:00000200:2.0:1616513097.022559:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.148@tcp1 of length 224 into portal 26 MB=0x603fcd2a6bd00 00000400:00000200:2.0:1616513097.022565:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.148@tcp1 of length 224/224 into md 0xf97cfc99 [1] + 0 00000400:00000200:2.0:1616513097.022570:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2000 00000400:00000200:28.0:1616513097.022571:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.148@tcp1(10.31.138.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513097.022573:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.022575:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:28.0:1616513097.022577:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513097.022580:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2000 00000800:00000200:28.0:1616513097.022582:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513097.022583:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513097.022584:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513097.022585:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.022586:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513097.022589:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.022595:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234261048576 00000800:00000200:28.0:1616513097.022596:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.022599:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:12.0:1616513097.022600:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1693234261048576/t0(0) o400->766aa7e3-8e20-3252-2b1c-6dc3596aac4c@10.31.138.148@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513097.022603:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.022611:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234261048576, offset 192 00000800:00000200:20.0:1616513097.022614:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513097.022615:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.148@tcp1 00000400:00000200:12.0:1616513097.022621:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.148@tcp1 routed destination 00000800:00000200:20.0:1616513097.022622:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.022624:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513097.022624:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.022631:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513097.022634:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.148@tcp1(10.31.138.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513097.022635:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd3a8 00000800:00000200:20.0:1616513097.022640:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513097.022643:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.022647:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.022650:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.022651:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.022654:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.022658:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.022659:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513097.022667:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.022671:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.022672:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513097.022676:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513097.022678:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.022682:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be6e8 00000800:00000200:18.0:1616513097.022684:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.027904:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513097.027958:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.027968:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.027977:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.45@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.027986:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.45@tcp1 of length 224 into portal 26 MB=0x60194bf315780 00000400:00000200:26.0:1616513097.027995:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.45@tcp1 of length 224/224 into md 0xf97cfca9 [1] + 0 00000400:00000200:26.0:1616513097.028000:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2770 00000400:00000200:26.0:1616513097.028002:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.028005:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513097.028009:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:2.0:1616513097.028011:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.028012:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.028015:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.028017:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.028070:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588234733440 00010000:00000200:12.0:1616513097.028080:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1690588234733440/t0(0) o400->2f1ad12d-12e3-3fec-26db-65a22323481a@10.31.141.45@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.028093:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588234733440, offset 192 00000400:00000200:12.0:1616513097.028099:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.45@tcp1 00000400:00000200:12.0:1616513097.028106:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.45@tcp1 routed destination 00000400:00000200:12.0:1616513097.028120:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.45@tcp1(10.31.141.45@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.028126:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.028131:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.028133:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.028136:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.028140:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.028146:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.028181:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.028193:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.028200:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513097.028202:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513097.028209:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513097.028214:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beee0 00000800:00000200:16.0:1616513097.028217:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.028220:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.028273:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.028325:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.028334:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.028342:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.45@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.028351:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.45@tcp1 of length 224 into portal 12 MB=0x60194bf315880 00000400:00000200:0.0:1616513097.028357:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.45@tcp1 of length 224/224 into md 0x865be291 [1] + 63032 00000400:00000200:0.0:1616513097.028362:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.028364:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513097.028370:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.028373:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.028374:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.028377:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.028432:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588234733696 00010000:00000200:28.0:1616513097.028454:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86c800 x1690588234733696/t0(0) o400->f557d763-ff2b-ba2a-d20c-e1ee33fc71e3@10.31.141.45@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.028467:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588234733696, offset 192 00000400:00000200:28.0:1616513097.028473:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.45@tcp1 00000400:00000200:28.0:1616513097.028480:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.45@tcp1 routed destination 00000400:00000200:28.0:1616513097.028494:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.45@tcp1(10.31.141.45@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.028500:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.028506:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.028510:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.028512:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.028516:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.028520:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.028572:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.028579:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.028582:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.028588:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.028592:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd4b8 00000800:00000200:14.0:1616513097.028596:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.028599:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.045802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513097.045856:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.045866:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513097.045875:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.140@tcp1 : PUT - for me 00000400:00000200:2.0:1616513097.045884:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.140@tcp1 of length 224 into portal 26 MB=0x5ffd7073cff40 00000400:00000200:2.0:1616513097.045890:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.140@tcp1 of length 224/224 into md 0xf97cfcc9 [1] + 0 00000400:00000200:2.0:1616513097.045895:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3188 00000400:00000200:2.0:1616513097.045898:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.045901:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513097.045905:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000800:00000200:2.0:1616513097.045908:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.045910:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.045911:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513097.045914:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.045966:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673888042816 00010000:00000200:12.0:1616513097.045976:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688673888042816/t0(0) o400->d859710f-98af-5985-d3f8-914f71e91d65@10.31.129.140@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.045990:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688673888042816, offset 192 00000400:00000200:12.0:1616513097.045995:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.140@tcp1 00000400:00000200:12.0:1616513097.046002:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.140@tcp1 routed destination 00000400:00000200:12.0:1616513097.046016:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.140@tcp1(10.31.129.140@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.046022:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.046027:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.046030:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.046033:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.046037:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.046044:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.046078:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.046090:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.046097:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513097.046100:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513097.046106:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513097.046111:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be2a8 00000800:00000200:16.0:1616513097.046114:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.046117:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.046183:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.046237:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.046246:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.046255:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.140@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.046264:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.140@tcp1 of length 224 into portal 12 MB=0x5ffd7073d0340 00000400:00000200:20.0:1616513097.046270:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.140@tcp1 of length 224/224 into md 0x865be291 [1] + 63256 00000400:00000200:20.0:1616513097.046277:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.046280:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513097.046293:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.046295:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.046296:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.046299:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.046351:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688673888043840 00010000:00000200:28.0:1616513097.046362:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c665a00 x1688673888043840/t0(0) o400->9319f1e4-8e08-e2a4-7669-0f880241f9f0@10.31.129.140@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.046375:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688673888043840, offset 192 00000400:00000200:28.0:1616513097.046381:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.140@tcp1 00000400:00000200:28.0:1616513097.046388:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.140@tcp1 routed destination 00000400:00000200:28.0:1616513097.046402:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.140@tcp1(10.31.129.140@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.046408:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.046413:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.046416:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.046418:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.046422:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.046427:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.046446:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.046452:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.046455:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.046462:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.046466:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd5c8 00000800:00000200:14.0:1616513097.046470:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.046473:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.075288:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.075346:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.075355:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.075365:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.11@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.075375:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.11@tcp1 of length 224 into portal 26 MB=0x5f8f9e9813950 00000400:00000200:16.0:1616513097.075382:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.11@tcp1 of length 224/224 into md 0xf97cfcd9 [1] + 0 00000400:00000200:16.0:1616513097.075386:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3e48 00000400:00000200:16.0:1616513097.075389:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.075392:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.075397:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000800:00000200:24.0:1616513097.075401:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.075401:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.075404:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.075406:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.075442:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:12.0:1616513097.075460:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681127131658576 00010000:00000200:12.0:1616513097.075470:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1681127131658576/t0(0) o400->667ab010-8498-6683-5189-57c554b31d03@10.31.132.11@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.075483:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681127131658576, offset 192 00000400:00000200:12.0:1616513097.075488:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.11@tcp1 00000400:00000200:12.0:1616513097.075495:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.11@tcp1 routed destination 00000800:00000200:0.0:1616513097.075498:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.075507:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:12.0:1616513097.075508:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.11@tcp1(10.31.132.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.075513:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.075517:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.219@tcp1 : PUT - for me 00000800:00000200:12.0:1616513097.075519:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.075521:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513097.075523:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:0.0:1616513097.075525:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.219@tcp1 of length 224 into portal 26 MB=0x5fff14da21740 00000800:00000200:12.0:1616513097.075528:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:0.0:1616513097.075532:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.219@tcp1 of length 224/224 into md 0xf97cfcf9 [1] + 0 00000100:00000200:12.0:1616513097.075534:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513097.075537:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e000 00000400:00000200:0.0:1616513097.075540:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.075543:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513097.075548:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e000 00000800:00000200:0.0:1616513097.075552:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513097.075555:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:0.0:1616513097.075555:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:12.0:1616513097.075560:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688786738222912 00000800:00000200:0.0:1616513097.075563:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513097.075564:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00010000:00000200:12.0:1616513097.075564:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688786738222912/t0(0) o400->0fed50f4-4c3a-8355-dd39-a25d8290a6d0@10.31.141.219@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513097.075565:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:20.0:1616513097.075567:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.075570:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.075571:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.075573:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:12.0:1616513097.075574:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688786738222912, offset 192 00000400:00000200:12.0:1616513097.075578:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.219@tcp1 00000400:00000200:0.0:1616513097.075578:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.219@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.075583:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfcb0 00000400:00000200:12.0:1616513097.075583:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.219@tcp1 routed destination 00000800:00000200:20.0:1616513097.075588:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.075591:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.219@tcp1 of length 224 into portal 12 MB=0x5fff14da21a00 00000400:00000200:12.0:1616513097.075595:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.219@tcp1(10.31.141.219@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513097.075596:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.219@tcp1 of length 224/224 into md 0x865be291 [1] + 63480 00000800:00000200:12.0:1616513097.075600:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513097.075603:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513097.075604:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:0.0:1616513097.075605:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513097.075606:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.075607:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.075610:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513097.075610:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.075612:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.075614:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513097.075614:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.075617:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513097.075673:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688786738223616 00000800:00000200:26.0:1616513097.075678:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00010000:00000200:28.0:1616513097.075685:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86d100 x1688786738223616/t0(0) o400->a4fd2db0-d819-0206-796e-94892dba3857@10.31.141.219@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:26.0:1616513097.075686:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.075688:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.075695:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:28.0:1616513097.075698:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688786738223616, offset 192 00000400:00000200:26.0:1616513097.075701:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf650 00000800:00000200:26.0:1616513097.075705:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513097.075706:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.219@tcp1 00000800:00000200:26.0:1616513097.075708:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513097.075713:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.219@tcp1 routed destination 00000400:00000200:28.0:1616513097.075727:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.219@tcp1(10.31.141.219@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.075733:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513097.075738:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.075741:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513097.075743:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.075747:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.075752:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.075763:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.075774:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.075776:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513097.075778:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513097.075783:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.075787:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd6d8 00000800:00000200:24.0:1616513097.075791:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.076780:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.076832:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.076852:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.076861:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.11@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.076870:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.11@tcp1 of length 224 into portal 12 MB=0x5f8f9e9813970 00000400:00000200:14.0:1616513097.076893:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.11@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 30800 00000400:00000200:14.0:1616513097.076897:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.076900:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513097.076907:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.076908:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.076909:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.076912:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.076928:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681127131658608 00010000:00000200:22.0:1616513097.076941:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfc380 x1681127131658608/t0(0) o400->c0890a42-8dbf-9c53-ba22-978e9586d616@10.31.132.11@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.076955:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681127131658608, offset 192 00000400:00000200:22.0:1616513097.076962:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.11@tcp1 00000400:00000200:22.0:1616513097.076970:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.11@tcp1 routed destination 00000400:00000200:22.0:1616513097.076984:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.11@tcp1(10.31.132.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.076991:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.076997:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.077000:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.077002:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.077007:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.077011:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.077063:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.077070:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.077073:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.077079:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.077083:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4be48 00000800:00000200:26.0:1616513097.077087:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.077090:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.079891:0:51274:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513097.079944:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.079954:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513097.079963:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.107@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.079972:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.107@tcp1 of length 224 into portal 26 MB=0x5fda82e70afc0 00000400:00000200:18.0:1616513097.079979:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.107@tcp1 of length 224/224 into md 0xf97cfd09 [1] + 0 00000400:00000200:18.0:1616513097.079984:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f7e8 00000400:00000200:18.0:1616513097.079986:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.079989:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513097.079993:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f7e8 00000800:00000200:18.0:1616513097.079996:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.079997:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.079999:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.080004:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.080007:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.080008:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.080017:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.107@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.080026:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.107@tcp1 of length 224 into portal 12 MB=0x5fda82e70b040 00000400:00000200:14.0:1616513097.080032:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.107@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 31024 00000400:00000200:14.0:1616513097.080037:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.080040:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513097.080047:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.080050:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.080052:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.080057:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1686273659023296 00000100:00000200:22.0:1616513097.080066:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1686273659023424 00010000:00000200:12.0:1616513097.080068:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1686273659023296/t0(0) o400->14b8eb5c-09a3-9323-3e02-6e470b301362@10.31.143.107@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:22.0:1616513097.080078:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfcc80 x1686273659023424/t0(0) o400->b2138d9a-2d6d-25fe-7439-85ea2c7c7ae0@10.31.143.107@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513097.080084:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1686273659023296, offset 192 00000400:00000200:12.0:1616513097.080090:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.107@tcp1 00000100:00000200:22.0:1616513097.080092:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1686273659023424, offset 192 00000400:00000200:12.0:1616513097.080098:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.107@tcp1 routed destination 00000400:00000200:22.0:1616513097.080099:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.107@tcp1 00000400:00000200:22.0:1616513097.080107:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.107@tcp1 routed destination 00000400:00000200:12.0:1616513097.080112:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.107@tcp1(10.31.143.107@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.080119:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513097.080122:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.107@tcp1(10.31.143.107@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.080125:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.080127:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.080129:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.080129:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.080134:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513097.080135:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.080138:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.080140:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.080141:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.080141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.080144:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.080148:0:51274:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513097.080196:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.080203:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.080206:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.080212:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513097.080215:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513097.080218:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfa90 00000800:00000200:20.0:1616513097.080221:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513097.080222:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513097.080224:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513097.080225:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513097.080226:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513097.080232:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.080236:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4be48 00000800:00000200:2.0:1616513097.080240:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.080625:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.080638:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.080647:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.080656:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.41@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.080665:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.41@tcp1 of length 224 into portal 26 MB=0x604d6e4282f40 00000400:00000200:16.0:1616513097.080672:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.41@tcp1 of length 224/224 into md 0xf97cfd29 [1] + 0 00000400:00000200:16.0:1616513097.080677:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f8f8 00000400:00000200:16.0:1616513097.080681:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.080684:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.080689:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f8f8 00000800:00000200:16.0:1616513097.080692:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.080693:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.080695:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.080700:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.080703:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.080704:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:12.0:1616513097.080704:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694170857615168 00010000:00000200:12.0:1616513097.080710:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1694170857615168/t0(0) o400->12bfbb1a-74ee-7d72-33f1-7c0f60125339@10.31.142.41@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:24.0:1616513097.080713:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.41@tcp1 : PUT - for me 00000100:00000200:12.0:1616513097.080721:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694170857615168, offset 192 00000400:00000200:24.0:1616513097.080722:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.41@tcp1 of length 224 into portal 12 MB=0x604d6e4283080 00000400:00000200:12.0:1616513097.080726:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.41@tcp1 00000400:00000200:24.0:1616513097.080731:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.41@tcp1 of length 224/224 into md 0x865be291 [1] + 63704 00000400:00000200:12.0:1616513097.080732:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.41@tcp1 routed destination 00000400:00000200:24.0:1616513097.080736:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.080739:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513097.080745:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.41@tcp1(10.31.142.41@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513097.080750:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.080751:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513097.080753:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.080755:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513097.080755:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.080758:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.080759:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.080762:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.080767:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.080767:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.080779:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.080787:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.080789:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513097.080790:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.080796:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.080802:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf7e8 00000800:00000200:0.0:1616513097.080805:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.080810:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694170857615488 00010000:00000200:28.0:1616513097.080822:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cef080 x1694170857615488/t0(0) o400->9973b1b3-6571-904b-450c-99f746f742fb@10.31.142.41@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.080847:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694170857615488, offset 192 00000400:00000200:28.0:1616513097.080853:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.41@tcp1 00000400:00000200:28.0:1616513097.080860:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.41@tcp1 routed destination 00000400:00000200:28.0:1616513097.080876:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.41@tcp1(10.31.142.41@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.080882:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.080888:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.080894:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.080896:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.080901:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.080905:0:51274:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.080915:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.080918:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.080921:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513097.080924:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513097.080926:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.080930:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd7e8 00000800:00000200:14.0:1616513097.080933:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.087532:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.087588:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.087597:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.087606:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.195@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.087616:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.195@tcp1 of length 224 into portal 26 MB=0x604a295c90340 00000400:00000200:20.0:1616513097.087622:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.195@tcp1 of length 224/224 into md 0xf97cfd49 [1] + 0 00000400:00000200:20.0:1616513097.087627:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e770 00000400:00000200:20.0:1616513097.087630:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.087633:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513097.087637:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e770 00000800:00000200:20.0:1616513097.087640:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.087641:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.087643:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.087648:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513097.087651:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513097.087651:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513097.087660:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.195@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.087669:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.195@tcp1 of length 224 into portal 12 MB=0x604a295c903c0 00000400:00000200:24.0:1616513097.087675:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.195@tcp1 of length 224/224 into md 0x865be291 [1] + 63928 00000400:00000200:24.0:1616513097.087681:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.087683:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513097.087689:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.087693:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.087694:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.087700:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946204455744 00010000:00000200:12.0:1616513097.087711:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1693946204455744/t0(0) o400->494f4090-4f61-c3a3-d5f3-b2db0b91238f@10.31.140.195@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.087726:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693946204455744, offset 192 00000400:00000200:12.0:1616513097.087732:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.195@tcp1 00000400:00000200:12.0:1616513097.087740:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.195@tcp1 routed destination 00000400:00000200:12.0:1616513097.087756:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.195@tcp1(10.31.140.195@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513097.087757:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693946204455872 00000800:00000200:12.0:1616513097.087763:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.087768:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:28.0:1616513097.087769:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8900 x1693946204455872/t0(0) o400->043c4316-8951-bb6e-76f9-70920af4213e@10.31.140.195@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513097.087771:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.087773:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.087778:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.087783:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:28.0:1616513097.087784:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693946204455872, offset 192 00000100:00000200:12.0:1616513097.087785:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513097.087790:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.195@tcp1 00000400:00000200:28.0:1616513097.087798:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.195@tcp1 routed destination 00000400:00000200:28.0:1616513097.087813:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.195@tcp1(10.31.140.195@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.087819:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.087825:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.087828:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.087830:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.087834:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513097.087835:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513097.087840:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513097.087842:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.087845:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513097.087846:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.087851:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.087856:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bff58 00000800:00000200:0.0:1616513097.087860:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513097.087894:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.087900:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.087903:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513097.087910:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.087914:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcda08 00000800:00000200:18.0:1616513097.087918:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.087921:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.104212:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513097.104268:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.104277:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.104286:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.51@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.104295:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.51@tcp1 of length 224 into portal 26 MB=0x5ffd4daaa8880 00000400:00000200:26.0:1616513097.104302:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.51@tcp1 of length 224/224 into md 0xf97cfd69 [1] + 0 00000400:00000200:26.0:1616513097.104307:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f188 00000400:00000200:26.0:1616513097.104310:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.104312:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513097.104316:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f188 00000800:00000200:26.0:1616513097.104321:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.104323:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.104324:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.104328:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.104391:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664550312064 00010000:00000200:12.0:1616513097.104401:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688664550312064/t0(0) o400->d801d4ab-c060-3e26-f604-4f700f02c396@10.31.138.51@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.104414:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664550312064, offset 192 00000400:00000200:12.0:1616513097.104420:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.51@tcp1 00000400:00000200:12.0:1616513097.104427:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.51@tcp1 routed destination 00000400:00000200:12.0:1616513097.104454:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.51@tcp1(10.31.138.51@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.104460:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.104466:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.104468:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.104470:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.104475:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.104481:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.104481:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.104534:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.104540:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.104543:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513097.104549:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.104554:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bec38 00000800:00000200:24.0:1616513097.104557:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.104560:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.104637:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.104649:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.104658:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.104668:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.51@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.104676:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.51@tcp1 of length 224 into portal 12 MB=0x5ffd4daaa8a80 00000400:00000200:20.0:1616513097.104682:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.51@tcp1 of length 224/224 into md 0x865be291 [1] + 64152 00000400:00000200:20.0:1616513097.104687:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.104689:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513097.104695:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.104698:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.104699:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.104702:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.104760:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664550312576 00010000:00000200:28.0:1616513097.104771:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c664800 x1688664550312576/t0(0) o400->872f0e94-83d9-43c8-eacb-8388ff2a7cf6@10.31.138.51@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.104784:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664550312576, offset 192 00000400:00000200:28.0:1616513097.104790:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.51@tcp1 00000400:00000200:28.0:1616513097.104797:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.51@tcp1 routed destination 00000400:00000200:28.0:1616513097.104811:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.51@tcp1(10.31.138.51@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.104818:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.104823:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.104826:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.104828:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.104833:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.104837:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.104888:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.104895:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.104897:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513097.104903:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.104907:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdb18 00000800:00000200:18.0:1616513097.104913:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.104916:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.105716:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.105784:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.105793:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.105802:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.214@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.105811:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.214@tcp1 of length 224 into portal 26 MB=0x5ffe5056c8980 00000400:00000200:14.0:1616513097.105818:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.214@tcp1 of length 224/224 into md 0xf97cfd79 [1] + 0 00000400:00000200:14.0:1616513097.105824:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e440 00000400:00000200:14.0:1616513097.105827:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.105830:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513097.105834:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e440 00000800:00000200:14.0:1616513097.105837:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.105838:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.105840:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.105845:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.105848:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.105849:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.105857:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.214@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.105866:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.214@tcp1 of length 224 into portal 12 MB=0x5ffe5056c8cc0 00000400:00000200:26.0:1616513097.105873:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.214@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 31248 00000400:00000200:26.0:1616513097.105878:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.105880:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513097.105887:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.105890:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.105892:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.105925:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688733987146112 00010000:00000200:12.0:1616513097.105936:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688733987146112/t0(0) o400->40fa3414-bfff-8044-87a5-6f2faf2c96b3@10.31.141.214@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.105951:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688733987146112, offset 192 00000100:00000200:22.0:1616513097.105956:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688733987146944 00000400:00000200:12.0:1616513097.105957:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.214@tcp1 00000400:00000200:12.0:1616513097.105964:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.214@tcp1 routed destination 00010000:00000200:22.0:1616513097.105968:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693a80 x1688733987146944/t0(0) o400->7da90d6c-0c40-96ce-c2c7-c4f64fd1fe84@10.31.141.214@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513097.105979:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.214@tcp1(10.31.141.214@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513097.105981:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688733987146944, offset 192 00000800:00000200:12.0:1616513097.105985:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513097.105987:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.214@tcp1 00000800:00000200:12.0:1616513097.105991:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513097.105993:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.214@tcp1 routed destination 00000800:00000200:12.0:1616513097.105995:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.105998:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.106002:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513097.106007:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.214@tcp1(10.31.141.214@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:12.0:1616513097.106009:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.106009:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.106013:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.106019:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.106022:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.106024:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.106028:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.106032:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.106043:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.106050:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.106053:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513097.106053:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.106062:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.106068:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513097.106068:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.106071:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.106072:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aff0 00000800:00000200:2.0:1616513097.106076:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513097.106078:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.106083:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be880 00000800:00000200:24.0:1616513097.106086:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.106089:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.132799:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513097.132855:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.132865:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513097.132875:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.99@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.132885:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.99@tcp1 of length 224 into portal 26 MB=0x5fff5c8dff800 00000400:00000200:18.0:1616513097.132892:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.99@tcp1 of length 224/224 into md 0xf97cfd99 [1] + 0 00000400:00000200:18.0:1616513097.132897:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777eff0 00000400:00000200:18.0:1616513097.132900:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.132903:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513097.132908:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eff0 00000800:00000200:26.0:1616513097.132911:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.132912:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.132915:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.132916:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.132974:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688805985744896 00010000:00000200:12.0:1616513097.132985:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688805985744896/t0(0) o400->c43f3f3b-0720-d0f4-9006-c2a5d000e5c2@10.31.136.99@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.133000:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688805985744896, offset 192 00000400:00000200:12.0:1616513097.133006:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.99@tcp1 00000400:00000200:12.0:1616513097.133014:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.99@tcp1 routed destination 00000400:00000200:12.0:1616513097.133029:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.99@tcp1(10.31.136.99@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.133036:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.133042:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.133044:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.133047:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.133052:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.133058:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.133058:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.133112:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.133120:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.133123:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513097.133125:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.133129:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.133135:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be5d8 00000800:00000200:0.0:1616513097.133138:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.133203:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.133256:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.133266:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.133274:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.99@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.133283:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.99@tcp1 of length 224 into portal 12 MB=0x5fff5c8dff900 00000400:00000200:14.0:1616513097.133289:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.99@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 31472 00000400:00000200:14.0:1616513097.133295:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.133297:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513097.133305:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.133308:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.133310:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.133313:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.133370:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688805985745152 00010000:00000200:22.0:1616513097.133382:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfba80 x1688805985745152/t0(0) o400->0d27cdc7-88d1-cccf-abc8-7f4f161eabb3@10.31.136.99@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.133395:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688805985745152, offset 192 00000400:00000200:22.0:1616513097.133401:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.99@tcp1 00000400:00000200:22.0:1616513097.133408:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.99@tcp1 routed destination 00000400:00000200:22.0:1616513097.133421:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.99@tcp1(10.31.136.99@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.133427:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.133433:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.133435:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.133453:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.133458:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.133461:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.133471:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.133477:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.133479:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513097.133482:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513097.133486:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.133496:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bed0 00000800:00000200:26.0:1616513097.133500:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.142330:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.142383:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.142393:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513097.142402:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.134@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.142411:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.134@tcp1 of length 224 into portal 26 MB=0x603fc5a402a40 00000400:00000200:24.0:1616513097.142417:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.134@tcp1 of length 224/224 into md 0xf97cfdc9 [1] + 0 00000400:00000200:24.0:1616513097.142422:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e7f8 00000400:00000200:24.0:1616513097.142425:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.142427:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513097.142432:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e7f8 00000800:00000200:16.0:1616513097.142452:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.142455:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.142458:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513097.142460:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.142462:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513097.142463:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513097.142471:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.134@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.142480:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.134@tcp1 of length 224 into portal 12 MB=0x603fc5a402ac0 00000400:00000200:16.0:1616513097.142486:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.134@tcp1 of length 224/224 into md 0x865be291 [1] + 64376 00000400:00000200:16.0:1616513097.142492:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.142494:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:12.0:1616513097.142494:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693232241060416 00000800:00000200:16.0:1616513097.142501:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:12.0:1616513097.142504:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1693232241060416/t0(0) o400->4b7b9e8c-6ff2-9324-15fb-a70279f1ef87@10.31.138.134@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513097.142505:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.142507:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.142517:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693232241060416, offset 192 00000400:00000200:12.0:1616513097.142523:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.134@tcp1 00000400:00000200:12.0:1616513097.142530:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.134@tcp1 routed destination 00000400:00000200:12.0:1616513097.142543:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.134@tcp1(10.31.138.134@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.142550:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.142555:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.142558:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.142560:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.142564:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513097.142566:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693232241060544 00000800:00000200:11.2:1616513097.142569:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.142570:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:28.0:1616513097.142579:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b107080 x1693232241060544/t0(0) o400->ca354380-a953-14ef-106d-47919181477f@10.31.138.134@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513097.142590:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:28.0:1616513097.142593:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693232241060544, offset 192 00000800:00000200:0.0:1616513097.142597:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513097.142599:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.134@tcp1 00000400:00000200:0.0:1616513097.142599:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.142600:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513097.142606:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.134@tcp1 routed destination 00000400:00000200:0.0:1616513097.142606:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.142611:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf870 00000800:00000200:0.0:1616513097.142615:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:28.0:1616513097.142621:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.134@tcp1(10.31.138.134@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.142628:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.142634:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.142636:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.142638:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.142643:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.142649:0:51282:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.142702:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.142709:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.142712:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.142718:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.142722:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdc28 00000800:00000200:14.0:1616513097.142726:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.142729:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.152511:0:51283:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513097.152567:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.152576:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513097.152585:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.191@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.152594:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.191@tcp1 of length 224 into portal 12 MB=0x6001bde6bddc0 00000400:00000200:18.0:1616513097.152601:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.191@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 31696 00000400:00000200:18.0:1616513097.152607:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.152609:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:11.2:1616513097.152611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513097.152616:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.152619:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.152621:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.152625:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513097.152664:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.152673:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.152682:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.191@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.152691:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.191@tcp1 of length 224 into portal 26 MB=0x6001bde6bdcc0 00000400:00000200:16.0:1616513097.152697:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.191@tcp1 of length 224/224 into md 0xf97cfdd9 [1] + 0 00000100:00000200:22.0:1616513097.152698:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969555992000 00000400:00000200:16.0:1616513097.152702:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e3b8 00000400:00000200:16.0:1616513097.152705:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.152708:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00010000:00000200:22.0:1616513097.152710:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464622d00 x1688969555992000/t0(0) o400->bab4dec9-d6bd-7345-23c6-464a5e107a50@10.31.138.191@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513097.152713:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e3b8 00000800:00000200:16.0:1616513097.152716:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.152717:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.152718:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.152720:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513097.152724:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688969555992000, offset 192 00000400:00000200:22.0:1616513097.152731:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.191@tcp1 00000400:00000200:22.0:1616513097.152738:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.191@tcp1 routed destination 00000400:00000200:22.0:1616513097.152753:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.191@tcp1(10.31.138.191@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.152760:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.152765:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.152768:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.152770:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.152775:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.152779:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969555991744 00000800:00000200:11.2:1616513097.152780:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:12.0:1616513097.152789:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688969555991744/t0(0) o400->cbdf0d10-5244-a157-311f-89319d08b9c9@10.31.138.191@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.152802:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688969555991744, offset 192 00000400:00000200:12.0:1616513097.152808:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.191@tcp1 00000400:00000200:12.0:1616513097.152814:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.191@tcp1 routed destination 00000400:00000200:12.0:1616513097.152829:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.191@tcp1(10.31.138.191@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513097.152831:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:12.0:1616513097.152835:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513097.152838:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.152840:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:12.0:1616513097.152840:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.152842:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.152844:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513097.152847:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:12.0:1616513097.152849:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513097.152851:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ac38 00000800:00000200:7.2:1616513097.152853:0:51283:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513097.152855:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.152855:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513097.152858:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513097.152865:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.152872:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.152874:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.152881:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.152886:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfa08 00000800:00000200:14.0:1616513097.152890:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.152893:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.159825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513097.159877:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.159886:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.159895:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.115@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.159905:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.115@tcp1 of length 224 into portal 26 MB=0x5f90cb906b340 00000400:00000200:26.0:1616513097.159912:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.115@tcp1 of length 224/224 into md 0xf97cfde9 [1] + 0 00000400:00000200:26.0:1616513097.159917:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e198 00000400:00000200:26.0:1616513097.159920:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.159922:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513097.159927:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e198 00000800:00000200:26.0:1616513097.159930:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.159930:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.159933:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.159936:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.159940:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513097.159941:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513097.159949:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.115@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.159958:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.115@tcp1 of length 224 into portal 12 MB=0x5f90cb906b3c0 00000400:00000200:18.0:1616513097.159965:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.115@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 31920 00000400:00000200:18.0:1616513097.159970:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.159972:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513097.159985:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.159986:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681207922701120 00000800:00000200:18.0:1616513097.159988:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.159990:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:12.0:1616513097.159996:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1681207922701120/t0(0) o400->40fd59f3-2375-da21-55a8-23fb3f15750e@10.31.143.115@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.160008:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681207922701120, offset 192 00000400:00000200:12.0:1616513097.160013:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.115@tcp1 00000400:00000200:12.0:1616513097.160020:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.115@tcp1 routed destination 00000400:00000200:12.0:1616513097.160032:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.115@tcp1(10.31.143.115@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.160038:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.160043:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.160045:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.160047:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513097.160049:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681207922701248 00000800:00000200:12.0:1616513097.160051:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.160057:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:22.0:1616513097.160061:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464626780 x1681207922701248/t0(0) o400->a1297c4d-9dd8-2167-4fcb-4112bb134af3@10.31.143.115@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.160075:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681207922701248, offset 192 00000400:00000200:22.0:1616513097.160081:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.115@tcp1 00000400:00000200:22.0:1616513097.160089:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.115@tcp1 routed destination 00000800:00000200:11.2:1616513097.160093:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513097.160104:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.115@tcp1(10.31.143.115@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.160110:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.160116:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.160119:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.160121:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.160126:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.160130:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513097.160147:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.160154:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.160156:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.160163:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.160168:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be110 00000800:00000200:20.0:1616513097.160172:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.160175:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513097.160181:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.160189:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.160191:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513097.160192:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513097.160198:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.160202:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b650 00000800:00000200:2.0:1616513097.160206:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.163926:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.163980:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.163990:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513097.163999:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.176@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.164008:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.176@tcp1 of length 224 into portal 26 MB=0x60184a098c500 00000400:00000200:24.0:1616513097.164014:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.176@tcp1 of length 224/224 into md 0xf97cfe09 [1] + 0 00000400:00000200:24.0:1616513097.164019:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ea18 00000400:00000200:24.0:1616513097.164022:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.164024:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513097.164029:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ea18 00000800:00000200:24.0:1616513097.164032:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.164034:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.164035:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513097.164038:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.164066:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:12.0:1616513097.164089:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519001941248 00010000:00000200:12.0:1616513097.164098:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1690519001941248/t0(0) o400->528a5d44-44f3-e928-5d9e-bad040e77898@10.31.129.176@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.164111:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690519001941248, offset 192 00000800:00000200:14.0:1616513097.164117:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513097.164118:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.176@tcp1 00000400:00000200:12.0:1616513097.164124:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.176@tcp1 routed destination 00000800:00000200:14.0:1616513097.164128:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.164136:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.176@tcp1 : PUT - for me 00000400:00000200:12.0:1616513097.164137:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.176@tcp1(10.31.129.176@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.164143:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513097.164144:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.176@tcp1 of length 224 into portal 12 MB=0x60184a098c980 00000800:00000200:12.0:1616513097.164148:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513097.164150:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.176@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 32144 00000800:00000200:12.0:1616513097.164150:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.164152:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513097.164155:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513097.164156:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:14.0:1616513097.164157:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:12.0:1616513097.164161:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.164161:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513097.164163:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.164166:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.164167:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.164170:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.164227:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690519001942400 00000800:00000200:20.0:1616513097.164232:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:22.0:1616513097.164239:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf9200 x1690519001942400/t0(0) o400->022b9999-8934-5419-42e8-c3b483a9f3f3@10.31.129.176@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513097.164239:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.164241:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.164248:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:22.0:1616513097.164253:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690519001942400, offset 192 00000400:00000200:20.0:1616513097.164254:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be330 00000800:00000200:20.0:1616513097.164257:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.164260:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513097.164261:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.176@tcp1 00000400:00000200:22.0:1616513097.164268:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.176@tcp1 routed destination 00000400:00000200:22.0:1616513097.164283:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.176@tcp1(10.31.129.176@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.164290:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.164295:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.164298:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.164300:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.164305:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.164309:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.164360:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.164367:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.164370:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.164377:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.164381:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aaa0 00000800:00000200:26.0:1616513097.164384:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.164387:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.165102:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.165113:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.165121:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513097.165122:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.165130:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.83@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.165139:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.83@tcp1 of length 224 into portal 26 MB=0x5ffd4139b75c0 00000400:00000200:0.0:1616513097.165148:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.83@tcp1 of length 224/224 into md 0xf97cfe19 [1] + 0 00000400:00000200:0.0:1616513097.165152:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e660 00000400:00000200:0.0:1616513097.165155:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.165158:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513097.165163:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e660 00000800:00000200:0.0:1616513097.165166:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.165169:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.165170:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.165174:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661210658240 00010000:00000200:12.0:1616513097.165179:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688661210658240/t0(0) o400->94e52fc8-d6d8-6c1c-ed9a-c7b8cd529989@10.31.129.83@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.165189:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661210658240, offset 192 00000400:00000200:12.0:1616513097.165194:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.83@tcp1 00000400:00000200:12.0:1616513097.165199:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.83@tcp1 routed destination 00000400:00000200:12.0:1616513097.165211:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.83@tcp1(10.31.129.83@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.165215:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513097.165218:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.165220:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.165222:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.165223:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.165226:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513097.165229:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.165229:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.165230:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513097.165237:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:20.0:1616513097.165239:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513097.165239:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513097.165243:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.165245:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513097.165246:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.83@tcp1 : PUT - for me 00000800:00000200:16.0:1616513097.165248:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.165250:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.165253:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfd38 00000400:00000200:18.0:1616513097.165254:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.83@tcp1 of length 224 into portal 12 MB=0x5ffd4139b79c0 00000800:00000200:20.0:1616513097.165255:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513097.165260:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.83@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 32368 00000400:00000200:18.0:1616513097.165265:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.165267:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513097.165273:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.165276:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.165278:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.165284:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661210659264 00010000:00000200:22.0:1616513097.165290:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464625e80 x1688661210659264/t0(0) o400->00a945a1-759c-a503-e19f-5cd1ef61ead8@10.31.129.83@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.165301:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661210659264, offset 192 00000400:00000200:22.0:1616513097.165306:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.83@tcp1 00000400:00000200:22.0:1616513097.165311:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.83@tcp1 routed destination 00000400:00000200:22.0:1616513097.165323:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.83@tcp1(10.31.129.83@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.165328:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.165332:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.165335:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.165336:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.165339:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.165343:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.165351:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.165354:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.165356:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513097.165360:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513097.165361:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.165364:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b320 00000800:00000200:26.0:1616513097.165366:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.167057:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.167112:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.167122:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513097.167130:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.124@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.167140:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.124@tcp1 of length 224 into portal 26 MB=0x5ffd3a1705d00 00000400:00000200:24.0:1616513097.167147:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.124@tcp1 of length 224/224 into md 0xf97cfe39 [1] + 0 00000400:00000200:24.0:1616513097.167151:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f760 00000400:00000200:24.0:1616513097.167154:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.167157:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513097.167163:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f760 00000800:00000200:24.0:1616513097.167166:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.167166:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.167169:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513097.167171:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.167198:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.167206:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.167211:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.167219:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.124@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.167226:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.124@tcp1 of length 224 into portal 12 MB=0x5ffd3a1705e80 00000100:00000200:12.0:1616513097.167229:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659295231232 00000400:00000200:14.0:1616513097.167231:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.124@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 32592 00000400:00000200:14.0:1616513097.167236:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.167238:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:12.0:1616513097.167240:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688659295231232/t0(0) o400->a7684c8f-c002-320e-dcc8-6aab1f21f750@10.31.136.124@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513097.167242:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.167244:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.167246:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.167248:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.167254:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688659295231232, offset 192 00000400:00000200:12.0:1616513097.167260:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.124@tcp1 00000400:00000200:12.0:1616513097.167277:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.124@tcp1 routed destination 00000400:00000200:12.0:1616513097.167290:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.124@tcp1(10.31.136.124@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.167300:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.167305:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.167307:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.167309:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.167313:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.167318:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.167319:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513097.167329:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659295231616 00010000:00000200:22.0:1616513097.167340:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf8480 x1688659295231616/t0(0) o400->a39265d0-faba-2945-f15b-d7bb8b502a1f@10.31.136.124@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.167354:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688659295231616, offset 192 00000400:00000200:22.0:1616513097.167359:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.124@tcp1 00000400:00000200:22.0:1616513097.167367:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.124@tcp1 routed destination 00000800:00000200:16.0:1616513097.167372:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.167379:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513097.167381:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.124@tcp1(10.31.136.124@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513097.167383:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513097.167387:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513097.167390:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513097.167392:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:16.0:1616513097.167395:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfba0 00000800:00000200:22.0:1616513097.167397:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.167399:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513097.167399:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.167402:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513097.167403:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.167407:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.167461:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.167468:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.167471:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513097.167472:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513097.167478:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.167482:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba08 00000800:00000200:2.0:1616513097.167486:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.168771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.168822:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.168830:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.168838:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.240@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.168846:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.240@tcp1 of length 224 into portal 12 MB=0x60182bc40b600 00000400:00000200:14.0:1616513097.168852:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.240@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 32816 00000400:00000200:14.0:1616513097.168857:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.168859:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513097.168865:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.168867:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.168869:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.168874:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.168925:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510875997696 00010000:00000200:22.0:1616513097.168936:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bfec00 x1690510875997696/t0(0) o400->9124b41b-2380-60e6-8644-a5faaf2769e7@10.31.132.240@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.168949:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510875997696, offset 192 00000400:00000200:22.0:1616513097.168955:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.240@tcp1 00000400:00000200:22.0:1616513097.168962:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.240@tcp1 routed destination 00000400:00000200:22.0:1616513097.168975:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.240@tcp1(10.31.132.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.168981:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.168986:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.168989:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.168991:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.168995:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.169039:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.169094:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.169101:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.169103:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513097.169110:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.169114:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a110 00000800:00000200:0.0:1616513097.169118:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.169121:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.169965:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.170018:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.170027:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.170036:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.240@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.170044:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.240@tcp1 of length 224 into portal 26 MB=0x60182bc40b580 00000400:00000200:16.0:1616513097.170051:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.240@tcp1 of length 224/224 into md 0xf97cfe59 [1] + 0 00000400:00000200:16.0:1616513097.170055:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fc28 00000400:00000200:16.0:1616513097.170058:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.170061:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.170065:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fc28 00000800:00000200:16.0:1616513097.170068:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.170071:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.170072:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.170076:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.170123:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510875997568 00010000:00000200:12.0:1616513097.170132:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1690510875997568/t0(0) o400->b17954f8-7655-541a-9218-23fb972a9281@10.31.132.240@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.170145:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510875997568, offset 192 00000400:00000200:12.0:1616513097.170150:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.240@tcp1 00000400:00000200:12.0:1616513097.170156:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.240@tcp1 routed destination 00000400:00000200:12.0:1616513097.170169:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.240@tcp1(10.31.132.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.170175:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.170179:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.170182:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.170184:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.170188:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.170192:0:51285:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.170193:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513097.170243:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.170249:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.170252:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.170258:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.170263:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be550 00000800:00000200:26.0:1616513097.170267:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.170269:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.215444:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513097.215499:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.215509:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513097.215512:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513097.215518:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.110@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.215529:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.110@tcp1 of length 224 into portal 26 MB=0x6059813e2c800 00000400:00000200:18.0:1616513097.215535:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.110@tcp1 of length 224/224 into md 0xf97cfe69 [1] + 0 00000400:00000200:18.0:1616513097.215540:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ef68 00000400:00000200:18.0:1616513097.215543:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.215546:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513097.215550:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ef68 00000800:00000200:18.0:1616513097.215554:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.215556:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.215558:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.215613:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695000587061248 00010000:00000200:12.0:1616513097.215623:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1695000587061248/t0(0) o400->621fc4b6-ebf6-80af-0a8e-e066bd337684@10.31.136.110@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.215636:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695000587061248, offset 192 00000400:00000200:12.0:1616513097.215642:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.110@tcp1 00000400:00000200:12.0:1616513097.215649:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.110@tcp1 routed destination 00000400:00000200:12.0:1616513097.215662:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.110@tcp1(10.31.136.110@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.215668:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.215673:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.215675:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.215679:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.215683:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.215690:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.215693:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.215748:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.215756:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.215758:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.215760:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.215765:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.215771:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bef68 00000800:00000200:0.0:1616513097.215774:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.216527:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.216583:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.216593:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.216604:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.110@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.216613:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.110@tcp1 of length 224 into portal 12 MB=0x6059813e2c940 00000400:00000200:20.0:1616513097.216620:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.110@tcp1 of length 224/224 into md 0x865be291 [1] + 64600 00000400:00000200:20.0:1616513097.216625:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.216628:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513097.216634:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.216637:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.216638:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.216643:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.216699:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695000587061568 00010000:00000200:28.0:1616513097.216712:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c665e80 x1695000587061568/t0(0) o400->082c53dc-557a-c769-a907-60c3379b2744@10.31.136.110@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.216727:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695000587061568, offset 192 00000400:00000200:28.0:1616513097.216733:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.110@tcp1 00000400:00000200:28.0:1616513097.216740:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.110@tcp1 routed destination 00000400:00000200:28.0:1616513097.216755:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.110@tcp1(10.31.136.110@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.216762:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.216768:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.216771:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.216773:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.216778:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.216817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.216872:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.216880:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.216883:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.216890:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.216893:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdd38 00000800:00000200:26.0:1616513097.216898:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.216901:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.247089:0:51293:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.247145:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.247156:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.247166:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.89@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.247177:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.89@tcp1 of length 224 into portal 26 MB=0x5ffd579ae3140 00000400:00000200:14.0:1616513097.247184:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.89@tcp1 of length 224/224 into md 0xf97cfe99 [1] + 0 00000400:00000200:14.0:1616513097.247189:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f980 00000400:00000200:14.0:1616513097.247192:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.247194:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513097.247199:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f980 00000800:00000200:18.0:1616513097.247200:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.247202:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.247205:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.247207:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.247268:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667218129216 00010000:00000200:12.0:1616513097.247279:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688667218129216/t0(0) o400->01338576-07f1-8e43-40c7-eee3c62b7b17@10.31.136.89@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.247294:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667218129216, offset 192 00000400:00000200:12.0:1616513097.247301:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.89@tcp1 00000400:00000200:12.0:1616513097.247309:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.89@tcp1 routed destination 00000400:00000200:12.0:1616513097.247324:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.89@tcp1(10.31.136.89@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.247331:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.247337:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.247340:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.247342:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.247348:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.247355:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.247355:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.247369:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.247376:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.247379:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513097.247386:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.247392:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf210 00000800:00000200:24.0:1616513097.247395:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.247398:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.248028:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.248082:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.248091:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.248100:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.89@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.248110:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.89@tcp1 of length 224 into portal 12 MB=0x5ffd579ae31c0 00000400:00000200:16.0:1616513097.248116:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.89@tcp1 of length 224/224 into md 0x865be291 [1] + 64824 00000400:00000200:16.0:1616513097.248121:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.248124:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.248132:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.248135:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.248135:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.248139:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.248197:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667218129344 00010000:00000200:28.0:1616513097.248210:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d1680 x1688667218129344/t0(0) o400->766efb8b-26ae-8c9b-4721-c5c0249d1b3d@10.31.136.89@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.248225:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667218129344, offset 192 00000400:00000200:28.0:1616513097.248231:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.89@tcp1 00000400:00000200:28.0:1616513097.248238:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.89@tcp1 routed destination 00000400:00000200:28.0:1616513097.248253:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.89@tcp1(10.31.136.89@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.248260:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.248267:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.248269:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.248272:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.248277:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.248286:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.248339:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.248347:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.248349:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.248356:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.248360:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdf58 00000800:00000200:26.0:1616513097.248366:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.248369:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.256640:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.256706:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.256715:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513097.256724:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.186@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.256733:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.186@tcp1 of length 224 into portal 26 MB=0x60183627e6a80 00000400:00000200:24.0:1616513097.256740:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.186@tcp1 of length 224/224 into md 0xf97cfec9 [1] + 0 00000400:00000200:24.0:1616513097.256745:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e2a8 00000400:00000200:24.0:1616513097.256747:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.256750:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513097.256754:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e2a8 00000800:00000200:24.0:1616513097.256757:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.256760:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.256760:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513097.256764:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.256781:0:51294:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:12.0:1616513097.256818:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513665059456 00010000:00000200:12.0:1616513097.256829:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1690513665059456/t0(0) o400->68d64690-31bc-8c15-7da1-934ad02a3d7e@10.31.132.186@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513097.256835:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.256843:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690513665059456, offset 192 00000800:00000200:18.0:1616513097.256845:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513097.256849:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.186@tcp1 00000400:00000200:18.0:1616513097.256854:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.186@tcp1 : PUT - for me 00000400:00000200:12.0:1616513097.256857:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.186@tcp1 routed destination 00000400:00000200:18.0:1616513097.256863:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.186@tcp1 of length 224 into portal 12 MB=0x60183627e6bc0 00000400:00000200:18.0:1616513097.256870:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.186@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 33040 00000400:00000200:12.0:1616513097.256872:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.186@tcp1(10.31.132.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513097.256875:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:12.0:1616513097.256879:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:18.0:1616513097.256880:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513097.256884:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.256887:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513097.256888:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.256889:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513097.256889:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513097.256891:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.256893:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513097.256895:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.256899:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.256901:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513097.256959:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.256966:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.256969:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:22.0:1616513097.256975:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690513665059776 00000400:00000200:20.0:1616513097.256976:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.256981:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfed0 00000800:00000200:20.0:1616513097.256984:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513097.256987:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464622880 x1690513665059776/t0(0) o400->e36eae56-f618-2a21-d8fc-4b955624683f@10.31.132.186@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513097.256987:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513097.257001:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690513665059776, offset 192 00000400:00000200:22.0:1616513097.257006:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.186@tcp1 00000400:00000200:22.0:1616513097.257013:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.186@tcp1 routed destination 00000400:00000200:22.0:1616513097.257027:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.186@tcp1(10.31.132.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.257033:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.257039:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.257041:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.257043:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.257048:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.257052:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.257105:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.257112:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.257114:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.257121:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.257125:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a4c8 00000800:00000200:26.0:1616513097.257129:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.257132:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.263598:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.263653:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.263662:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.263672:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.89@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.263682:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.89@tcp1 of length 224 into portal 26 MB=0x601aa4ec21e40 00000400:00000200:0.0:1616513097.263688:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.89@tcp1 of length 224/224 into md 0xf97cfee9 [1] + 0 00000400:00000200:0.0:1616513097.263693:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fdc0 00000400:00000200:0.0:1616513097.263696:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.263699:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513097.263703:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fdc0 00000800:00000200:0.0:1616513097.263706:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.263707:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.263711:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.263713:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.263767:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690680837676608 00010000:00000200:12.0:1616513097.263778:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1690680837676608/t0(0) o400->c6997ea3-6336-a326-3fff-c676f65a49cf@10.31.140.89@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.263795:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690680837676608, offset 192 00000400:00000200:12.0:1616513097.263801:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.89@tcp1 00000400:00000200:12.0:1616513097.263810:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.89@tcp1 routed destination 00000400:00000200:12.0:1616513097.263825:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.89@tcp1(10.31.140.89@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.263831:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.263837:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.263839:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.263842:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.263846:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.263851:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.263853:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513097.263904:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.263910:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.263913:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.263920:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.263925:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be660 00000800:00000200:20.0:1616513097.263929:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.263932:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.264254:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.264264:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.264272:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513097.264274:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513097.264281:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.89@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.264301:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.89@tcp1 of length 224 into portal 12 MB=0x601aa4ec22240 00000400:00000200:24.0:1616513097.264307:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.89@tcp1 of length 224/224 into md 0x865be291 [1] + 65048 00000400:00000200:24.0:1616513097.264311:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.264314:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513097.264320:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.264324:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.264326:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.264384:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690680837677632 00010000:00000200:28.0:1616513097.264397:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce9b00 x1690680837677632/t0(0) o400->ac01f08c-6db5-f6a7-2d6d-4add6f3eb909@10.31.140.89@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.264412:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690680837677632, offset 192 00000400:00000200:28.0:1616513097.264418:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.89@tcp1 00000400:00000200:28.0:1616513097.264425:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.89@tcp1 routed destination 00000400:00000200:28.0:1616513097.264456:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.89@tcp1(10.31.140.89@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.264463:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.264469:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.264472:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.264474:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.264479:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.264483:0:51295:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.264536:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.264543:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.264546:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.264553:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.264557:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdf58 00000800:00000200:14.0:1616513097.264561:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.264564:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.268345:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513097.268399:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.268409:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.268418:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.118@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.268428:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.118@tcp1 of length 224 into portal 26 MB=0x603da55084540 00000400:00000200:26.0:1616513097.268449:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.118@tcp1 of length 224/224 into md 0xf97cff09 [1] + 0 00000800:00000200:2.0:1616513097.268453:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513097.268454:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e330 00000400:00000200:26.0:1616513097.268457:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.268460:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513097.268464:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e330 00000800:00000200:26.0:1616513097.268467:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.268470:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.268472:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.268525:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:12.0:1616513097.268528:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693086124623168 00010000:00000200:12.0:1616513097.268538:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1693086124623168/t0(0) o400->f259b593-17ee-1349-17cf-722422c520c2@10.31.141.118@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.268552:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693086124623168, offset 192 00000400:00000200:12.0:1616513097.268559:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.118@tcp1 00000400:00000200:12.0:1616513097.268567:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.118@tcp1 routed destination 00000400:00000200:12.0:1616513097.268581:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.118@tcp1(10.31.141.118@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.268588:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513097.268594:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.268594:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.268596:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.268599:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513097.268603:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.268603:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:12.0:1616513097.268610:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513097.268612:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.118@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.268626:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.118@tcp1 of length 224 into portal 12 MB=0x603da55084900 00000400:00000200:20.0:1616513097.268632:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.118@tcp1 of length 224/224 into md 0x865be291 [1] + 65272 00000400:00000200:20.0:1616513097.268638:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.268640:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513097.268647:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513097.268650:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513097.268652:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.268656:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.268659:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.268662:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513097.268663:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.268667:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.268671:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be990 00000800:00000200:20.0:1616513097.268674:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.268717:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693086124624128 00010000:00000200:28.0:1616513097.268729:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c665100 x1693086124624128/t0(0) o400->373f65c7-7906-6ac9-207a-7d37957c485c@10.31.141.118@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.268744:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693086124624128, offset 192 00000400:00000200:28.0:1616513097.268750:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.118@tcp1 00000400:00000200:28.0:1616513097.268757:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.118@tcp1 routed destination 00000400:00000200:28.0:1616513097.268774:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.118@tcp1(10.31.141.118@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.268780:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.268786:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.268789:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.268791:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.268796:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.268800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.268852:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.268858:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.268861:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.268868:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.268872:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdd38 00000800:00000200:14.0:1616513097.268876:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.268879:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.308231:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.308287:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.308308:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513097.308317:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.186@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.308326:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.186@tcp1 of length 224 into portal 26 MB=0x60182956bc500 00000400:00000200:24.0:1616513097.308333:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.186@tcp1 of length 224/224 into md 0xf97cff29 [1] + 0 00000400:00000200:24.0:1616513097.308337:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f210 00000400:00000200:24.0:1616513097.308340:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.308343:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513097.308347:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f210 00000800:00000200:24.0:1616513097.308351:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.308352:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.308354:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513097.308356:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.308411:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510224508160 00010000:00000200:12.0:1616513097.308423:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1690510224508160/t0(0) o400->3d5f0cac-774f-e2e2-18b7-2ecdcf69aef6@10.31.140.186@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.308450:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510224508160, offset 192 00000400:00000200:12.0:1616513097.308457:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.186@tcp1 00000400:00000200:12.0:1616513097.308464:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.186@tcp1 routed destination 00000400:00000200:12.0:1616513097.308481:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.186@tcp1(10.31.140.186@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.308487:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.308493:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.308496:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.308498:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.308503:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.308508:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.308510:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513097.308561:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.308568:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.308571:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.308578:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.308584:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bedd0 00000800:00000200:20.0:1616513097.308587:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.308590:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.308796:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513097.308853:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.308863:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513097.308872:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.186@tcp1 : PUT - for me 00000400:00000200:2.0:1616513097.308881:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.186@tcp1 of length 224 into portal 12 MB=0x60182956bc980 00000400:00000200:2.0:1616513097.308887:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.186@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 33264 00000400:00000200:2.0:1616513097.308893:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.308895:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513097.308910:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.308911:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.308913:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513097.308915:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.308979:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510224509312 00010000:00000200:22.0:1616513097.308992:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d7980 x1690510224509312/t0(0) o400->17b2e9ef-fbc9-83bd-95fb-d02193658d78@10.31.140.186@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.309017:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510224509312, offset 192 00000400:00000200:22.0:1616513097.309024:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.186@tcp1 00000400:00000200:22.0:1616513097.309031:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.186@tcp1 routed destination 00000400:00000200:22.0:1616513097.309046:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.186@tcp1(10.31.140.186@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.309053:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.309059:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.309062:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.309064:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.309070:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.309074:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.309084:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.309091:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.309094:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.309101:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.309105:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a908 00000800:00000200:14.0:1616513097.309110:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.309113:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.317649:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513097.317703:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.317712:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.317721:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.247@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.317731:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.247@tcp1 of length 224 into portal 26 MB=0x604192fcd46c0 00000400:00000200:26.0:1616513097.317737:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.247@tcp1 of length 224/224 into md 0xf97cff49 [1] + 0 00000400:00000200:26.0:1616513097.317742:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fb18 00000400:00000200:26.0:1616513097.317745:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.317748:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513097.317752:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fb18 00000800:00000200:26.0:1616513097.317755:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.317757:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.317758:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.317764:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.317767:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.317768:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513097.317777:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.247@tcp1 : PUT - for me 00000400:00000200:2.0:1616513097.317786:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.247@tcp1 of length 224 into portal 12 MB=0x604192fcd4a80 00000400:00000200:2.0:1616513097.317793:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.247@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 33488 00000400:00000200:2.0:1616513097.317800:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.317803:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513097.317809:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.317813:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.317814:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.317818:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356082939584 00010000:00000200:12.0:1616513097.317829:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1693356082939584/t0(0) o400->2285178c-787d-4979-2bc3-199ab9742dcc@10.31.128.247@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.317844:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693356082939584, offset 192 00000400:00000200:12.0:1616513097.317850:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.247@tcp1 00000400:00000200:12.0:1616513097.317858:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.247@tcp1 routed destination 00000400:00000200:12.0:1616513097.317873:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.247@tcp1(10.31.128.247@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513097.317874:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356082940544 00000800:00000200:12.0:1616513097.317879:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.317885:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513097.317886:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf3d0d80 x1693356082940544/t0(0) o400->23d3c885-84e9-c914-1e9b-db50f844a0a4@10.31.128.247@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513097.317888:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.317890:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.317895:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513097.317901:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693356082940544, offset 192 00000100:00000200:12.0:1616513097.317902:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.317902:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513097.317907:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.247@tcp1 00000400:00000200:22.0:1616513097.317915:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.247@tcp1 routed destination 00000400:00000200:22.0:1616513097.317930:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.247@tcp1(10.31.128.247@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.317937:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.317942:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.317945:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.317947:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.317952:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.317956:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513097.317957:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.317964:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513097.317967:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.317969:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513097.317970:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:16.0:1616513097.317976:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513097.317977:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.317980:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513097.317981:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513097.317981:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be7f8 00000800:00000200:16.0:1616513097.317985:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513097.317987:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.317991:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a000 00000800:00000200:18.0:1616513097.317994:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.342485:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.342539:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.342549:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.342558:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.13@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.342569:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.13@tcp1 of length 224 into portal 26 MB=0x603d1734e1600 00000400:00000200:20.0:1616513097.342576:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.21.13@tcp1 of length 224/224 into md 0xf97cff59 [1] + 0 00000400:00000200:20.0:1616513097.342581:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fed0 00000400:00000200:20.0:1616513097.342584:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.342586:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513097.342591:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fed0 00000800:00000200:0.0:1616513097.342593:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.342594:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.342598:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513097.342600:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.342603:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.342603:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.342611:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.13@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.342621:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.13@tcp1 of length 224 into portal 12 MB=0x603d1734e19c0 00000400:00000200:0.0:1616513097.342627:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.21.13@tcp1 of length 224/224 into md 0x865be291 [1] + 65496 00000400:00000200:0.0:1616513097.342632:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.342635:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513097.342641:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.342645:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.342646:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.342680:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693047977809408 00010000:00000200:12.0:1616513097.342693:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1693047977809408/t0(0) o400->7bc5c3f4-4bae-ebcf-b7a3-7c476c7827b7@10.31.21.13@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.342707:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693047977809408, offset 192 00000400:00000200:12.0:1616513097.342713:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.13@tcp1 00000100:00000200:28.0:1616513097.342716:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693047977810368 00000400:00000200:12.0:1616513097.342721:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.13@tcp1 routed destination 00010000:00000200:28.0:1616513097.342730:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde869f80 x1693047977810368/t0(0) o400->6666dc47-ae5c-9379-ac2e-22b416a942d2@10.31.21.13@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513097.342736:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.13@tcp1(10.31.21.13@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.342743:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:28.0:1616513097.342744:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693047977810368, offset 192 00000400:00000200:28.0:1616513097.342750:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.13@tcp1 00000800:00000200:12.0:1616513097.342750:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.342754:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.342756:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513097.342759:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.13@tcp1 routed destination 00000800:00000200:12.0:1616513097.342762:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.342766:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.342769:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513097.342774:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.13@tcp1(10.31.21.13@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.342780:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.342787:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.342789:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.342792:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.342797:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.342804:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513097.342820:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.342828:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513097.342830:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513097.342831:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513097.342837:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513097.342843:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf3a8 00000800:00000200:16.0:1616513097.342846:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513097.342859:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.342866:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.342869:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.342876:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.342880:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdc28 00000800:00000200:2.0:1616513097.342884:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.342887:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.344141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.344211:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.344221:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.344229:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.151@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.344240:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.151@tcp1 of length 224 into portal 26 MB=0x602169647cb80 00000400:00000200:0.0:1616513097.344246:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.151@tcp1 of length 224/224 into md 0xf97cff79 [1] + 0 00000400:00000200:0.0:1616513097.344251:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f4b8 00000400:00000200:0.0:1616513097.344254:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.344257:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513097.344261:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f4b8 00000800:00000200:24.0:1616513097.344264:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.344264:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.344267:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.344269:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.344331:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145894087552 00010000:00000200:12.0:1616513097.344342:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1691145894087552/t0(0) o400->69467242-871c-763a-34d3-a314f482e108@10.31.140.151@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.344356:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691145894087552, offset 192 00000400:00000200:12.0:1616513097.344362:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.151@tcp1 00000400:00000200:12.0:1616513097.344370:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.151@tcp1 routed destination 00000400:00000200:12.0:1616513097.344385:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.151@tcp1(10.31.140.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.344392:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.344397:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.344400:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.344402:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.344407:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.344411:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.344413:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513097.344463:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.344470:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.344472:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.344479:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.344485:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bee58 00000800:00000200:20.0:1616513097.344488:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.344491:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.344676:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.344687:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.344695:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513097.344697:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513097.344704:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.151@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.344713:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.151@tcp1 of length 224 into portal 12 MB=0x602169647d000 00000400:00000200:24.0:1616513097.344719:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.151@tcp1 of length 224/224 into md 0x865be291 [1] + 65720 00000400:00000200:24.0:1616513097.344724:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.344727:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513097.344733:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.344736:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.344738:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.344797:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691145894088704 00010000:00000200:28.0:1616513097.344809:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8480 x1691145894088704/t0(0) o400->efb42e0c-79c4-7d89-33a3-25e7c2d6b41f@10.31.140.151@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.344824:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691145894088704, offset 192 00000400:00000200:28.0:1616513097.344830:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.151@tcp1 00000400:00000200:28.0:1616513097.344838:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.151@tcp1 routed destination 00000400:00000200:28.0:1616513097.344853:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.151@tcp1(10.31.140.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.344860:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.344865:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.344868:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.344872:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.344877:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.344881:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.344934:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.344942:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.344944:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.344951:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.344955:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdb18 00000800:00000200:26.0:1616513097.344959:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.344962:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.361700:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513097.361755:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.361764:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513097.361774:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.20@tcp1 : PUT - for me 00000400:00000200:2.0:1616513097.361784:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.20@tcp1 of length 224 into portal 26 MB=0x5fff3420a52c0 00000400:00000200:2.0:1616513097.361790:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.20@tcp1 of length 224/224 into md 0xf97cff89 [1] + 0 00000400:00000200:2.0:1616513097.361795:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777eee0 00000400:00000200:2.0:1616513097.361798:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.361801:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513097.361805:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eee0 00000800:00000200:14.0:1616513097.361808:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.361808:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.361811:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513097.361813:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.361869:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795133661888 00010000:00000200:12.0:1616513097.361880:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688795133661888/t0(0) o400->d3a705de-084b-d7da-01ec-50cf8735f666@10.31.136.20@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.361894:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688795133661888, offset 192 00000400:00000200:12.0:1616513097.361901:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.20@tcp1 00000400:00000200:12.0:1616513097.361908:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.20@tcp1 routed destination 00000400:00000200:12.0:1616513097.361923:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.20@tcp1(10.31.136.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.361929:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.361935:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.361938:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.361940:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.361945:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.361950:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.361951:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513097.362004:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.362011:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.362014:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.362021:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.362026:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bee58 00000800:00000200:20.0:1616513097.362030:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513097.362030:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513097.362033:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513097.362087:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.362097:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.362105:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.20@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.362114:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.20@tcp1 of length 224 into portal 12 MB=0x5fff3420a5400 00000400:00000200:26.0:1616513097.362121:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.20@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 33712 00000400:00000200:26.0:1616513097.362126:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.362128:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513097.362136:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.362139:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.362141:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.362144:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.362204:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795133662208 00010000:00000200:22.0:1616513097.362216:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693600 x1688795133662208/t0(0) o400->7b1378de-43a3-7cf3-bdb2-b3d902a7bacb@10.31.136.20@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.362231:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688795133662208, offset 192 00000400:00000200:22.0:1616513097.362237:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.20@tcp1 00000400:00000200:22.0:1616513097.362245:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.20@tcp1 routed destination 00000400:00000200:22.0:1616513097.362259:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.20@tcp1(10.31.136.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.362268:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.362274:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.362277:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.362279:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.362284:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.362288:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.362297:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.362302:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.362305:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513097.362307:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513097.362311:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.362315:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ba90 00000800:00000200:14.0:1616513097.362319:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.373011:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.373066:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.373075:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.373085:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.33@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.373094:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.33@tcp1 of length 224 into portal 26 MB=0x5ffe74acd2340 00000400:00000200:0.0:1616513097.373101:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.33@tcp1 of length 224/224 into md 0xf97cffb9 [1] + 0 00000400:00000200:0.0:1616513097.373105:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f650 00000400:00000200:0.0:1616513097.373110:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.373113:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513097.373117:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f650 00000800:00000200:24.0:1616513097.373119:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.373120:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.373123:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.373125:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.373179:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688743741039424 00010000:00000200:12.0:1616513097.373189:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688743741039424/t0(0) o400->a59fa261-dea7-3adb-35b7-5b37c41eb2a4@10.31.142.33@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.373202:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688743741039424, offset 192 00000400:00000200:12.0:1616513097.373208:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.33@tcp1 00000400:00000200:12.0:1616513097.373215:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.33@tcp1 routed destination 00000400:00000200:12.0:1616513097.373229:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.33@tcp1(10.31.142.33@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.373235:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.373240:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.373242:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.373245:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.373249:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.373253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.373255:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513097.373305:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.373312:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.373315:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.373322:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.373327:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be7f8 00000800:00000200:20.0:1616513097.373331:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.373334:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.373590:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.373600:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.373608:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513097.373611:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513097.373618:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.33@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.373626:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.33@tcp1 of length 224 into portal 12 MB=0x5ffe74acd2f00 00000400:00000200:24.0:1616513097.373633:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.33@tcp1 of length 224/224 into md 0x865be291 [1] + 65944 00000400:00000200:24.0:1616513097.373640:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.373642:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513097.373649:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.373652:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.373654:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.373713:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688743741042432 00010000:00000200:28.0:1616513097.373725:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceb180 x1688743741042432/t0(0) o400->2147a5cd-7f49-a583-1572-a9893497543d@10.31.142.33@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.373739:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688743741042432, offset 192 00000400:00000200:28.0:1616513097.373745:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.33@tcp1 00000400:00000200:28.0:1616513097.373753:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.33@tcp1 routed destination 00000400:00000200:28.0:1616513097.373768:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.33@tcp1(10.31.142.33@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.373774:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.373781:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.373784:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.373785:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.373790:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.373795:0:51308:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.373849:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.373857:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.373859:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.373866:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.373870:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcda08 00000800:00000200:26.0:1616513097.373874:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.373877:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.395732:0:5993:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.395787:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.395797:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.395805:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.223@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.395815:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.223@tcp1 of length 224 into portal 26 MB=0x5ffe3951fe880 00000400:00000200:20.0:1616513097.395821:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.223@tcp1 of length 224/224 into md 0xf97cffc9 [1] + 0 00000400:00000200:20.0:1616513097.395826:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fd38 00000400:00000200:20.0:1616513097.395829:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.395832:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513097.395836:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fd38 00000800:00000200:20.0:1616513097.395839:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.395843:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.395843:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513097.395851:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.395854:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.395854:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.395863:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.223@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.395872:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.223@tcp1 of length 224 into portal 12 MB=0x5ffe3951fe900 00000400:00000200:16.0:1616513097.395879:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.223@tcp1 of length 224/224 into md 0x865be291 [1] + 66168 00000400:00000200:16.0:1616513097.395885:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.395887:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.395894:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.395898:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.395899:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727808108672 00000800:00000200:16.0:1616513097.395900:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:12.0:1616513097.395909:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688727808108672/t0(0) o400->61cbcd09-817f-375f-70c3-9f4f9179769c@10.31.141.223@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.395922:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688727808108672, offset 192 00000400:00000200:12.0:1616513097.395929:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.223@tcp1 00000400:00000200:12.0:1616513097.395935:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.223@tcp1 routed destination 00000400:00000200:12.0:1616513097.395950:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.223@tcp1(10.31.141.223@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.395956:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:28.0:1616513097.395959:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688727808108800 00000800:00000200:12.0:1616513097.395961:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.395963:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.395965:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.395971:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:28.0:1616513097.395972:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5580 x1688727808108800/t0(0) o400->16cb6aad-d5d4-907f-8bf8-0c3c88f569a6@10.31.141.223@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513097.395974:0:5993:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.395977:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513097.395986:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688727808108800, offset 192 00000800:00000200:24.0:1616513097.395986:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:28.0:1616513097.395992:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.223@tcp1 00000800:00000200:24.0:1616513097.395993:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.395996:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513097.395997:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513097.396000:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.223@tcp1 routed destination 00000400:00000200:24.0:1616513097.396003:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.396008:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be990 00000800:00000200:24.0:1616513097.396012:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:28.0:1616513097.396015:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.223@tcp1(10.31.141.223@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.396022:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.396028:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.396031:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.396033:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.396038:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.396043:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.396098:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.396105:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.396108:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.396115:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.396119:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd7e8 00000800:00000200:2.0:1616513097.396123:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.396126:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.410043:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.410097:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.410106:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.410115:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.125@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.410125:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.125@tcp1 of length 224 into portal 26 MB=0x60198fea71dc0 00000400:00000200:16.0:1616513097.410132:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.125@tcp1 of length 224/224 into md 0xf97cffe9 [1] + 0 00000400:00000200:16.0:1616513097.410137:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f320 00000400:00000200:16.0:1616513097.410139:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.410142:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.410146:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f320 00000800:00000200:16.0:1616513097.410149:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.410151:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.410152:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.410155:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.410209:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690606479285696 00010000:00000200:12.0:1616513097.410220:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1690606479285696/t0(0) o400->e0121669-bad6-5f46-dc73-c8053b65119c@10.31.140.125@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.410234:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690606479285696, offset 192 00000400:00000200:12.0:1616513097.410242:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.125@tcp1 00000400:00000200:12.0:1616513097.410250:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.125@tcp1 routed destination 00000400:00000200:12.0:1616513097.410265:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.125@tcp1(10.31.140.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.410271:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.410277:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.410280:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.410282:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.410287:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.410291:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513097.410291:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:12.0:1616513097.410294:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513097.410344:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.410351:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.410354:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.410356:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.410360:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513097.410364:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513097.410366:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfed0 00000800:00000200:20.0:1616513097.410369:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513097.410373:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.410382:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.125@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.410391:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.125@tcp1 of length 224 into portal 12 MB=0x60198fea723c0 00000400:00000200:26.0:1616513097.410398:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.125@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 33936 00000400:00000200:26.0:1616513097.410405:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.410408:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513097.410415:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.410417:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.410418:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.410422:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.410481:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690606479287232 00010000:00000200:22.0:1616513097.410493:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691f80 x1690606479287232/t0(0) o400->e931ae3b-3a3a-e910-bc7f-7439b90eb9bb@10.31.140.125@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.410508:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690606479287232, offset 192 00000400:00000200:22.0:1616513097.410514:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.125@tcp1 00000400:00000200:22.0:1616513097.410521:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.125@tcp1 routed destination 00000400:00000200:22.0:1616513097.410536:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.125@tcp1(10.31.140.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.410542:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.410548:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.410551:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.410553:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.410558:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.410562:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.410614:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.410621:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.410625:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.410627:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.410634:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.410638:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4aee0 00000800:00000200:2.0:1616513097.410642:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.430822:0:51315:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513097.430877:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.430887:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513097.430896:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.22@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.430907:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.22@tcp1 of length 224 into portal 12 MB=0x5ffd5757b2640 00000400:00000200:18.0:1616513097.430914:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.22@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 34160 00000400:00000200:18.0:1616513097.430919:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.430922:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513097.430929:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.430932:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.430933:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.430937:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.430995:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667147675200 00010000:00000200:22.0:1616513097.431007:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464620480 x1688667147675200/t0(0) o400->dccf286d-11b8-30b9-e7e5-11eaefb3d159@10.31.136.22@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.431022:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667147675200, offset 192 00000400:00000200:22.0:1616513097.431028:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.22@tcp1 00000400:00000200:22.0:1616513097.431037:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.22@tcp1 routed destination 00000800:00000200:11.2:1616513097.431041:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:22.0:1616513097.431052:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.22@tcp1(10.31.136.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.431058:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513097.431058:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.431064:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.431067:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.431067:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.431069:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:22.0:1616513097.431074:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:0.0:1616513097.431082:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.22@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.431091:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.22@tcp1 of length 224 into portal 26 MB=0x5ffd5757b24c0 00000400:00000200:0.0:1616513097.431098:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.22@tcp1 of length 224/224 into md 0xf97d0009 [1] + 0 00000400:00000200:0.0:1616513097.431103:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ecc0 00000400:00000200:0.0:1616513097.431106:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.431109:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:16.0:1616513097.431113:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:0.0:1616513097.431113:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ecc0 00000800:00000200:0.0:1616513097.431117:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:0.0:1616513097.431119:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:16.0:1616513097.431121:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (25)-- 00000800:00000200:0.0:1616513097.431121:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513097.431124:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513097.431125:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513097.431132:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513097.431136:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a550 00000800:00000200:16.0:1616513097.431140:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.431181:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667147674816 00010000:00000200:12.0:1616513097.431191:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688667147674816/t0(0) o400->20afae92-f3b3-2aba-0e5e-71d15f6fd137@10.31.136.22@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.431206:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667147674816, offset 192 00000400:00000200:12.0:1616513097.431212:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.22@tcp1 00000400:00000200:12.0:1616513097.431220:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.22@tcp1 routed destination 00000400:00000200:12.0:1616513097.431235:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.22@tcp1(10.31.136.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.431241:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.431247:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.431250:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.431252:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.431257:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.431261:0:51315:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.431264:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513097.431272:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.431279:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.431281:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.431288:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.431293:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bef68 00000800:00000200:14.0:1616513097.431296:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.431299:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.432324:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.432381:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.432390:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.432399:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.37@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.432408:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.37@tcp1 of length 224 into portal 26 MB=0x5ffd3d68f85c0 00000400:00000200:20.0:1616513097.432414:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.37@tcp1 of length 224/224 into md 0xf97d0019 [1] + 0 00000400:00000200:20.0:1616513097.432419:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f870 00000400:00000200:20.0:1616513097.432422:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.432424:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513097.432429:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f870 00000800:00000200:20.0:1616513097.432432:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.432445:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.432450:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.432453:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.432524:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660186465728 00010000:00000200:12.0:1616513097.432534:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688660186465728/t0(0) o400->97cf0e5f-37b3-2183-fe03-bd8c493a4ae6@10.31.129.37@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.432549:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660186465728, offset 192 00000400:00000200:12.0:1616513097.432555:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.37@tcp1 00000400:00000200:12.0:1616513097.432562:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.37@tcp1 routed destination 00000400:00000200:12.0:1616513097.432577:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.37@tcp1(10.31.129.37@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.432584:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.2:1616513097.432587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.432591:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.432594:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.432596:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.432601:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.432605:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.432607:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513097.432641:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.432650:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.432659:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.37@tcp1 : PUT - for me 00000800:00000200:16.0:1616513097.432659:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.432666:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513097.432667:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.37@tcp1 of length 224 into portal 12 MB=0x5ffd3d68f8980 00000400:00000200:16.0:1616513097.432669:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.432674:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.37@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 34384 00000400:00000200:16.0:1616513097.432676:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.432679:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.432681:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:16.0:1616513097.432681:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfba0 00000800:00000200:16.0:1616513097.432685:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513097.432688:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513097.432688:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513097.432691:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.432693:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.432695:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.432755:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660186466688 00010000:00000200:22.0:1616513097.432767:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c693180 x1688660186466688/t0(0) o400->941f8f31-2888-0c63-b528-40eb46061b6e@10.31.129.37@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.432781:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660186466688, offset 192 00000400:00000200:22.0:1616513097.432788:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.37@tcp1 00000400:00000200:22.0:1616513097.432796:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.37@tcp1 routed destination 00000400:00000200:22.0:1616513097.432811:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.37@tcp1(10.31.129.37@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.432817:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.432823:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.432826:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.432828:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.432832:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.432836:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.432847:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.432854:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.432857:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.432864:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.432868:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b980 00000800:00000200:14.0:1616513097.432872:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.432875:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.461586:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513097.461600:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.461609:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.461618:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.179@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.461627:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.179@tcp1 of length 224 into portal 26 MB=0x5ffdf3334dec0 00000400:00000200:26.0:1616513097.461634:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.179@tcp1 of length 224/224 into md 0xf97d0039 [1] + 0 00000400:00000200:26.0:1616513097.461638:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e908 00000400:00000200:26.0:1616513097.461640:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.461643:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513097.461647:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e908 00000800:00000200:26.0:1616513097.461650:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.461652:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.461654:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.461657:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.461712:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708985446080 00010000:00000200:12.0:1616513097.461723:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688708985446080/t0(0) o400->c313ecb4-56f4-f9e4-8205-59d856497292@10.31.132.179@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.461738:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688708985446080, offset 192 00000400:00000200:12.0:1616513097.461744:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.179@tcp1 00000400:00000200:12.0:1616513097.461752:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.179@tcp1 routed destination 00000400:00000200:12.0:1616513097.461767:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.179@tcp1(10.31.132.179@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.461774:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.461779:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.461782:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.461784:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.461790:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.461796:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.461798:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.461854:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.461861:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.461864:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513097.461870:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.461876:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be330 00000800:00000200:0.0:1616513097.461879:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.461882:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.461990:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.462048:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.462058:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.462067:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.179@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.462077:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.179@tcp1 of length 224 into portal 12 MB=0x5ffdf3334e040 00000400:00000200:16.0:1616513097.462083:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.179@tcp1 of length 224/224 into md 0x865be291 [1] + 66392 00000400:00000200:16.0:1616513097.462088:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.462091:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.462099:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.462102:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513097.462103:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.462107:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.462122:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708985446464 00010000:00000200:28.0:1616513097.462135:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d4c80 x1688708985446464/t0(0) o400->3ec52b1b-3efc-ad1a-a84f-5e12c1a6749c@10.31.132.179@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.462150:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688708985446464, offset 192 00000400:00000200:28.0:1616513097.462158:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.179@tcp1 00000400:00000200:28.0:1616513097.462165:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.179@tcp1 routed destination 00000400:00000200:28.0:1616513097.462180:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.179@tcp1(10.31.132.179@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.462187:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.462193:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.462196:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.462198:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.462203:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.462209:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.462262:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.462269:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.462271:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.462278:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.462281:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd6d8 00000800:00000200:14.0:1616513097.462285:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.462288:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.480766:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.480823:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.480832:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.480841:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.81@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.480850:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.81@tcp1 of length 224 into portal 26 MB=0x5ffd4e455fac0 00000400:00000200:0.0:1616513097.480857:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.81@tcp1 of length 224/224 into md 0xf97d0049 [1] + 0 00000400:00000200:0.0:1616513097.480864:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e990 00000400:00000200:0.0:1616513097.480867:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.480869:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513097.480874:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e990 00000800:00000200:20.0:1616513097.480876:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.480877:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.480880:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513097.480883:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.480886:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.480887:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.480895:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.81@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.480904:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.81@tcp1 of length 224 into portal 12 MB=0x5ffd4e455fb40 00000400:00000200:20.0:1616513097.480911:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.81@tcp1 of length 224/224 into md 0x865be291 [1] + 66616 00000400:00000200:20.0:1616513097.480916:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.480919:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513097.480926:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.480929:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.480931:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.480937:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664712542912 00010000:00000200:12.0:1616513097.480947:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688664712542912/t0(0) o400->16357870-2781-14f6-32a8-24210cf838d9@10.31.136.81@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.480960:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664712542912, offset 192 00000400:00000200:12.0:1616513097.480966:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.81@tcp1 00000400:00000200:12.0:1616513097.480973:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.81@tcp1 routed destination 00000400:00000200:12.0:1616513097.480986:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.81@tcp1(10.31.136.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513097.480990:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664712543040 00000800:00000200:12.0:1616513097.480992:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513097.480998:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.481000:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.481002:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:28.0:1616513097.481003:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c667080 x1688664712543040/t0(0) o400->826afbaf-e32c-190e-624a-b051c6b91e17@10.31.136.81@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513097.481007:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.481012:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513097.481014:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513097.481017:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664712543040, offset 192 00000400:00000200:28.0:1616513097.481024:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.81@tcp1 00000400:00000200:28.0:1616513097.481031:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.81@tcp1 routed destination 00000400:00000200:28.0:1616513097.481046:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.81@tcp1(10.31.136.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.481052:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.481058:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.481061:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.481063:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513097.481066:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513097.481068:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513097.481073:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513097.481074:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513097.481076:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513097.481077:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513097.481083:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513097.481088:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfa08 00000800:00000200:16.0:1616513097.481092:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513097.481125:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.481132:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.481134:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.481140:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.481144:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd5c8 00000800:00000200:26.0:1616513097.481150:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.481153:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.550450:0:51328:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.550506:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.550517:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.550526:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.131@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.550538:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.131@tcp1 of length 224 into portal 26 MB=0x603e437ec5000 00000400:00000200:14.0:1616513097.550546:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.131@tcp1 of length 224/224 into md 0xf97d0069 [1] + 0 00000400:00000200:14.0:1616513097.550551:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f540 00000400:00000200:14.0:1616513097.550554:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.550557:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513097.550562:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f540 00000800:00000200:2.0:1616513097.550562:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.550565:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.550568:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.550570:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.550603:0:51328:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:12.0:1616513097.550632:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693128585924608 00010000:00000200:12.0:1616513097.550646:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1693128585924608/t0(0) o400->9e2aad2d-fe48-48b2-9519-2246d9526426@10.31.132.131@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513097.550656:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.550662:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693128585924608, offset 192 00000800:00000200:26.0:1616513097.550665:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:12.0:1616513097.550669:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.131@tcp1 00000400:00000200:26.0:1616513097.550673:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.131@tcp1 : PUT - for me 00000400:00000200:12.0:1616513097.550678:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.131@tcp1 routed destination 00000400:00000200:26.0:1616513097.550681:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.131@tcp1 of length 224 into portal 12 MB=0x603e437ec50c0 00000400:00000200:26.0:1616513097.550698:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.131@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 34608 00000400:00000200:26.0:1616513097.550703:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.550706:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513097.550706:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.131@tcp1(10.31.132.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.550712:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513097.550714:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.550717:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.550717:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:12.0:1616513097.550719:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513097.550721:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513097.550722:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.550724:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.550730:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513097.550737:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513097.550738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513097.550779:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693128585924800 00010000:00000200:22.0:1616513097.550792:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691200 x1693128585924800/t0(0) o400->7ddca444-5cfb-8121-a71b-376138e1049a@10.31.132.131@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513097.550793:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.550801:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.550803:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:22.0:1616513097.550806:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693128585924800, offset 192 00000400:00000200:20.0:1616513097.550810:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513097.550812:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.131@tcp1 00000400:00000200:20.0:1616513097.550816:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be5d8 00000400:00000200:22.0:1616513097.550819:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.131@tcp1 routed destination 00000800:00000200:20.0:1616513097.550819:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.550822:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513097.550833:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.131@tcp1(10.31.132.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.550840:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.550845:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.550848:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.550850:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.550855:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.550859:0:51328:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.550869:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.550874:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.550877:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513097.550878:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513097.550884:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.550888:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b870 00000800:00000200:2.0:1616513097.550892:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.573761:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.573818:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.573828:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.573837:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.21.11@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.573849:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.21.11@tcp1 of length 440 into portal 12 MB=0x603d0eaf2f5c0 00000400:00000200:0.0:1616513097.573855:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.21.11@tcp1 of length 440/440 into md 0x865be291 [1] + 66840 00000400:00000200:0.0:1616513097.573861:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.573864:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513097.573871:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.573872:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.573874:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.573877:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.573937:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693045690136000 00010000:00000200:28.0:1616513097.574026:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86b600 x1693045690136000/t0(0) o41->818c9703-1177-c410-c401-4735e4ab1220@10.31.21.11@tcp1:723/0 lens 440/368 e 0 to 0 dl 1616513103 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513097.574041:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693045690136000, offset 192 00000400:00000200:28.0:1616513097.574048:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.21.11@tcp1 00000400:00000200:28.0:1616513097.574056:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.21.11@tcp1 routed destination 00000400:00000200:28.0:1616513097.574072:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.21.11@tcp1(10.31.21.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.574078:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513097.574084:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.574089:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513097.574091:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.574097:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.574101:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.574155:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.574162:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.574165:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.574172:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.574178:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd4b8 00000800:00000200:20.0:1616513097.574182:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.574185:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.577728:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.577784:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.577793:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.577802:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.176@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.577813:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.176@tcp1 of length 224 into portal 26 MB=0x5ffd3d4c8e480 00000400:00000200:16.0:1616513097.577820:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.176@tcp1 of length 224/224 into md 0xf97d0089 [1] + 0 00000400:00000200:16.0:1616513097.577826:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ebb0 00000400:00000200:16.0:1616513097.577829:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.577832:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.577837:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ebb0 00000800:00000200:16.0:1616513097.577841:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.577842:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.577843:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.577846:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.577902:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660156671104 00010000:00000200:12.0:1616513097.577913:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688660156671104/t0(0) o400->e3aeb34b-30a5-6fbf-eaef-cf6018114a42@10.31.128.176@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.577927:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660156671104, offset 192 00000400:00000200:12.0:1616513097.577933:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.176@tcp1 00000400:00000200:12.0:1616513097.577941:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.176@tcp1 routed destination 00000400:00000200:12.0:1616513097.577956:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.176@tcp1(10.31.128.176@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.577962:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.577969:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.577972:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.577974:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.577979:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513097.577986:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513097.577986:0:51331:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.578055:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.578062:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.578065:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513097.578071:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.578077:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bec38 00000800:00000200:18.0:1616513097.578081:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.578084:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.578108:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.578160:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.578169:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.578178:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.176@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.578187:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.176@tcp1 of length 224 into portal 12 MB=0x5ffd3d4c8e800 00000400:00000200:20.0:1616513097.578194:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.176@tcp1 of length 224/224 into md 0x865be291 [1] + 67280 00000400:00000200:20.0:1616513097.578199:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.578202:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513097.578208:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.578211:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.578213:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.578216:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.578272:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660156672000 00010000:00000200:28.0:1616513097.578284:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661f80 x1688660156672000/t0(0) o400->408ae192-7b28-60ea-c768-3dc5fc77d752@10.31.128.176@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.578299:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660156672000, offset 192 00000400:00000200:28.0:1616513097.578305:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.176@tcp1 00000400:00000200:28.0:1616513097.578312:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.176@tcp1 routed destination 00000400:00000200:28.0:1616513097.578327:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.176@tcp1(10.31.128.176@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.578334:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513097.578340:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.578343:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513097.578345:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.578350:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.578355:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.578365:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.578371:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.578373:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513097.578375:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.578379:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.578383:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd3a8 00000800:00000200:0.0:1616513097.578387:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.582489:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.582545:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.582554:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.582563:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.158@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.582572:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.158@tcp1 of length 224 into portal 26 MB=0x6019e0c584180 00000400:00000200:20.0:1616513097.582579:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.158@tcp1 of length 224/224 into md 0xf97d00a9 [1] + 0 00000400:00000200:20.0:1616513097.582583:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fcb0 00000400:00000200:20.0:1616513097.582586:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.582589:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513097.582593:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fcb0 00000800:00000200:20.0:1616513097.582596:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.582597:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.582599:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.582601:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.582655:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690628183835008 00010000:00000200:12.0:1616513097.582666:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1690628183835008/t0(0) o400->90585914-e6f7-d2c8-9a3e-8fa9216acc66@10.31.140.158@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.582680:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690628183835008, offset 192 00000400:00000200:12.0:1616513097.582689:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.158@tcp1 00000400:00000200:12.0:1616513097.582697:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.158@tcp1 routed destination 00000400:00000200:12.0:1616513097.582712:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.158@tcp1(10.31.140.158@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.582719:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.582724:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.582727:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.582729:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.582734:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.582740:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.582741:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513097.582801:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.582808:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.582811:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.582817:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.582823:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf7e8 00000800:00000200:14.0:1616513097.582826:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.582829:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.582843:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.582897:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.582907:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513097.582909:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513097.582916:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.158@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.582924:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.158@tcp1 of length 224 into portal 12 MB=0x6019e0c5844c0 00000400:00000200:16.0:1616513097.582931:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.158@tcp1 of length 224/224 into md 0x865be291 [1] + 67504 00000400:00000200:16.0:1616513097.582936:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.582939:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.582945:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.582949:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.582950:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.583009:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690628183835840 00010000:00000200:28.0:1616513097.583022:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d6c00 x1690628183835840/t0(0) o400->a4727e39-8d52-b345-e7d9-97e3e2769798@10.31.140.158@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.583037:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690628183835840, offset 192 00000400:00000200:28.0:1616513097.583043:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.158@tcp1 00000400:00000200:28.0:1616513097.583051:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.158@tcp1 routed destination 00000400:00000200:28.0:1616513097.583066:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.158@tcp1(10.31.140.158@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.583072:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513097.583078:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.583081:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513097.583083:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.583087:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.583092:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.583102:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.583108:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.583110:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513097.583112:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513097.583117:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.583121:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc110 00000800:00000200:24.0:1616513097.583125:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.610134:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513097.610189:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.610200:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513097.610209:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.173@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.610221:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.173@tcp1 of length 224 into portal 26 MB=0x6026e0b73c3c0 00000400:00000200:18.0:1616513097.610228:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.173@tcp1 of length 224/224 into md 0xf97d00c9 [1] + 0 00000400:00000200:18.0:1616513097.610232:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ec38 00000400:00000200:18.0:1616513097.610235:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.610238:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513097.610243:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ec38 00000800:00000200:26.0:1616513097.610244:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.610246:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.610249:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.610251:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.610309:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691521522058176 00010000:00000200:12.0:1616513097.610320:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1691521522058176/t0(0) o400->00ad7ff2-0607-f6f4-2144-daa21c2738c1@10.31.129.173@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.610335:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691521522058176, offset 192 00000400:00000200:12.0:1616513097.610341:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.173@tcp1 00000400:00000200:12.0:1616513097.610349:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.173@tcp1 routed destination 00000400:00000200:12.0:1616513097.610364:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.173@tcp1(10.31.129.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.610371:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.610377:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.610380:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.610382:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513097.610387:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513097.610388:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.610392:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.610394:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513097.610401:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.610411:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.610420:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.173@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.610428:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.173@tcp1 of length 224 into portal 12 MB=0x6026e0b73c900 00000400:00000200:0.0:1616513097.610447:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.173@tcp1 of length 224/224 into md 0x865be291 [1] + 67728 00000400:00000200:0.0:1616513097.610452:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513097.610455:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:0.0:1616513097.610457:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.610462:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513097.610463:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513097.610465:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513097.610466:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513097.610468:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.610469:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513097.610472:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.610478:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf650 00000800:00000200:14.0:1616513097.610482:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.610484:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513097.610547:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691521522059520 00010000:00000200:28.0:1616513097.610560:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86b180 x1691521522059520/t0(0) o400->4f3cb6ae-c2c2-f20f-3bda-5772ba37be5c@10.31.129.173@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.610585:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691521522059520, offset 192 00000100:00000200:7.0:1616513097.610591:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929488, portal 4 00000400:00000200:28.0:1616513097.610592:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.173@tcp1 00000100:00000200:7.0:1616513097.610595:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929488, offset 0 00000400:00000200:28.0:1616513097.610599:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.173@tcp1 routed destination 00000400:00000200:7.0:1616513097.610599:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:6.0:1616513097.610600:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929504, portal 4 00000400:00000200:7.0:1616513097.610604:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000100:00000200:6.0:1616513097.610606:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929504, offset 0 00000400:00000200:6.0:1616513097.610613:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:7.0:1616513097.610614:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513097.610616:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.173@tcp1(10.31.129.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513097.610619:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:28.0:1616513097.610624:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:7.0:1616513097.610625:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513097.610626:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:7.0:1616513097.610628:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:6.0:1616513097.610629:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000800:00000200:28.0:1616513097.610630:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.610632:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.0:1616513097.610633:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:28.0:1616513097.610635:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.610640:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:6.0:1616513097.610640:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:27.2:1616513097.610642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:11.2:1616513097.610644:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513097.610645:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:6.0:1616513097.610651:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513097.610653:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:6.0:1616513097.610655:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:27.0:1616513097.610659:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:6.0:1616513097.610659:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:27.0:1616513097.610667:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513097.610670:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513097.610677:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513097.610684:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600d80 x1693067725929488/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513104 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513097.610697:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637e880 00000800:00000200:20.0:1616513097.610698:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:27.0:1616513097.610701:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513097.610704:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:20.0:1616513097.610705:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:1.2:1616513097.610706:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:20.0:1616513097.610708:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.610715:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.610719:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc220 00000800:00000200:20.0:1616513097.610723:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.610726:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:1.0:1616513097.610726:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513097.610733:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513097.610736:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513097.610744:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513097.610752:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bdbf00 x1693067725929504/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513104 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513097.610764:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf018f8 00000800:00000200:1.0:1616513097.610769:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513097.610772:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:27.2:1616513097.610852:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513097.610861:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513097.610868:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513097.610872:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513097.610876:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:1.2:1616513097.610878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:27.0:1616513097.610885:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627c10 00000800:00000200:1.0:1616513097.610888:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:27.0:1616513097.610892:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716735d [1] + 192 00000400:00000200:27.0:1616513097.610897:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:5.0:1616513097.610898:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513097.610898:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000100:00000200:27.0:1616513097.610902:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600d80 x1693067725929488/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513104 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513097.610906:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000800:00000200:27.0:1616513097.610916:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:1.0:1616513097.610916:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627c20 00000100:00000200:7.0:1616513097.610918:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600d80 x1693067725929488/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513104 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513097.610919:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513097.610920:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:1.0:1616513097.610922:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x437002d5 [1] + 192 00000100:00000200:7.0:1616513097.610925:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600d80 x1693067725929488/t0(0) o13->boslfs02-OST001a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513104 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513097.610928:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:7.0:1616513097.610929:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f7e8 00000100:00000200:1.0:1616513097.610932:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bdbf00 x1693067725929504/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513104 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513097.610947:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513097.610950:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513097.610952:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:6.0:1616513097.610954:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bdbf00 x1693067725929504/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513104 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:6.0:1616513097.610967:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bdbf00 x1693067725929504/t0(0) o13->boslfs02-OST0017-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513104 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513097.610987:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:11.2:1616513097.622435:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.622490:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.622500:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.622509:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.100@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.622520:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.100@tcp1 of length 224 into portal 26 MB=0x6001f15185d40 00000400:00000200:16.0:1616513097.622526:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.100@tcp1 of length 224/224 into md 0xf97d00d9 [1] + 0 00000400:00000200:16.0:1616513097.622531:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e550 00000400:00000200:16.0:1616513097.622534:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.622537:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.622542:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e550 00000800:00000200:16.0:1616513097.622545:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.622548:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.622550:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.622552:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.622605:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688983358168384 00010000:00000200:12.0:1616513097.622616:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688983358168384/t0(0) o400->504492ae-6ab5-3e4d-e2b2-05c1d90fae18@10.31.142.100@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.622630:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688983358168384, offset 192 00000400:00000200:12.0:1616513097.622636:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.100@tcp1 00000400:00000200:12.0:1616513097.622644:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.100@tcp1 routed destination 00000400:00000200:12.0:1616513097.622660:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.100@tcp1(10.31.142.100@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.622671:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.622677:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.622680:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.622682:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.622688:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.622688:0:51336:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:12.0:1616513097.622694:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513097.622746:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.622756:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.622765:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.100@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.622774:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.100@tcp1 of length 224 into portal 12 MB=0x6001f15185e40 00000400:00000200:26.0:1616513097.622781:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.100@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 34832 00000400:00000200:26.0:1616513097.622786:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.622789:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513097.622796:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513097.622799:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513097.622800:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:26.0:1616513097.622807:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513097.622808:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513097.622810:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.622811:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513097.622818:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.622823:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be2a8 00000800:00000200:18.0:1616513097.622827:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.622873:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688983358168640 00010000:00000200:22.0:1616513097.622886:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c690900 x1688983358168640/t0(0) o400->b2443e44-8d22-9e45-0e46-ac86670751d1@10.31.142.100@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.622900:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688983358168640, offset 192 00000400:00000200:22.0:1616513097.622907:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.100@tcp1 00000400:00000200:22.0:1616513097.622914:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.100@tcp1 routed destination 00000400:00000200:22.0:1616513097.622931:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.100@tcp1(10.31.142.100@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.622938:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.622943:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.622946:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.622948:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.622953:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.622958:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.623011:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.623018:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.623020:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513097.623027:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.623031:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a880 00000800:00000200:20.0:1616513097.623035:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.623038:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.631088:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.631143:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.631153:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.631162:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.68@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.631171:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.68@tcp1 of length 224 into portal 26 MB=0x5ffd338baf780 00000400:00000200:0.0:1616513097.631177:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.68@tcp1 of length 224/224 into md 0xf97d00f9 [1] + 0 00000400:00000200:0.0:1616513097.631182:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fa08 00000400:00000200:0.0:1616513097.631185:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.631188:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513097.631192:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fa08 00000800:00000200:0.0:1616513097.631195:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.631197:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.631198:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.631203:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.631252:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657538512768 00010000:00000200:12.0:1616513097.631262:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688657538512768/t0(0) o400->5ab31469-0c7c-8ebf-85fe-7a7b27abf695@10.31.133.68@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.631276:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657538512768, offset 192 00000400:00000200:12.0:1616513097.631282:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.68@tcp1 00000400:00000200:12.0:1616513097.631289:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.68@tcp1 routed destination 00000800:00000200:7.2:1616513097.631291:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513097.631303:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.68@tcp1(10.31.133.68@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.631309:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.631314:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.631317:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.631319:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513097.631323:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:12.0:1616513097.631329:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513097.631348:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.631357:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513097.631366:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.53@tcp1 : PUT - for me 00000400:00000200:2.0:1616513097.631376:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.53@tcp1 of length 224 into portal 12 MB=0x6001f2dc11180 00000400:00000200:2.0:1616513097.631383:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.53@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 35056 00000400:00000200:2.0:1616513097.631388:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.631391:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513097.631398:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513097.631401:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513097.631401:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513097.631408:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513097.631409:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513097.631411:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513097.631412:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.631418:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.631424:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be6e8 00000800:00000200:26.0:1616513097.631428:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.631476:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688983771877760 00010000:00000200:22.0:1616513097.631488:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8480 x1688983771877760/t0(0) o400->069c399e-de25-93ac-dec9-69e51d9b967a@10.31.129.53@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513097.631491:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513097.631503:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688983771877760, offset 192 00000400:00000200:22.0:1616513097.631509:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.53@tcp1 00000400:00000200:22.0:1616513097.631517:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.53@tcp1 routed destination 00000400:00000200:22.0:1616513097.631532:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.53@tcp1(10.31.129.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.631538:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.631544:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.631547:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.631549:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.631549:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.631554:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513097.631558:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.631567:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.53@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.631575:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.53@tcp1 of length 224 into portal 26 MB=0x6001f2dc10c80 00000400:00000200:20.0:1616513097.631582:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.53@tcp1 of length 224/224 into md 0xf97d0109 [1] + 0 00000400:00000200:20.0:1616513097.631587:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777eb28 00000400:00000200:20.0:1616513097.631590:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.631593:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513097.631598:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eb28 00000800:00000200:20.0:1616513097.631601:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513097.631603:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:20.0:1616513097.631604:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:12.0:1616513097.631609:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688983771876480 00000800:00000200:20.0:1616513097.631610:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513097.631612:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513097.631613:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.631614:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:12.0:1616513097.631614:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688983771876480/t0(0) o400->a032f1d1-6011-60cb-7b68-98b89b8011c5@10.31.129.53@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:24.0:1616513097.631621:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.631625:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4abb0 00000100:00000200:12.0:1616513097.631625:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688983771876480, offset 192 00000800:00000200:24.0:1616513097.631629:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:12.0:1616513097.631629:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.53@tcp1 00000400:00000200:12.0:1616513097.631635:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.53@tcp1 routed destination 00000400:00000200:12.0:1616513097.631647:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.53@tcp1(10.31.129.53@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.631652:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.631657:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.631659:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.631662:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.631665:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513097.631669:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513097.631670:0:51337:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.631682:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.631690:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.631692:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513097.631694:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513097.631699:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.631704:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bea18 00000800:00000200:14.0:1616513097.631708:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.632017:0:51337:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513097.632027:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.632033:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513097.632036:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513097.632041:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.68@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.632052:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.68@tcp1 of length 224 into portal 12 MB=0x5ffd338baf980 00000400:00000200:26.0:1616513097.632058:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.68@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 35280 00000400:00000200:26.0:1616513097.632063:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.632066:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513097.632072:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.632074:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.632076:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.632083:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657538513280 00010000:00000200:22.0:1616513097.632090:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c691680 x1688657538513280/t0(0) o400->c332bf28-5601-e839-99de-3610629e8fb3@10.31.133.68@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.632101:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657538513280, offset 192 00000400:00000200:22.0:1616513097.632106:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.68@tcp1 00000400:00000200:22.0:1616513097.632112:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.68@tcp1 routed destination 00000400:00000200:22.0:1616513097.632125:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.68@tcp1(10.31.133.68@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.632131:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.632135:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.632138:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.632142:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.632145:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.632149:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.632158:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.632162:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.632164:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513097.632166:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.632170:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.632173:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a2a8 00000800:00000200:0.0:1616513097.632176:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.646283:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.646338:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.646347:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.646356:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.197@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.646366:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.197@tcp1 of length 224 into portal 26 MB=0x6018d4cd5da80 00000400:00000200:16.0:1616513097.646373:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.197@tcp1 of length 224/224 into md 0xf97d0119 [1] + 0 00000400:00000200:16.0:1616513097.646377:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e880 00000400:00000200:16.0:1616513097.646380:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.646383:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.646387:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e880 00000800:00000200:16.0:1616513097.646390:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.646391:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.646393:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.646395:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.646464:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690556251363968 00010000:00000200:12.0:1616513097.646475:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1690556251363968/t0(0) o400->2b40407e-33f4-837d-0440-06d806aad6d1@10.31.128.197@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513097.646483:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:12.0:1616513097.646489:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690556251363968, offset 192 00000400:00000200:12.0:1616513097.646495:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.197@tcp1 00000400:00000200:12.0:1616513097.646503:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.197@tcp1 routed destination 00000400:00000200:12.0:1616513097.646518:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.197@tcp1(10.31.128.197@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.646525:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.646530:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.646533:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.646535:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513097.646540:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513097.646542:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000100:00000200:12.0:1616513097.646546:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513097.646551:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513097.646560:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.197@tcp1 : PUT - for me 00000400:00000200:2.0:1616513097.646568:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.197@tcp1 of length 224 into portal 12 MB=0x6018d4cd5df80 00000400:00000200:2.0:1616513097.646575:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.197@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 35504 00000400:00000200:2.0:1616513097.646580:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.646582:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513097.646589:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513097.646592:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513097.646594:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.646599:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.646607:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.646610:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.646617:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.646622:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be2a8 00000800:00000200:14.0:1616513097.646627:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.646630:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.646654:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690556251365248 00010000:00000200:22.0:1616513097.646666:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dd580 x1690556251365248/t0(0) o400->0b4b6a76-c4f2-9607-e3b4-a1fcbbd3224a@10.31.128.197@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.646680:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690556251365248, offset 192 00000400:00000200:22.0:1616513097.646687:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.197@tcp1 00000400:00000200:22.0:1616513097.646694:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.197@tcp1 routed destination 00000400:00000200:22.0:1616513097.646709:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.197@tcp1(10.31.128.197@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.646716:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.646721:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.646724:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.646726:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.646731:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.646735:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.646790:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.646798:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.646800:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513097.646803:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.646807:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.646811:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a5d8 00000800:00000200:20.0:1616513097.646815:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:27.2:1616513097.689338:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513097.689357:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513097.689364:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513097.689372:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513097.689380:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 26 MB=0x5f9313e25e9d0 00000400:00000200:27.0:1616513097.689387:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0xf97d0129 [1] + 0 00000400:00000200:27.0:1616513097.689392:0:6473:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f298 00000400:00000200:27.0:1616513097.689395:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513097.689398:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:27.0:1616513097.689403:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f298 00000800:00000200:27.0:1616513097.689406:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513097.689408:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513097.689409:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513097.689416:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513097.689419:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:31.0:1616513097.689420:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:27.0:1616513097.689428:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:3.0:1616513097.689428:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:31.0:1616513097.689429:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:7.0:1616513097.689435:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000800:00000200:3.0:1616513097.689448:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:7.0:1616513097.689449:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513097.689456:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:31.0:1616513097.689458:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25ea50 00000400:00000200:7.0:1616513097.689460:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:3.0:1616513097.689460:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513097.689462:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25ead0 00000400:00000200:31.0:1616513097.689466:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 92288 00000400:00000200:7.0:1616513097.689469:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25ea10 00000400:00000200:3.0:1616513097.689469:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25eb50 00000100:00000200:12.0:1616513097.689470:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939088 00000400:00000200:31.0:1616513097.689471:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:27.0:1616513097.689473:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 92512 00000400:00000200:27.0:1616513097.689477:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:3.0:1616513097.689479:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 92736 00010000:00000200:12.0:1616513097.689483:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1681364774939088/t0(0) o400->2a9c6460-7cbc-99eb-d035-661c67cc9089@10.242.46.231@o2ib1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:3.0:1616513097.689485:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:7.0:1616513097.689487:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 92960 00000100:00000200:31.0:1616513097.689492:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:7.0:1616513097.689492:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:12.0:1616513097.689498:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681364774939088, offset 192 00000100:00000200:27.0:1616513097.689502:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:31.0:1616513097.689503:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:12.0:1616513097.689505:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:31.0:1616513097.689507:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:31.0:1616513097.689509:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000400:00000200:12.0:1616513097.689514:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000100:00000200:3.0:1616513097.689519:0:6476:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:31.0:1616513097.689520:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000800:00000200:27.0:1616513097.689520:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (24)++ 00000800:00000200:31.0:1616513097.689524:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000100:00000200:7.0:1616513097.689529:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513097.689531:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513097.689536:0:6473:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:3.0:1616513097.689537:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:7.0:1616513097.689538:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:12.0:1616513097.689539:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513097.689540:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)-- 00000400:00000200:31.0:1616513097.689541:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:7.0:1616513097.689541:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:3.0:1616513097.689541:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:27.0:1616513097.689542:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:7.0:1616513097.689543:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:3.0:1616513097.689545:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)-- 00000800:00000200:12.0:1616513097.689546:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:31.0:1616513097.689547:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25ea30 00000800:00000200:27.0:1616513097.689547:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:12.0:1616513097.689549:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (24)++ 00000400:00000200:31.0:1616513097.689552:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 93184 00000800:00000200:12.0:1616513097.689552:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:27.0:1616513097.689553:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:7.0:1616513097.689553:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:31.0:1616513097.689556:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:31.0:1616513097.689558:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:12.0:1616513097.689558:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (27)-- 00000400:00000200:27.0:1616513097.689559:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:7.0:1616513097.689560:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:7.0:1616513097.689562:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:7.0:1616513097.689563:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000400:00000200:27.0:1616513097.689564:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25e9f0 00000800:00000200:7.0:1616513097.689565:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000100:00000200:12.0:1616513097.689566:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:31.0:1616513097.689567:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:27.0:1616513097.689569:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 93408 00000800:00000200:31.0:1616513097.689570:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (26)-- 00000400:00000200:7.0:1616513097.689570:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:31.0:1616513097.689571:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000100:00000200:11.0:1616513097.689571:0:170091:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939216 00000400:00000200:27.0:1616513097.689572:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:31.0:1616513097.689573:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000100:00000200:27.0:1616513097.689574:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:7.0:1616513097.689574:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25ebd0 00000800:00000200:31.0:1616513097.689576:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (25)-- 00000400:00000200:31.0:1616513097.689578:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:27.0:1616513097.689582:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (24)++ 00010000:00000200:11.0:1616513097.689583:0:170091:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cdc566780 x1681364774939216/t0(0) o400->boslfs02-MDT0001-lwp-OST0002_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:7.0:1616513097.689583:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 93632 00000400:00000200:31.0:1616513097.689584:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:27.0:1616513097.689584:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (25)-- 00000400:00000200:7.0:1616513097.689585:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:27.0:1616513097.689586:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)-- 00000100:00000200:7.0:1616513097.689587:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:27.0:1616513097.689588:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:31.0:1616513097.689589:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf7e8 00000800:00000200:31.0:1616513097.689592:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)-- 00000800:00000200:27.0:1616513097.689592:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513097.689594:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:31.0:1616513097.689598:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513097.689598:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000100:00000200:11.0:1616513097.689598:0:170091:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939216, offset 192 00000800:00000200:7.0:1616513097.689598:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (24)++ 00000800:00000200:7.0:1616513097.689599:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:7.0:1616513097.689600:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)-- 00000800:00000200:7.0:1616513097.689602:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:27.0:1616513097.689603:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25ea90 00000400:00000200:11.0:1616513097.689603:0:170091:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:15.0:1616513097.689604:0:145749:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939344 00000800:00000200:7.0:1616513097.689604:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:31.0:1616513097.689605:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000100:00000200:19.0:1616513097.689606:0:237846:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939472 00000400:00000200:27.0:1616513097.689608:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 93856 00000400:00000200:7.0:1616513097.689608:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000100:00000200:3.0:1616513097.689608:0:252828:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939152 00000400:00000200:11.0:1616513097.689609:0:170091:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000100:00000200:23.0:1616513097.689610:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939184 00000400:00000200:27.0:1616513097.689612:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513097.689613:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:3.0:1616513097.689618:0:252828:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8f30a8ad00 x1681364774939152/t0(0) o400->boslfs02-MDT0001-lwp-OST0006_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:19.0:1616513097.689619:0:237846:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8da4dd600d80 x1681364774939472/t0(0) o400->boslfs02-MDT0001-lwp-OST0012_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:15.0:1616513097.689619:0:145749:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d841ded1b00 x1681364774939344/t0(0) o400->boslfs02-MDT0001-lwp-OST0022_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:11.0:1616513097.689621:0:170091:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00010000:00000200:23.0:1616513097.689622:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cdc567080 x1681364774939184/t0(0) o400->boslfs02-MDT0001-lwp-OST002a_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.0:1616513097.689626:0:170091:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513097.689627:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (24)++ 00000400:00000200:7.0:1616513097.689628:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25eb30 00000400:00000200:31.0:1616513097.689629:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25eab0 00000800:00000200:27.0:1616513097.689630:0:6473:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:7.0:1616513097.689631:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 94080 00000800:00000200:27.0:1616513097.689632:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:11.0:1616513097.689632:0:170091:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:7.0:1616513097.689633:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:27.0:1616513097.689634:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:11.0:1616513097.689634:0:170091:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (25)++ 00000100:00000200:3.0:1616513097.689634:0:252828:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939152, offset 192 00000800:00000200:11.0:1616513097.689635:0:170091:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (26)++ 00000400:00000200:31.0:1616513097.689636:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 94304 00000800:00000200:27.0:1616513097.689637:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)++ 00000400:00000200:31.0:1616513097.689640:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:11.0:1616513097.689640:0:170091:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:27.0:1616513097.689641:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000100:00000200:19.0:1616513097.689641:0:237846:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939472, offset 192 00000100:00000200:15.0:1616513097.689641:0:145749:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939344, offset 192 00000100:00000200:7.0:1616513097.689641:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:23.0:1616513097.689646:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939184, offset 192 00000800:00000200:7.0:1616513097.689646:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (26)++ 00000400:00000200:27.0:1616513097.689647:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:7.0:1616513097.689647:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:7.0:1616513097.689648:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:7.0:1616513097.689649:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:3.0:1616513097.689649:0:252828:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:31.0:1616513097.689651:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.0:1616513097.689651:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513097.689652:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25ea70 00000400:00000200:3.0:1616513097.689655:0:252828:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:15.0:1616513097.689658:0:145749:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:11.0:1616513097.689662:0:170091:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939120 00000400:00000200:19.0:1616513097.689664:0:237846:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00010000:00000200:11.0:1616513097.689667:0:170091:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d841ded1f80 x1681364774939120/t0(0) o400->boslfs02-MDT0001-lwp-OST0032_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:23.0:1616513097.689669:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:3.0:1616513097.689669:0:252828:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:27.0:1616513097.689674:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 94528 00000800:00000200:31.0:1616513097.689675:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:31.0:1616513097.689677:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (27)-- 00000400:00000200:27.0:1616513097.689677:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:11.0:1616513097.689677:0:170091:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939120, offset 192 00000400:00000200:7.0:1616513097.689677:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:3.0:1616513097.689677:0:252828:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:31.0:1616513097.689678:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)-- 00000100:00000200:27.0:1616513097.689678:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:15.0:1616513097.689679:0:145749:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:31.0:1616513097.689680:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:31.0:1616513097.689683:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:31.0:1616513097.689684:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000400:00000200:11.0:1616513097.689684:0:170091:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:31.0:1616513097.689686:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000800:00000200:31.0:1616513097.689689:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (25)-- 00000400:00000200:31.0:1616513097.689690:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:15.0:1616513097.689692:0:145749:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:31.0:1616513097.689695:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:31.0:1616513097.689698:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3bc28 00000800:00000200:15.0:1616513097.689698:0:145749:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:19.0:1616513097.689700:0:237846:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:19.0:1616513097.689713:0:237846:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513097.689720:0:237846:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:23.0:1616513097.689722:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:23.0:1616513097.689735:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513097.689740:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (24)++ 00000800:00000200:23.0:1616513097.689741:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513097.689742:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:3.0:1616513097.689743:0:252828:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513097.689744:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:11.0:1616513097.689744:0:170091:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:27.0:1616513097.689746:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:3.0:1616513097.689746:0:252828:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:3.0:1616513097.689747:0:252828:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:27.0:1616513097.689749:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:3.0:1616513097.689751:0:252828:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (26)-- 00000400:00000200:11.0:1616513097.689754:0:170091:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513097.689757:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:11.0:1616513097.689758:0:170091:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:31.0:1616513097.689759:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000400:00000200:7.0:1616513097.689759:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25eb10 00000800:00000200:15.0:1616513097.689761:0:145749:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513097.689762:0:237846:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513097.689762:0:170091:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000400:00000200:7.0:1616513097.689762:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 94752 00000800:00000200:31.0:1616513097.689763:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:23.0:1616513097.689763:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513097.689763:0:145749:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:27.0:1616513097.689764:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:11.0:1616513097.689764:0:170091:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:7.0:1616513097.689764:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:19.0:1616513097.689765:0:237846:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:15.0:1616513097.689765:0:145749:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (27)++ 00000100:00000200:7.0:1616513097.689765:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:3.0:1616513097.689766:0:252828:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939600 00000800:00000200:23.0:1616513097.689768:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:7.0:1616513097.689768:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)++ 00000400:00000200:27.0:1616513097.689769:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25ebb0 00000400:00000200:31.0:1616513097.689770:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:15.0:1616513097.689770:0:145749:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:11.0:1616513097.689770:0:170091:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (31)++ 00010000:00000200:3.0:1616513097.689771:0:252828:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8f30a8b180 x1681364774939600/t0(0) o400->boslfs02-MDT0001-lwp-OST000a_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:27.0:1616513097.689773:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 94976 00000800:00000200:19.0:1616513097.689773:0:237846:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (31)++ 00000800:00000200:11.0:1616513097.689773:0:170091:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (31)-- 00000400:00000200:31.0:1616513097.689775:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25eb90 00000400:00000200:27.0:1616513097.689776:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:23.0:1616513097.689778:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (31)++ 00000800:00000200:19.0:1616513097.689778:0:237846:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:7.0:1616513097.689778:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:7.0:1616513097.689779:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (30)-- 00000400:00000200:31.0:1616513097.689780:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 95200 00000100:00000200:15.0:1616513097.689780:0:145749:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939280 00000800:00000200:7.0:1616513097.689780:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:7.0:1616513097.689782:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000400:00000200:31.0:1616513097.689783:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513097.689783:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:23.0:1616513097.689783:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (30)-- 00000100:00000200:11.0:1616513097.689784:0:170091:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939440 00000400:00000200:7.0:1616513097.689786:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00010000:00000200:15.0:1616513097.689787:0:145749:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d841ded5580 x1681364774939280/t0(0) o400->boslfs02-MDT0001-lwp-OST001e_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:3.0:1616513097.689787:0:252828:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939600, offset 192 00000100:00000200:31.0:1616513097.689788:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:27.0:1616513097.689788:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)++ 00010000:00000200:11.0:1616513097.689788:0:170091:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cc96a1f80 x1681364774939440/t0(0) o400->boslfs02-MDT0001-lwp-OST0016_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:27.0:1616513097.689790:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (30)-- 00000400:00000200:7.0:1616513097.689790:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25eaf0 00000800:00000200:27.0:1616513097.689791:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000100:00000200:23.0:1616513097.689791:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939312 00000800:00000200:31.0:1616513097.689793:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:27.0:1616513097.689793:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:31.0:1616513097.689795:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (30)-- 00000400:00000200:3.0:1616513097.689795:0:252828:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:31.0:1616513097.689797:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:27.0:1616513097.689797:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00010000:00000200:23.0:1616513097.689797:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cdc566300 x1681364774939312/t0(0) o400->boslfs02-MDT0001-lwp-OST0026_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:7.0:1616513097.689798:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 95424 00000800:00000200:31.0:1616513097.689799:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000100:00000200:11.0:1616513097.689800:0:170091:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939440, offset 192 00000400:00000200:7.0:1616513097.689800:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:3.0:1616513097.689800:0:252828:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:31.0:1616513097.689801:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (29)-- 00000100:00000200:15.0:1616513097.689801:0:145749:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939280, offset 192 00000100:00000200:7.0:1616513097.689801:0:6474:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:31.0:1616513097.689802:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:11.0:1616513097.689806:0:170091:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:31.0:1616513097.689807:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:15.0:1616513097.689810:0:145749:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000100:00000200:19.0:1616513097.689811:0:237846:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939248 00000400:00000200:31.0:1616513097.689812:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd26e8 00000100:00000200:23.0:1616513097.689812:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939312, offset 192 00000400:00000200:3.0:1616513097.689813:0:252828:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:23.0:1616513097.689817:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00010000:00000200:19.0:1616513097.689817:0:237846:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d841ded4c80 x1681364774939248/t0(0) o400->boslfs02-MDT0001-lwp-OST002e_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.0:1616513097.689817:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:7.0:1616513097.689818:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:7.0:1616513097.689819:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:3.0:1616513097.689819:0:252828:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:11.0:1616513097.689820:0:170091:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:7.0:1616513097.689820:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)++ 00000400:00000200:27.0:1616513097.689821:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:7.0:1616513097.689821:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (28)-- 00000400:00000200:7.0:1616513097.689822:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:7.0:1616513097.689825:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:7.0:1616513097.689827:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0b980 00000100:00000200:19.0:1616513097.689829:0:237846:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939248, offset 192 00000400:00000200:11.0:1616513097.689830:0:170091:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513097.689834:0:170091:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:19.0:1616513097.689835:0:237846:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:15.0:1616513097.689836:0:145749:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:15.0:1616513097.689848:0:145749:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:31.0:1616513097.689852:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:15.0:1616513097.689853:0:145749:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:31.0:1616513097.689854:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:31.0:1616513097.689856:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (27)-- 00000400:00000200:23.0:1616513097.689856:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:31.0:1616513097.689857:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:31.0:1616513097.689862:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:31.0:1616513097.689864:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3b870 00000400:00000200:23.0:1616513097.689869:0:170661:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513097.689874:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:23.0:1616513097.689875:0:170661:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:7.0:1616513097.689875:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:3.0:1616513097.689875:0:252828:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513097.689877:0:170091:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513097.689877:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:3.0:1616513097.689877:0:252828:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (26)++ 00000400:00000200:19.0:1616513097.689878:0:237846:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:7.0:1616513097.689878:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.0:1616513097.689879:0:170091:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (26)++ 00000800:00000200:3.0:1616513097.689879:0:252828:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (26)++ 00000400:00000200:7.0:1616513097.689881:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:3.0:1616513097.689881:0:252828:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:11.0:1616513097.689882:0:170091:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (28)++ 00000400:00000200:7.0:1616513097.689882:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000800:00000200:11.0:1616513097.689884:0:170091:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (28)-- 00000100:00000200:11.0:1616513097.689889:0:170091:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939568 00000100:00000200:3.0:1616513097.689889:0:252828:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939408 00000400:00000200:19.0:1616513097.689891:0:237846:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00010000:00000200:11.0:1616513097.689894:0:170091:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d841ded6780 x1681364774939568/t0(0) o400->boslfs02-MDT0001-lwp-OST003e_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:3.0:1616513097.689894:0:252828:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cc96a4800 x1681364774939408/t0(0) o400->boslfs02-MDT0001-lwp-OST001a_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:31.0:1616513097.689898:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:19.0:1616513097.689899:0:237846:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:15.0:1616513097.689899:0:145749:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513097.689899:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)-- 00000400:00000200:27.0:1616513097.689900:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 224 into portal 12 MB=0x5f9313e25eb70 00000800:00000200:31.0:1616513097.689901:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000800:00000200:23.0:1616513097.689901:0:170661:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513097.689901:0:145749:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (25)++ 00000100:00000200:11.0:1616513097.689901:0:170091:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939568, offset 192 00000800:00000200:23.0:1616513097.689903:0:170661:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:15.0:1616513097.689903:0:145749:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (27)++ 00000400:00000200:27.0:1616513097.689905:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.231@o2ib1 of length 224/224 into md 0x3657608d [1] + 95648 00000800:00000200:19.0:1616513097.689905:0:237846:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:15.0:1616513097.689906:0:145749:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.231@o2ib1: no credits 00000800:00000200:19.0:1616513097.689907:0:237846:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:15.0:1616513097.689907:0:145749:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)-- 00000100:00000200:7.0:1616513097.689907:0:188035:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939536 00000400:00000200:27.0:1616513097.689908:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:23.0:1616513097.689908:0:170661:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (29)++ 00000400:00000200:11.0:1616513097.689909:0:170091:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:23.0:1616513097.689911:0:170661:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.231@o2ib1: no credits 00000100:00000200:3.0:1616513097.689911:0:252828:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939408, offset 192 00000100:00000200:27.0:1616513097.689912:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:23.0:1616513097.689913:0:170661:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (30)-- 00000100:00000200:15.0:1616513097.689913:0:145749:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939376 00010000:00000200:7.0:1616513097.689913:0:188035:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cca3eb180 x1681364774939536/t0(0) o400->boslfs02-MDT0001-lwp-OST000e_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:11.0:1616513097.689914:0:170091:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:31.0:1616513097.689915:0:6475:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.231@o2ib1: no credits 00000800:00000200:31.0:1616513097.689917:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:19.0:1616513097.689917:0:237846:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (29)++ 00000400:00000200:3.0:1616513097.689917:0:252828:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:31.0:1616513097.689918:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:15.0:1616513097.689918:0:145749:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d8cc96a7080 x1681364774939376/t0(0) o400->boslfs02-MDT0001-lwp-OST0036_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:19.0:1616513097.689920:0:237846:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.231@o2ib1: no credits 00000100:00000200:7.0:1616513097.689921:0:188035:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939536, offset 192 00000800:00000200:19.0:1616513097.689922:0:237846:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (29)-- 00000400:00000200:31.0:1616513097.689923:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:11.0:1616513097.689924:0:170091:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:7.0:1616513097.689924:0:188035:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:31.0:1616513097.689926:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3fb18 00000100:00000200:23.0:1616513097.689926:0:170661:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681364774939504 00000800:00000200:27.0:1616513097.689927:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)++ 00000100:00000200:15.0:1616513097.689928:0:145749:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939376, offset 192 00000800:00000200:11.0:1616513097.689928:0:170091:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513097.689930:0:6473:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.231@o2ib1: no credits 00010000:00000200:23.0:1616513097.689931:0:170661:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d841ded2d00 x1681364774939504/t0(0) o400->boslfs02-MDT0001-lwp-OST003a_UUID@10.242.46.231@o2ib1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:3.0:1616513097.689931:0:252828:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:27.0:1616513097.689932:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:27.0:1616513097.689933:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)-- 00000400:00000200:15.0:1616513097.689933:0:145749:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513097.689936:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:27.0:1616513097.689940:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.231@o2ib1 00000100:00000200:23.0:1616513097.689942:0:170661:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681364774939504, offset 192 00000400:00000200:3.0:1616513097.689943:0:252828:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513097.689944:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:27.0:1616513097.689945:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:27.0:1616513097.689947:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)-- 00000400:00000200:23.0:1616513097.689947:0:170661:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:7.0:1616513097.689948:0:188035:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:3.0:1616513097.689948:0:252828:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:27.0:1616513097.689949:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:27.0:1616513097.689951:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (28)-- 00000400:00000200:27.0:1616513097.689953:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.0:1616513097.689954:0:170091:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513097.689955:0:6475:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000400:00000200:15.0:1616513097.689956:0:145749:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:11.0:1616513097.689956:0:170091:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (27)++ 00000400:00000200:27.0:1616513097.689958:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:11.0:1616513097.689958:0:170091:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:7.0:1616513097.689958:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:11.0:1616513097.689960:0:170091:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:7.0:1616513097.689960:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (30)-- 00000400:00000200:27.0:1616513097.689961:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2110 00000400:00000200:7.0:1616513097.689961:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:23.0:1616513097.689963:0:170661:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:7.0:1616513097.689964:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:7.0:1616513097.689965:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a110 00000800:00000200:3.0:1616513097.689968:0:252828:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513097.689969:0:6475:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513097.689970:0:6473:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:7.0:1616513097.689970:0:6474:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:3.0:1616513097.689971:0:252828:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:31.0:1616513097.689972:0:6475:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (28)++ 00000800:00000200:3.0:1616513097.689972:0:252828:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (29)++ 00000800:00000200:7.0:1616513097.689973:0:6474:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513097.689975:0:6474:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (31)++ 00000800:00000200:27.0:1616513097.689976:0:6473:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513097.689976:0:252828:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.231@o2ib1: no credits 00000800:00000200:31.0:1616513097.689978:0:6475:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (32)++ 00000800:00000200:27.0:1616513097.689978:0:6473:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (32)++ 00000800:00000200:3.0:1616513097.689978:0:252828:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (32)-- 00000800:00000200:31.0:1616513097.689982:0:6475:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.231@o2ib1: no credits 00000800:00000200:31.0:1616513097.689983:0:6475:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (33)-- 00000800:00000200:7.0:1616513097.689983:0:6474:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (33)++ 00000800:00000200:31.0:1616513097.689985:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (33)-- 00000800:00000200:7.0:1616513097.689985:0:6474:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.231@o2ib1: no credits 00000800:00000200:7.0:1616513097.689986:0:6474:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (33)-- 00000800:00000200:31.0:1616513097.689987:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (31)++ 00000800:00000200:27.0:1616513097.689987:0:6473:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (32)++ 00000800:00000200:27.0:1616513097.689990:0:6473:0:(o2iblnd_cb.c:900:kiblnd_post_tx_locked()) 10.242.46.231@o2ib1: no credits 00000800:00000200:27.0:1616513097.689991:0:6473:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (33)-- 00000800:00000200:7.0:1616513097.690000:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (32)-- 00000800:00000200:27.0:1616513097.690001:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:31.0:1616513097.690002:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.231@o2ib1 00000800:00000200:7.0:1616513097.690003:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (30)++ 00000800:00000200:31.0:1616513097.690008:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (31)++ 00000800:00000200:7.0:1616513097.690008:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (31)-- 00000400:00000200:7.0:1616513097.690009:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513097.690009:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (31)++ 00000800:00000200:31.0:1616513097.690010:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:31.0:1616513097.690011:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (31)-- 00000800:00000200:3.0:1616513097.690011:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (31)-- 00000400:00000200:7.0:1616513097.690012:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:7.0:1616513097.690013:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0aee0 00000400:00000200:3.0:1616513097.690013:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.0:1616513097.690014:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)++ 00000400:00000200:3.0:1616513097.690019:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:3.0:1616513097.690022:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e7f8 00000800:00000200:11.0:1616513097.690028:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:7.0:1616513097.690028:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (30)-- 00000800:00000200:3.0:1616513097.690029:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000400:00000200:11.0:1616513097.690030:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.0:1616513097.690031:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:7.0:1616513097.690034:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:31.0:1616513097.690035:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000400:00000200:11.0:1616513097.690035:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:7.0:1616513097.690035:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:11.0:1616513097.690037:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707a08 00000800:00000200:3.0:1616513097.690037:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (28)++ 00000400:00000200:7.0:1616513097.690038:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:31.0:1616513097.690039:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.231@o2ib1 00000400:00000200:7.0:1616513097.690039:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a330 00000800:00000200:3.0:1616513097.690039:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:11.0:1616513097.690040:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (29)-- 00000800:00000200:31.0:1616513097.690041:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (29)++ 00000400:00000200:3.0:1616513097.690041:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513097.690042:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (28)-- 00000800:00000200:7.0:1616513097.690042:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:11.0:1616513097.690043:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)++ 00000800:00000200:31.0:1616513097.690046:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (27)-- 00000800:00000200:11.0:1616513097.690046:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (27)-- 00000400:00000200:3.0:1616513097.690046:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:31.0:1616513097.690048:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)++ 00000400:00000200:11.0:1616513097.690048:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1616513097.690048:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2a18 00000800:00000200:31.0:1616513097.690051:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (26)-- 00000800:00000200:3.0:1616513097.690051:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (26)-- 00000400:00000200:11.0:1616513097.690052:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000400:00000200:31.0:1616513097.690053:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513097.690053:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)++ 00000400:00000200:11.0:1616513097.690054:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f7e8 00000800:00000200:11.0:1616513097.690056:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (25)-- 00000800:00000200:3.0:1616513097.690057:0:6476:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (25)-- 00000400:00000200:31.0:1616513097.690058:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:11.0:1616513097.690058:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:3.0:1616513097.690058:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:31.0:1616513097.690060:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841de3e990 00000800:00000200:31.0:1616513097.690062:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (24)-- 00000800:00000200:11.0:1616513097.690062:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.231@o2ib1 00000400:00000200:3.0:1616513097.690063:0:6476:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:11.0:1616513097.690064:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:7.0:1616513097.690064:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:11.0:1616513097.690066:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:3.0:1616513097.690066:0:6476:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c4a0ab28 00000800:00000200:11.0:1616513097.690067:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:3.0:1616513097.690068:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:11.2:1616513097.695932:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.695988:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.695998:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513097.696007:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.124@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.696017:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.124@tcp1 of length 224 into portal 26 MB=0x603fc1384fb00 00000400:00000200:24.0:1616513097.696024:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.124@tcp1 of length 224/224 into md 0xf97d0149 [1] + 0 00000400:00000200:24.0:1616513097.696030:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f078 00000400:00000200:24.0:1616513097.696035:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.696038:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:16.0:1616513097.696042:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513097.696043:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f078 00000800:00000200:24.0:1616513097.696047:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513097.696050:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513097.696052:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.696054:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513097.696055:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513097.696063:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.124@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.696072:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.124@tcp1 of length 224 into portal 12 MB=0x603fc1384fb80 00000400:00000200:16.0:1616513097.696079:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.124@tcp1 of length 224/224 into md 0x865be291 [1] + 67952 00000400:00000200:16.0:1616513097.696084:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.696087:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.696095:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.696098:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.696100:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.696119:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693231054387968 00010000:00000200:12.0:1616513097.696131:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1693231054387968/t0(0) o400->d0398c04-0eaf-ce40-eb00-2d74570c34fe@10.31.138.124@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.696148:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693231054387968, offset 192 00000400:00000200:12.0:1616513097.696154:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.124@tcp1 00000100:00000200:28.0:1616513097.696160:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693231054388096 00000400:00000200:12.0:1616513097.696161:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.124@tcp1 routed destination 00010000:00000200:28.0:1616513097.696174:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d6780 x1693231054388096/t0(0) o400->404925fa-5010-abe4-2fd7-8e8baf319488@10.31.138.124@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513097.696177:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.124@tcp1(10.31.138.124@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.696184:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:28.0:1616513097.696189:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693231054388096, offset 192 00000800:00000200:12.0:1616513097.696190:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.696193:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513097.696196:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.124@tcp1 00000800:00000200:12.0:1616513097.696196:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.696201:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513097.696203:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.124@tcp1 routed destination 00000800:00000200:7.2:1616513097.696208:0:51344:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.696209:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513097.696219:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.124@tcp1(10.31.138.124@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.696225:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513097.696231:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.696234:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513097.696236:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.696241:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.696246:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513097.696263:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.696270:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.696273:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513097.696280:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.696286:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be5d8 00000800:00000200:26.0:1616513097.696289:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.696292:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513097.696299:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.696307:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.696309:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.696310:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.696316:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.696320:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc330 00000800:00000200:20.0:1616513097.696324:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.724008:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.724027:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.724036:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513097.724040:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513097.724046:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.203@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.724056:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.203@tcp1 of length 224 into portal 26 MB=0x5ffd4de19a8c0 00000400:00000200:16.0:1616513097.724063:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.203@tcp1 of length 224/224 into md 0xf97d0169 [1] + 0 00000400:00000200:16.0:1616513097.724068:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f430 00000400:00000200:16.0:1616513097.724071:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.724074:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.724079:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f430 00000800:00000200:16.0:1616513097.724083:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.724088:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.724090:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.724103:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664607926464 00010000:00000200:12.0:1616513097.724115:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688664607926464/t0(0) o400->5e757498-95d5-79d9-2963-1af97b49fc26@10.31.132.203@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.724130:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664607926464, offset 192 00000400:00000200:12.0:1616513097.724137:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.203@tcp1 00000400:00000200:12.0:1616513097.724145:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.203@tcp1 routed destination 00000400:00000200:12.0:1616513097.724160:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.203@tcp1(10.31.132.203@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.724167:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.724174:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.724177:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.724179:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.724185:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.724191:0:51347:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.724192:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513097.724204:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.724210:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.724213:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.724220:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.724226:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be330 00000800:00000200:14.0:1616513097.724230:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.724233:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.724657:0:51347:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513097.724713:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.724723:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.724732:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.203@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.724741:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.203@tcp1 of length 224 into portal 12 MB=0x5ffd4de19aac0 00000400:00000200:26.0:1616513097.724748:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.203@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 35728 00000400:00000200:26.0:1616513097.724753:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.724756:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513097.724763:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.724766:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513097.724767:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.724772:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.724830:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664607926976 00010000:00000200:22.0:1616513097.724843:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695e80 x1688664607926976/t0(0) o400->b530049f-55d1-68f0-2889-178cc51fceb6@10.31.132.203@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.724858:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664607926976, offset 192 00000400:00000200:22.0:1616513097.724865:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.203@tcp1 00000400:00000200:22.0:1616513097.724873:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.203@tcp1 routed destination 00000400:00000200:22.0:1616513097.724888:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.203@tcp1(10.31.132.203@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.724895:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.724900:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.724903:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.724905:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.724910:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.724916:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.724971:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.724979:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.724981:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513097.724988:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.724992:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b078 00000800:00000200:24.0:1616513097.724996:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.724999:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.739406:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.739462:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.739471:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.739480:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.107@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.739491:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.107@tcp1 of length 224 into portal 26 MB=0x601477499e940 00000400:00000200:14.0:1616513097.739498:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.107@tcp1 of length 224/224 into md 0xf97d0179 [1] + 0 00000400:00000200:14.0:1616513097.739503:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ee58 00000400:00000200:14.0:1616513097.739505:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.739508:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513097.739513:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ee58 00000800:00000200:18.0:1616513097.739516:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.739516:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.739519:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.739521:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.739579:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690256270813504 00010000:00000200:12.0:1616513097.739590:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1690256270813504/t0(0) o400->3c079d98-db48-38c6-8231-419ca271066d@10.31.142.107@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.739605:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690256270813504, offset 192 00000400:00000200:12.0:1616513097.739611:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.107@tcp1 00000400:00000200:12.0:1616513097.739620:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.107@tcp1 routed destination 00000400:00000200:12.0:1616513097.739637:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.107@tcp1(10.31.142.107@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.739644:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:7.2:1616513097.739649:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.739650:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.739653:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.739655:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513097.739661:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:12.0:1616513097.739668:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513097.739708:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.739717:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513097.739726:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.107@tcp1 : PUT - for me 00000400:00000200:2.0:1616513097.739735:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.107@tcp1 of length 224 into portal 12 MB=0x601477499ea40 00000400:00000200:2.0:1616513097.739742:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.107@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 35952 00000400:00000200:2.0:1616513097.739747:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.739750:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513097.739756:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513097.739760:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513097.739762:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.739764:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.739772:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513097.739775:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513097.739776:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513097.739782:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.739788:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bef68 00000800:00000200:26.0:1616513097.739791:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.739822:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690256270813760 00010000:00000200:22.0:1616513097.739835:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647de300 x1690256270813760/t0(0) o400->6692726b-56b9-4310-002b-283c691b3ff8@10.31.142.107@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.739849:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690256270813760, offset 192 00000400:00000200:22.0:1616513097.739857:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.107@tcp1 00000400:00000200:22.0:1616513097.739865:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.107@tcp1 routed destination 00000400:00000200:22.0:1616513097.739880:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.107@tcp1(10.31.142.107@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.739887:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.739893:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.739895:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.739897:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.739902:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.739910:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.739925:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.739932:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.739935:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513097.739942:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.739946:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bc28 00000800:00000200:0.0:1616513097.739950:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.739953:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.749125:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.749179:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.749188:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.749196:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.163@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.749206:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.163@tcp1 of length 224 into portal 26 MB=0x604191e6fc640 00000400:00000200:14.0:1616513097.749212:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.163@tcp1 of length 224/224 into md 0xf97d0199 [1] + 0 00000400:00000200:14.0:1616513097.749216:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f3a8 00000400:00000200:14.0:1616513097.749219:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.749221:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513097.749225:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f3a8 00000800:00000200:14.0:1616513097.749228:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.749231:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513097.749232:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.749236:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513097.749287:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693355791599168 00010000:00000200:12.0:1616513097.749298:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1693355791599168/t0(0) o400->f4a88395-facf-06ba-8b4f-bf40254015cc@10.31.129.163@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.749311:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693355791599168, offset 192 00000400:00000200:12.0:1616513097.749317:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.163@tcp1 00000400:00000200:12.0:1616513097.749324:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.163@tcp1 routed destination 00000400:00000200:12.0:1616513097.749338:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.163@tcp1(10.31.129.163@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.749344:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.749349:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513097.749352:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513097.749354:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.749359:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.749364:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513097.749365:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513097.749417:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.749424:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.749427:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513097.749448:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.749454:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be990 00000800:00000200:18.0:1616513097.749458:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.749461:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.749641:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513097.749652:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.749660:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513097.749662:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513097.749669:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.163@tcp1 : PUT - for me 00000400:00000200:2.0:1616513097.749678:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.163@tcp1 of length 224 into portal 12 MB=0x604191e6fca40 00000400:00000200:2.0:1616513097.749684:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.163@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 36176 00000400:00000200:2.0:1616513097.749689:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.749692:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513097.749698:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.749701:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.749703:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.749764:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693355791600192 00010000:00000200:22.0:1616513097.749777:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dde80 x1693355791600192/t0(0) o400->5707a491-d51c-7f4f-c3f1-c95f353740db@10.31.129.163@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.749792:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693355791600192, offset 192 00000400:00000200:22.0:1616513097.749798:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.163@tcp1 00000400:00000200:22.0:1616513097.749806:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.163@tcp1 routed destination 00000400:00000200:22.0:1616513097.749821:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.163@tcp1(10.31.129.163@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.749827:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.749833:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.749836:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.749838:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.749842:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.749851:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.749906:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.749913:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.749916:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513097.749922:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.749926:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b760 00000800:00000200:24.0:1616513097.749930:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.749933:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.760128:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.760184:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.760193:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.760202:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.175@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.760213:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.175@tcp1 of length 224 into portal 26 MB=0x5ffd3eaf3c800 00000400:00000200:0.0:1616513097.760220:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.175@tcp1 of length 224/224 into md 0xf97d01b9 [1] + 0 00000400:00000200:0.0:1616513097.760224:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e6e8 00000400:00000200:0.0:1616513097.760227:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.760230:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513097.760234:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e6e8 00000800:00000200:16.0:1616513097.760237:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.760238:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.760240:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.760242:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513097.760307:0:246878:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660528580608 00000800:00000200:7.2:1616513097.760307:0:51351:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513097.760312:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:12.0:1616513097.760317:0:246878:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688660528580608/t0(0) o400->df086ab3-d569-d27f-20fe-b1b4c69f5421@10.31.129.175@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513097.760330:0:246878:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660528580608, offset 192 00000400:00000200:12.0:1616513097.760336:0:246878:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.175@tcp1 00000400:00000200:12.0:1616513097.760343:0:246878:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.175@tcp1 routed destination 00000400:00000200:12.0:1616513097.760356:0:246878:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.175@tcp1(10.31.129.175@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513097.760362:0:246878:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513097.760368:0:246878:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513097.760372:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.760372:0:246878:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513097.760374:0:246878:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513097.760378:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513097.760379:0:246878:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513097.760382:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:12.0:1616513097.760385:0:246878:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513097.760388:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:18.0:1616513097.760391:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.216@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.760396:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.175@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.760401:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.216@tcp1 of length 224 into portal 26 MB=0x5fffd448e8d80 00000400:00000200:24.0:1616513097.760405:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.175@tcp1 of length 224 into portal 12 MB=0x5ffd3eaf3cd40 00000400:00000200:18.0:1616513097.760408:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.216@tcp1 of length 224/224 into md 0xf97d01d9 [1] + 0 00000400:00000200:24.0:1616513097.760411:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.175@tcp1 of length 224/224 into md 0x865be291 [1] + 68176 00000400:00000200:18.0:1616513097.760412:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777edd0 00000400:00000200:18.0:1616513097.760415:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513097.760416:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.760418:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:24.0:1616513097.760419:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513097.760424:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777edd0 00000800:00000200:26.0:1616513097.760425:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:24.0:1616513097.760426:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513097.760427:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:24.0:1616513097.760429:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513097.760430:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:24.0:1616513097.760431:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513097.760432:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513097.760446:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.760446:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:0.0:1616513097.760446:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688838125555072 00000400:00000200:26.0:1616513097.760449:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513097.760451:0:51351:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513097.760454:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:0.0:1616513097.760455:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688838125555072/t0(0) o400->6cbbf34b-3d79-5250-56c1-d0202dab1be9@10.31.142.216@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513097.760457:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.760463:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bee58 00000800:00000200:26.0:1616513097.760466:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513097.760468:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688838125555072, offset 192 00000800:00000200:26.0:1616513097.760470:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513097.760473:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.216@tcp1 00000800:00000200:26.0:1616513097.760477:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513097.760480:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513097.760480:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.216@tcp1 routed destination 00000400:00000200:26.0:1616513097.760485:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.216@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.760493:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.216@tcp1 of length 224 into portal 12 MB=0x5fffd448e9000 00000400:00000200:0.0:1616513097.760493:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.216@tcp1(10.31.142.216@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513097.760499:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.216@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 36400 00000800:00000200:0.0:1616513097.760500:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513097.760504:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:26.0:1616513097.760505:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513097.760507:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660528581952 00000100:00000200:26.0:1616513097.760507:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513097.760507:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.760509:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.760512:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513097.760513:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.760516:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.760518:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.760518:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:28.0:1616513097.760520:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ce8000 x1688660528581952/t0(0) o400->dcc05e09-ba7d-7475-94e0-cf095182f0b1@10.31.129.175@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:0.0:1616513097.760520:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513097.760528:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.760533:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:28.0:1616513097.760535:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660528581952, offset 192 00000400:00000200:16.0:1616513097.760536:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513097.760537:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513097.760541:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.175@tcp1 00000400:00000200:16.0:1616513097.760543:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:28.0:1616513097.760548:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.175@tcp1 routed destination 00000400:00000200:16.0:1616513097.760548:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922dba0 00000800:00000200:16.0:1616513097.760554:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:28.0:1616513097.760564:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.175@tcp1(10.31.129.175@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.760572:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.760578:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:22.0:1616513097.760578:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688838125555712 00000800:00000200:28.0:1616513097.760580:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.760582:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.760587:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:22.0:1616513097.760590:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c694c80 x1688838125555712/t0(0) o400->29ee5b95-f120-df8a-b414-c9f464f8e4dd@10.31.142.216@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513097.760591:0:51351:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.760600:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.760604:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:22.0:1616513097.760605:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688838125555712, offset 192 00000400:00000200:18.0:1616513097.760606:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513097.760608:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513097.760611:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.216@tcp1 00000400:00000200:18.0:1616513097.760612:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.760616:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc440 00000400:00000200:22.0:1616513097.760619:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.216@tcp1 routed destination 00000800:00000200:18.0:1616513097.760624:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:22.0:1616513097.760634:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.216@tcp1(10.31.142.216@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.760640:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.760646:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.760651:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.760653:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.760658:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.760663:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.760674:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.760682:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.760685:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.760686:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.760691:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.760695:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b5c8 00000800:00000200:20.0:1616513097.760699:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.770800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.770854:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.770863:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.770871:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.205@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.770880:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.205@tcp1 of length 224 into portal 12 MB=0x5ffd4f6a40c40 00000400:00000200:0.0:1616513097.770886:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.205@tcp1 of length 224/224 into md 0x865be291 [1] + 68400 00000400:00000200:0.0:1616513097.770891:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.770893:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513097.770905:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.770907:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.770908:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.770912:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.770969:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665019649088 00010000:00000200:28.0:1616513097.770981:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86da00 x1688665019649088/t0(0) o400->c58cd2ff-babd-b9c3-6ce2-17653d5632c8@10.31.136.205@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.770995:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665019649088, offset 192 00000800:00000200:7.2:1616513097.770997:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513097.771001:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.205@tcp1 00000400:00000200:28.0:1616513097.771009:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.205@tcp1 routed destination 00000400:00000200:28.0:1616513097.771024:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.205@tcp1(10.31.136.205@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.771030:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.771036:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.771039:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.771041:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513097.771046:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513097.771054:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.771064:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513097.771073:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.205@tcp1 : PUT - for me 00000400:00000200:2.0:1616513097.771081:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.205@tcp1 of length 224 into portal 26 MB=0x5ffd4f6a40bc0 00000400:00000200:2.0:1616513097.771088:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.205@tcp1 of length 224/224 into md 0xf97d01e9 [1] + 0 00000400:00000200:2.0:1616513097.771093:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e220 00000400:00000200:2.0:1616513097.771095:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.771098:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513097.771103:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e220 00000800:00000200:2.0:1616513097.771105:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513097.771109:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513097.771111:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513097.771116:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513097.771117:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513097.771119:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513097.771119:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:0.0:1616513097.771121:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665019648960 00000400:00000200:26.0:1616513097.771126:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513097.771130:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc660 00010000:00000200:0.0:1616513097.771130:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688665019648960/t0(0) o400->dcf99e2d-e871-b7bc-8565-3339a2eacbae@10.31.136.205@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513097.771134:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:0.0:1616513097.771142:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665019648960, offset 192 00000400:00000200:0.0:1616513097.771148:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.205@tcp1 00000400:00000200:0.0:1616513097.771154:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.205@tcp1 routed destination 00000400:00000200:0.0:1616513097.771167:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.205@tcp1(10.31.136.205@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513097.771174:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513097.771179:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513097.771181:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.771183:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.771187:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.771192:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513097.771193:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513097.771246:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.771253:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.771256:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513097.771262:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.771268:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d100 00000800:00000200:24.0:1616513097.771272:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.771274:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.807436:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.807492:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.807503:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.807512:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.203@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.807523:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.203@tcp1 of length 224 into portal 26 MB=0x5ffd52cf9d900 00000400:00000200:16.0:1616513097.807530:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.203@tcp1 of length 224/224 into md 0xf97d01f9 [1] + 0 00000400:00000200:16.0:1616513097.807535:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777eaa0 00000400:00000200:16.0:1616513097.807538:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.807541:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1616513097.807545:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513097.807547:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777eaa0 00000800:00000200:16.0:1616513097.807551:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513097.807553:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513097.807555:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.807555:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513097.807558:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.807563:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.203@tcp1 : PUT - for me 00000100:00000200:4.0:1616513097.807570:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665931241728 00000400:00000200:0.0:1616513097.807572:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.203@tcp1 of length 224 into portal 12 MB=0x5ffd52cf9d980 00000400:00000200:0.0:1616513097.807577:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.203@tcp1 of length 224/224 into md 0x865be291 [1] + 68624 00010000:00000200:4.0:1616513097.807581:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688665931241728/t0(0) o400->99ce7c45-98c3-6f37-d2cc-8c4d17cd5bea@10.31.136.203@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513097.807582:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.807584:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513097.807591:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.807593:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:4.0:1616513097.807594:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665931241728, offset 192 00000800:00000200:0.0:1616513097.807595:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:4.0:1616513097.807601:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.203@tcp1 00000400:00000200:4.0:1616513097.807608:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.203@tcp1 routed destination 00000400:00000200:4.0:1616513097.807623:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.203@tcp1(10.31.136.203@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.807633:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513097.807639:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.807642:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513097.807644:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513097.807649:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513097.807655:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513097.807657:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665931241856 00000800:00000200:7.2:1616513097.807657:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:28.0:1616513097.807669:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde86e300 x1688665931241856/t0(0) o400->56a7607b-4e8b-564c-b016-726623fa2628@10.31.136.203@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.807682:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665931241856, offset 192 00000400:00000200:28.0:1616513097.807688:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.203@tcp1 00000400:00000200:28.0:1616513097.807695:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.203@tcp1 routed destination 00000400:00000200:28.0:1616513097.807709:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.203@tcp1(10.31.136.203@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513097.807714:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513097.807715:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513097.807721:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513097.807721:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513097.807724:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513097.807724:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:28.0:1616513097.807725:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.807730:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513097.807731:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513097.807735:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513097.807736:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000800:00000200:18.0:1616513097.807740:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.807743:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513097.807792:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.807799:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.807802:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513097.807803:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.807809:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.807812:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc770 00000800:00000200:20.0:1616513097.807816:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.870670:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.870740:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.870750:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.870759:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.88@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.870768:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.88@tcp1 of length 224 into portal 26 MB=0x603669a6e1540 00000400:00000200:14.0:1616513097.870775:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.88@tcp1 of length 224/224 into md 0xf97d0209 [1] + 0 00000400:00000200:14.0:1616513097.870780:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e088 00000400:00000200:14.0:1616513097.870783:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.870786:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513097.870790:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e088 00000800:00000200:26.0:1616513097.870794:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.870794:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.870797:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.870801:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.870804:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.870805:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.870814:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.88@tcp1 : PUT - for me 00000100:00000200:4.0:1616513097.870814:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692589072717120 00000400:00000200:26.0:1616513097.870822:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.88@tcp1 of length 224 into portal 12 MB=0x603669a6e15c0 00010000:00000200:4.0:1616513097.870826:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1692589072717120/t0(0) o400->caafb5d8-d29d-670d-2c04-05b651218fb1@10.31.140.88@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513097.870829:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.88@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 36624 00000400:00000200:26.0:1616513097.870834:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.870837:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:4.0:1616513097.870841:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692589072717120, offset 192 00000800:00000200:26.0:1616513097.870845:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513097.870847:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.88@tcp1 00000800:00000200:26.0:1616513097.870848:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.870850:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513097.870854:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.88@tcp1 routed destination 00000400:00000200:4.0:1616513097.870869:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.88@tcp1(10.31.140.88@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.870876:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513097.870882:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.870884:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513097.870886:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513097.870891:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.870896:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513097.870898:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513097.870916:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692589072717248 00010000:00000200:22.0:1616513097.870929:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c690d80 x1692589072717248/t0(0) o400->dee1a039-9dc6-32f1-ff4e-1491c12d6f74@10.31.140.88@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.870944:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692589072717248, offset 192 00000400:00000200:22.0:1616513097.870950:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.88@tcp1 00000800:00000200:2.0:1616513097.870950:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513097.870957:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.88@tcp1 routed destination 00000800:00000200:2.0:1616513097.870958:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.870960:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513097.870961:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513097.870967:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513097.870973:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.88@tcp1(10.31.140.88@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513097.870973:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb320 00000800:00000200:2.0:1616513097.870977:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:22.0:1616513097.870979:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.870985:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.870988:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.870990:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.870995:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.871002:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.871059:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.871066:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.871068:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513097.871075:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.871079:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bba0 00000800:00000200:0.0:1616513097.871083:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.871088:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.871094:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.871101:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.10@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.871110:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.10@tcp1 of length 224 into portal 26 MB=0x5ffdcff7b87c0 00000800:00000200:24.0:1616513097.871113:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.871116:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.10@tcp1 of length 224/224 into md 0xf97d0219 [1] + 0 00000400:00000200:0.0:1616513097.871121:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e4c8 00000400:00000200:0.0:1616513097.871123:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.871126:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513097.871131:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e4c8 00000800:00000200:0.0:1616513097.871133:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.871136:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.871138:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513097.871142:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699527727040 00010000:00000200:4.0:1616513097.871148:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688699527727040/t0(0) o400->0c7a03b5-f6a6-b269-30e3-ac43e5f976b4@10.31.136.10@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.871158:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699527727040, offset 192 00000400:00000200:4.0:1616513097.871163:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.10@tcp1 00000400:00000200:4.0:1616513097.871168:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.10@tcp1 routed destination 00000400:00000200:4.0:1616513097.871180:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.10@tcp1(10.31.136.10@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.871185:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513097.871190:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.871192:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513097.871195:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513097.871198:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.871202:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513097.871202:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513097.871215:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.871222:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.871224:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513097.871225:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513097.871231:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.871237:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbdc0 00000800:00000200:18.0:1616513097.871240:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.871371:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.871426:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.871448:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.871452:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.871461:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.10@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.871470:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.10@tcp1 of length 224 into portal 12 MB=0x5ffdcff7b8940 00000400:00000200:20.0:1616513097.871476:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.10@tcp1 of length 224/224 into md 0x865be291 [1] + 68848 00000400:00000200:20.0:1616513097.871482:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.871484:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513097.871492:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.871495:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.871497:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:31.0:1616513097.871544:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513097.871552:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000100:00000200:28.0:1616513097.871556:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699527727424 00010000:00000200:28.0:1616513097.871569:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c664380 x1688699527727424/t0(0) o400->9a21ead5-d1ef-6499-92db-4592b44d037b@10.31.136.10@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.871583:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699527727424, offset 192 00000400:00000200:28.0:1616513097.871589:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.10@tcp1 00000400:00000200:28.0:1616513097.871597:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.10@tcp1 routed destination 00000400:00000200:28.0:1616513097.871611:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.10@tcp1(10.31.136.10@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.871618:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513097.871624:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.871627:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513097.871629:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.871633:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.871639:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.871649:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.871653:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.871655:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513097.871659:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.871660:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.871663:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc880 00000800:00000200:0.0:1616513097.871665:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.874801:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.874854:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.874863:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513097.874873:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.15@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.874882:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.15@tcp1 of length 224 into portal 26 MB=0x60393d8894c00 00000400:00000200:24.0:1616513097.874888:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.15@tcp1 of length 224/224 into md 0xf97d0239 [1] + 0 00000400:00000200:24.0:1616513097.874894:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777ed48 00000400:00000200:24.0:1616513097.874896:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.874899:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513097.874904:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777ed48 00000800:00000200:24.0:1616513097.874907:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.874909:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.874909:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513097.874914:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513097.874923:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783388216320 00000800:00000200:11.2:1616513097.874931:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:4.0:1616513097.874934:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1692783388216320/t0(0) o400->2b33e6f2-fafb-126e-cd84-36d27a8291e2@10.31.140.15@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.874947:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692783388216320, offset 192 00000400:00000200:4.0:1616513097.874953:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.15@tcp1 00000400:00000200:4.0:1616513097.874961:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.15@tcp1 routed destination 00000400:00000200:4.0:1616513097.874975:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.15@tcp1(10.31.140.15@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.874981:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513097.874986:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.874989:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513097.874990:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513097.874991:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513097.874995:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513097.874999:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513097.875001:0:51363:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513097.875001:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:16.0:1616513097.875007:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.15@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.875016:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.15@tcp1 of length 224 into portal 12 MB=0x60393d8894f40 00000400:00000200:16.0:1616513097.875022:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.15@tcp1 of length 224/224 into md 0x865be291 [1] + 69072 00000400:00000200:16.0:1616513097.875027:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.875030:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.875036:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.875041:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.875043:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.875046:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513097.875062:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.875069:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513097.875072:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513097.875079:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513097.875084:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb540 00000800:00000200:14.0:1616513097.875088:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.875091:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513097.875102:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692783388217152 00010000:00000200:28.0:1616513097.875114:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3600 x1692783388217152/t0(0) o400->782f0885-f49d-1859-9327-34711524682d@10.31.140.15@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.875129:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692783388217152, offset 192 00000400:00000200:28.0:1616513097.875135:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.15@tcp1 00000400:00000200:28.0:1616513097.875142:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.15@tcp1 routed destination 00000400:00000200:28.0:1616513097.875157:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.15@tcp1(10.31.140.15@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.875164:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513097.875169:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.875172:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513097.875174:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.875179:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.875183:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.875194:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.875199:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.875202:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.875203:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.875208:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.875212:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc990 00000800:00000200:20.0:1616513097.875216:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.884541:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.884594:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.884604:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.884613:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.129@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.884623:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.129@tcp1 of length 224 into portal 12 MB=0x604a35d2bc7c0 00000400:00000200:16.0:1616513097.884629:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.129@tcp1 of length 224/224 into md 0x865be291 [1] + 69296 00000400:00000200:16.0:1616513097.884635:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.884637:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.884644:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.884647:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.884649:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.884651:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.884694:0:51364:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:28.0:1616513097.884711:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949549594560 00010000:00000200:28.0:1616513097.884724:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d4380 x1693949549594560/t0(0) o400->f3f3373b-54f9-3db2-8230-dc89b9608b9d@10.31.140.129@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.884738:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949549594560, offset 192 00000400:00000200:28.0:1616513097.884744:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.129@tcp1 00000400:00000200:28.0:1616513097.884752:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.129@tcp1 routed destination 00000800:00000200:26.0:1616513097.884760:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513097.884769:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.129@tcp1(10.31.140.129@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513097.884770:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.884775:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:26.0:1616513097.884779:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.129@tcp1 : PUT - for me 00000800:00000200:28.0:1616513097.884781:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.884784:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513097.884786:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000400:00000200:26.0:1616513097.884790:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.129@tcp1 of length 224 into portal 26 MB=0x604a35d2bc4c0 00000800:00000200:28.0:1616513097.884791:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513097.884797:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.129@tcp1 of length 224/224 into md 0xf97d0269 [1] + 0 00000400:00000200:26.0:1616513097.884802:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f100 00000400:00000200:26.0:1616513097.884804:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.884807:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513097.884811:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f100 00000800:00000200:26.0:1616513097.884814:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513097.884815:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513097.884817:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513097.884822:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513097.884823:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513097.884825:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513097.884826:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513097.884833:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513097.884837:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccaa0 00000800:00000200:18.0:1616513097.884840:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513097.884874:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949549593792 00010000:00000200:4.0:1616513097.884884:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1693949549593792/t0(0) o400->34e76dc5-6071-d702-d93e-e0bfe11dbd89@10.31.140.129@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.884897:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949549593792, offset 192 00000400:00000200:4.0:1616513097.884903:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.129@tcp1 00000400:00000200:4.0:1616513097.884910:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.129@tcp1 routed destination 00000400:00000200:4.0:1616513097.884926:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.129@tcp1(10.31.140.129@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.884932:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513097.884937:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.884939:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513097.884941:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513097.884946:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.884951:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513097.884952:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513097.885004:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.885010:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.885013:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513097.885020:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.885026:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba198 00000800:00000200:24.0:1616513097.885029:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.885032:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.893745:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.893798:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.893808:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513097.893814:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513097.893817:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.120@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.893826:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.120@tcp1 of length 224 into portal 26 MB=0x6012e12b51600 00000400:00000200:16.0:1616513097.893833:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.120@tcp1 of length 224/224 into md 0xf97d0289 [1] + 0 00000400:00000200:16.0:1616513097.893838:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777e5d8 00000400:00000200:16.0:1616513097.893841:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.893844:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513097.893848:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777e5d8 00000800:00000200:16.0:1616513097.893851:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.893854:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.893856:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513097.893867:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690147254244864 00010000:00000200:4.0:1616513097.893879:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1690147254244864/t0(0) o400->d464eee0-fadd-befe-393d-238f7e6ebc71@10.31.140.120@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.893894:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690147254244864, offset 192 00000400:00000200:4.0:1616513097.893910:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.120@tcp1 00000400:00000200:4.0:1616513097.893918:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.120@tcp1 routed destination 00000400:00000200:4.0:1616513097.893932:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.120@tcp1(10.31.140.120@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.893938:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513097.893943:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.893946:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513097.893948:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513097.893952:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513097.893958:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513097.893959:0:51365:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.894015:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.894022:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.894024:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.894031:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.894037:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba2a8 00000800:00000200:2.0:1616513097.894040:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.894043:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.894839:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513097.894866:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.894875:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513097.894884:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.120@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.894892:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.120@tcp1 of length 224 into portal 12 MB=0x6012e12b51cc0 00000400:00000200:18.0:1616513097.894900:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.120@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 36848 00000400:00000200:18.0:1616513097.894906:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.894908:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513097.894915:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.894918:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.894918:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.894922:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.894981:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690147254246592 00010000:00000200:22.0:1616513097.894993:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464620900 x1690147254246592/t0(0) o400->d96e627f-dc13-f2a0-02aa-c67fd10276f4@10.31.140.120@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.895007:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690147254246592, offset 192 00000400:00000200:22.0:1616513097.895013:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.120@tcp1 00000400:00000200:22.0:1616513097.895020:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.120@tcp1 routed destination 00000400:00000200:22.0:1616513097.895035:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.120@tcp1(10.31.140.120@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.895042:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.895047:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.895050:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.895052:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.895057:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.895062:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.895117:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.895123:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.895126:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513097.895132:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.895137:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bd38 00000800:00000200:24.0:1616513097.895142:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.895145:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.916272:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513097.916328:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.916338:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513097.916347:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.80@tcp1 : PUT - for me 00000400:00000200:0.0:1616513097.916357:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.80@tcp1 of length 224 into portal 26 MB=0x60182bf10c780 00000400:00000200:0.0:1616513097.916364:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.80@tcp1 of length 224/224 into md 0xf97d0299 [1] + 0 00000400:00000200:0.0:1616513097.916369:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fe48 00000400:00000200:0.0:1616513097.916372:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.916375:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513097.916379:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fe48 00000800:00000200:16.0:1616513097.916382:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.916383:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.916385:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513097.916387:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513097.916456:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510923188096 00010000:00000200:4.0:1616513097.916467:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1690510923188096/t0(0) o400->65edb78c-0d5b-ddb7-1a20-5414406b97e5@10.31.133.80@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.916482:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510923188096, offset 192 00000400:00000200:4.0:1616513097.916488:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.80@tcp1 00000400:00000200:4.0:1616513097.916507:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.80@tcp1 routed destination 00000400:00000200:4.0:1616513097.916520:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.80@tcp1(10.31.133.80@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.916526:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513097.916531:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.916534:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513097.916535:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513097.916540:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.916546:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513097.916546:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513097.916601:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.916608:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.916611:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.916618:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.916623:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bacc0 00000800:00000200:2.0:1616513097.916627:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.916630:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.916988:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.917042:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.917051:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.917061:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.80@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.917070:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.80@tcp1 of length 224 into portal 12 MB=0x60182bf10c940 00000400:00000200:14.0:1616513097.917076:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.80@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 37072 00000400:00000200:14.0:1616513097.917081:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.917084:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513097.917091:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.917094:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.917095:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.917099:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.917156:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510923188544 00010000:00000200:22.0:1616513097.917168:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bf9680 x1690510923188544/t0(0) o400->0932db64-aa86-3d48-984f-94d23584815a@10.31.133.80@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.917194:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510923188544, offset 192 00000400:00000200:22.0:1616513097.917201:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.80@tcp1 00000400:00000200:22.0:1616513097.917208:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.80@tcp1 routed destination 00000400:00000200:22.0:1616513097.917223:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.80@tcp1(10.31.133.80@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.917230:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513097.917236:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.917239:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.917241:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.917246:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.917253:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.917310:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.917316:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.917319:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513097.917326:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.917330:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4acc0 00000800:00000200:24.0:1616513097.917334:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.917337:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.925446:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513097.925506:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.925515:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.925524:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.98@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.925534:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.98@tcp1 of length 224 into portal 12 MB=0x5f906ce1cdb00 00000400:00000200:16.0:1616513097.925541:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.98@tcp1 of length 224/224 into md 0x865be291 [1] + 69520 00000400:00000200:16.0:1616513097.925546:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.925550:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.925557:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.925559:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.925560:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.925565:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513097.925619:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681182506670848 00010000:00000200:28.0:1616513097.925630:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5100 x1681182506670848/t0(0) o400->89f393ec-abbd-83eb-1945-bca0d9236db6@10.31.128.98@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.925643:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681182506670848, offset 192 00000400:00000200:28.0:1616513097.925649:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.98@tcp1 00000400:00000200:28.0:1616513097.925657:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.98@tcp1 routed destination 00000400:00000200:28.0:1616513097.925670:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.98@tcp1(10.31.128.98@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.925676:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513097.925681:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.925684:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513097.925686:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513097.925690:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.925697:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.925752:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.925759:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.925761:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.925768:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.925772:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccbb0 00000800:00000200:2.0:1616513097.925776:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.925779:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513097.926548:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.926602:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.926611:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513097.926620:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.81@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.926629:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.81@tcp1 of length 224 into portal 26 MB=0x60182fe68f1c0 00000400:00000200:24.0:1616513097.926636:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.81@tcp1 of length 224/224 into md 0xf97d02b9 [1] + 0 00000400:00000200:24.0:1616513097.926641:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f6d8 00000400:00000200:24.0:1616513097.926643:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.926646:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513097.926651:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f6d8 00000800:00000200:24.0:1616513097.926654:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.926656:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.926657:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513097.926660:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513097.926664:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511985930688 00010000:00000200:4.0:1616513097.926670:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1690511985930688/t0(0) o400->a5d1d5ad-80e2-4844-8b39-e696ecb8b746@10.31.133.81@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.926681:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511985930688, offset 192 00000400:00000200:4.0:1616513097.926686:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.81@tcp1 00000400:00000200:4.0:1616513097.926692:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.81@tcp1 routed destination 00000400:00000200:4.0:1616513097.926707:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.81@tcp1(10.31.133.81@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.926713:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513097.926717:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.926719:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513097.926721:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513097.926725:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.926729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513097.926729:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513097.926782:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.926790:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.926793:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513097.926795:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.926799:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.926805:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb430 00000800:00000200:0.0:1616513097.926808:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.927008:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513097.927062:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.927071:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513097.927080:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.81@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.927089:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.81@tcp1 of length 224 into portal 12 MB=0x60182fe68f340 00000400:00000200:18.0:1616513097.927095:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.81@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 37296 00000400:00000200:18.0:1616513097.927100:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.927103:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513097.927109:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.927112:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.927114:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.927116:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513097.927173:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511985931072 00010000:00000200:22.0:1616513097.927184:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464625a00 x1690511985931072/t0(0) o400->a44eec90-e909-451a-7e90-1d6633e84811@10.31.133.81@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.927199:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511985931072, offset 192 00000400:00000200:22.0:1616513097.927205:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.81@tcp1 00000400:00000200:22.0:1616513097.927213:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.81@tcp1 routed destination 00000400:00000200:22.0:1616513097.927227:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.81@tcp1(10.31.133.81@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513097.927234:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513097.927240:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.927243:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513097.927245:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513097.927249:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.927254:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.927308:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.927315:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.927318:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.927324:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.927328:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4add0 00000800:00000200:2.0:1616513097.927334:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.927337:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.927385:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513097.927401:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.927407:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513097.927409:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513097.927414:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.98@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.927420:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.98@tcp1 of length 224 into portal 26 MB=0x5f906ce1cda80 00000400:00000200:18.0:1616513097.927426:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.98@tcp1 of length 224/224 into md 0xf97d02c9 [1] + 0 00000400:00000200:18.0:1616513097.927430:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777f5c8 00000400:00000200:18.0:1616513097.927432:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.927448:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513097.927452:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777f5c8 00000800:00000200:18.0:1616513097.927455:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.927457:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.927458:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513097.927462:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681182506670720 00010000:00000200:4.0:1616513097.927467:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1681182506670720/t0(0) o400->269ef27e-0f4f-ed07-6ef3-109e1f96b0a0@10.31.128.98@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.927476:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681182506670720, offset 192 00000400:00000200:4.0:1616513097.927480:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.98@tcp1 00000400:00000200:4.0:1616513097.927485:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.98@tcp1 routed destination 00000400:00000200:4.0:1616513097.927497:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.98@tcp1(10.31.128.98@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.927502:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513097.927506:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.927508:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513097.927509:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513097.927512:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.927516:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513097.927516:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513097.927527:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.927532:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.927534:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.927536:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513097.927541:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513097.927547:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba220 00000800:00000200:20.0:1616513097.927552:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.939165:0:51370:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513097.939220:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513097.939229:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513097.939238:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.87@tcp1 : PUT - for me 00000400:00000200:2.0:1616513097.939248:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.87@tcp1 of length 224 into portal 26 MB=0x6054a5ff8e140 00000400:00000200:2.0:1616513097.939255:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.11.87@tcp1 of length 224/224 into md 0xf97d02e9 [1] + 0 00000400:00000200:2.0:1616513097.939259:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dd777fba0 00000400:00000200:2.0:1616513097.939262:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513097.939278:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513097.939281:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513097.939281:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd777fba0 00000800:00000200:2.0:1616513097.939284:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.939287:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513097.939289:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.939292:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513097.939292:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513097.939301:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.11.87@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.939310:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.11.87@tcp1 of length 224 into portal 12 MB=0x6054a5ff8e4c0 00000400:00000200:26.0:1616513097.939316:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.11.87@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 37520 00000400:00000200:26.0:1616513097.939321:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.939324:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513097.939330:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.939334:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.939335:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513097.939344:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694666856128832 00010000:00000200:4.0:1616513097.939354:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1694666856128832/t0(0) o400->88fa82b7-c32d-cc88-a1e1-b2d8ee28a133@10.31.11.87@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.939367:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694666856128832, offset 192 00000400:00000200:4.0:1616513097.939373:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.87@tcp1 00000400:00000200:4.0:1616513097.939380:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.87@tcp1 routed destination 00000400:00000200:4.0:1616513097.939394:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.87@tcp1(10.31.11.87@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.939400:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513097.939405:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.939409:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513097.939411:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513097.939412:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694666856129728 00000800:00000200:4.0:1616513097.939416:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.939420:0:51370:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513097.939422:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:22.0:1616513097.939424:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c690480 x1694666856129728/t0(0) o400->075301c8-1b3b-8988-120e-30846dbaad6f@10.31.11.87@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513097.939456:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694666856129728, offset 192 00000400:00000200:22.0:1616513097.939463:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.11.87@tcp1 00000400:00000200:22.0:1616513097.939470:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.11.87@tcp1 routed destination 00000800:00000200:18.0:1616513097.939472:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.939479:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.939481:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513097.939483:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513097.939486:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.11.87@tcp1(10.31.11.87@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513097.939489:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513097.939494:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513097.939495:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbba0 00000800:00000200:18.0:1616513097.939498:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:22.0:1616513097.939500:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513097.939502:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513097.939504:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513097.939509:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.939514:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.939565:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513097.939571:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513097.939574:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513097.939580:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513097.939583:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a3b8 00000800:00000200:16.0:1616513097.939587:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.939590:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.942327:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513097.942382:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.942391:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513097.942400:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.149@tcp1 : PUT - for me 00000400:00000200:24.0:1616513097.942410:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.149@tcp1 of length 224 into portal 26 MB=0x603d094614a40 00000400:00000200:24.0:1616513097.942416:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.104.149@tcp1 of length 224/224 into md 0xf97d0309 [1] + 0 00000400:00000200:24.0:1616513097.942421:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be198 00000400:00000200:24.0:1616513097.942424:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513097.942426:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513097.942431:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be198 00000800:00000200:20.0:1616513097.942447:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.942448:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.942451:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513097.942454:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513097.942457:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.942457:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.942465:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.149@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.942473:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.149@tcp1 of length 224 into portal 12 MB=0x603d094614c80 00000400:00000200:20.0:1616513097.942479:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.149@tcp1 of length 224/224 into md 0x865be291 [1] + 69744 00000400:00000200:20.0:1616513097.942484:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.942487:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:4.0:1616513097.942490:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693044237748800 00000800:00000200:20.0:1616513097.942493:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.942496:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.942498:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:4.0:1616513097.942499:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1693044237748800/t0(0) o400->38f8fead-6e40-e8d5-5cf0-74ee16d5e8b5@10.242.104.149@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.942512:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693044237748800, offset 192 00000400:00000200:4.0:1616513097.942517:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.149@tcp1 00000400:00000200:4.0:1616513097.942524:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.149@tcp1 routed destination 00000400:00000200:4.0:1616513097.942537:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.149@tcp1(10.242.104.149@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.942542:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513097.942548:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.942550:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513097.942552:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513097.942556:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513097.942562:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513097.942563:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513097.942572:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693044237749376 00010000:00000200:28.0:1616513097.942584:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c667500 x1693044237749376/t0(0) o400->a3bb5799-414e-7876-7ce0-854e915538c3@10.242.104.149@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513097.942598:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693044237749376, offset 192 00000400:00000200:28.0:1616513097.942605:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.149@tcp1 00000400:00000200:28.0:1616513097.942612:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.149@tcp1 routed destination 00000800:00000200:26.0:1616513097.942618:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513097.942625:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513097.942627:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.149@tcp1(10.242.104.149@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513097.942628:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513097.942631:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513097.942634:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:26.0:1616513097.942634:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:28.0:1616513097.942639:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:26.0:1616513097.942640:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba3b8 00000800:00000200:28.0:1616513097.942642:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513097.942643:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513097.942646:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.942651:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.942656:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.942670:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.942677:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.942680:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.942681:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.942686:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.942691:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcccc0 00000800:00000200:0.0:1616513097.942694:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.950419:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513097.950484:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513097.950494:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513097.950502:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.126@tcp1 : PUT - for me 00000400:00000200:20.0:1616513097.950511:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.126@tcp1 of length 224 into portal 26 MB=0x603a3d1446580 00000400:00000200:20.0:1616513097.950518:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.126@tcp1 of length 224/224 into md 0xf97d0319 [1] + 0 00000400:00000200:20.0:1616513097.950522:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfb18 00000400:00000200:20.0:1616513097.950525:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.950528:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513097.950532:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfb18 00000800:00000200:20.0:1616513097.950535:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513097.950538:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.950539:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513097.950546:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.950549:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.950549:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.950558:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.126@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.950566:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.126@tcp1 of length 224 into portal 12 MB=0x603a3d1446600 00000400:00000200:16.0:1616513097.950573:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.126@tcp1 of length 224/224 into md 0x865be291 [1] + 69968 00000400:00000200:16.0:1616513097.950578:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513097.950581:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.950587:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.950590:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:4.0:1616513097.950591:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692851985737088 00000800:00000200:16.0:1616513097.950592:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:4.0:1616513097.950601:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1692851985737088/t0(0) o400->f16d34ba-287d-81d7-d8d4-0629f7300bb9@10.31.143.126@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.950613:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692851985737088, offset 192 00000400:00000200:4.0:1616513097.950619:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.126@tcp1 00000400:00000200:4.0:1616513097.950625:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.126@tcp1 routed destination 00000400:00000200:4.0:1616513097.950638:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.126@tcp1(10.31.143.126@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.950643:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513097.950648:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:28.0:1616513097.950651:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692851985737216 00000800:00000200:4.0:1616513097.950651:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513097.950652:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513097.950657:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513097.950662:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:28.0:1616513097.950664:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d1f80 x1692851985737216/t0(0) o400->b4e4811f-1ef1-b671-9025-32bd29650b2d@10.31.143.126@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513097.950664:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513097.950678:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692851985737216, offset 192 00000400:00000200:28.0:1616513097.950684:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.126@tcp1 00000400:00000200:28.0:1616513097.950691:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.126@tcp1 routed destination 00000400:00000200:28.0:1616513097.950706:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.126@tcp1(10.31.143.126@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.950713:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513097.950718:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513097.950719:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513097.950721:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513097.950723:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513097.950726:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513097.950728:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513097.950728:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513097.950733:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513097.950735:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.950741:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba908 00000800:00000200:2.0:1616513097.950744:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513097.950747:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513097.950796:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.950803:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.950806:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513097.950807:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.950812:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.950816:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccdd0 00000800:00000200:0.0:1616513097.950820:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513097.955920:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513097.955964:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513097.955976:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513097.955985:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513097.955994:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.117@tcp1 : PUT - for me 00000400:00000200:16.0:1616513097.956004:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.117@tcp1 of length 224 into portal 12 MB=0x5fffae389e640 00000400:00000200:16.0:1616513097.956010:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.117@tcp1 of length 224/224 into md 0x865be291 [1] + 70192 00000400:00000200:16.0:1616513097.956015:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513097.956016:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513097.956018:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513097.956024:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:14.0:1616513097.956025:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:16.0:1616513097.956027:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513097.956028:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513097.956031:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513097.956033:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.117@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.956042:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.117@tcp1 of length 224 into portal 26 MB=0x5fffae389e5c0 00000400:00000200:14.0:1616513097.956048:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.117@tcp1 of length 224/224 into md 0xf97d0339 [1] + 0 00000400:00000200:14.0:1616513097.956052:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfc28 00000400:00000200:14.0:1616513097.956055:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.956057:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513097.956061:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfc28 00000800:00000200:14.0:1616513097.956064:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.956066:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.956068:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.956070:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513097.956110:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688827907925568 00010000:00000200:28.0:1616513097.956122:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3a80 x1688827907925568/t0(0) o400->f31c87f9-b7c5-e5e1-3519-96cfb5426089@10.31.138.117@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:4.0:1616513097.956125:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688827907925440 00010000:00000200:4.0:1616513097.956135:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688827907925440/t0(0) o400->c291518d-f94a-a576-43a8-873e7bcd9cc4@10.31.138.117@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513097.956137:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688827907925568, offset 192 00000400:00000200:28.0:1616513097.956143:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.117@tcp1 00000100:00000200:4.0:1616513097.956147:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688827907925440, offset 192 00000400:00000200:28.0:1616513097.956151:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.117@tcp1 routed destination 00000400:00000200:4.0:1616513097.956152:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.117@tcp1 00000400:00000200:4.0:1616513097.956160:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.117@tcp1 routed destination 00000400:00000200:28.0:1616513097.956166:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.117@tcp1(10.31.138.117@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513097.956173:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:4.0:1616513097.956173:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.117@tcp1(10.31.138.117@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.956179:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513097.956180:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.956183:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513097.956184:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513097.956185:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513097.956186:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513097.956188:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513097.956190:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513097.956192:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513097.956195:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513097.956198:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513097.956198:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513097.956261:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513097.956267:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513097.956268:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.956271:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513097.956274:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513097.956277:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513097.956277:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.956281:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccee0 00000400:00000200:20.0:1616513097.956283:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513097.956285:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513097.956288:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb298 00000800:00000200:2.0:1616513097.956288:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513097.956292:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513097.956295:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.958465:0:51372:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.958478:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.958486:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513097.958495:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.5@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.958504:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.5@tcp1 of length 224 into portal 26 MB=0x5ffd422439740 00000400:00000200:14.0:1616513097.958511:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.5@tcp1 of length 224/224 into md 0xf97d0349 [1] + 0 00000400:00000200:14.0:1616513097.958516:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf188 00000400:00000200:14.0:1616513097.958518:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.958521:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513097.958525:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf188 00000800:00000200:14.0:1616513097.958528:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513097.958531:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513097.958532:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513097.958536:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513097.958595:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661456557888 00010000:00000200:4.0:1616513097.958606:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688661456557888/t0(0) o400->d0ad09cf-3ce5-f809-7bb8-742a0dfffcc6@10.31.133.5@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513097.958620:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661456557888, offset 192 00000400:00000200:4.0:1616513097.958627:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.5@tcp1 00000400:00000200:4.0:1616513097.958634:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.5@tcp1 routed destination 00000400:00000200:4.0:1616513097.958649:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.5@tcp1(10.31.133.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513097.958655:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513097.958661:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.958664:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513097.958666:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513097.958671:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.958675:0:51372:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513097.958677:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513097.958728:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513097.958735:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513097.958738:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513097.958740:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513097.958744:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513097.958750:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba330 00000800:00000200:2.0:1616513097.958755:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513097.959252:0:51372:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513097.959263:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.959271:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513097.959273:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513097.959279:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.5@tcp1 : PUT - for me 00000400:00000200:26.0:1616513097.959287:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.5@tcp1 of length 224 into portal 12 MB=0x5ffd4224398c0 00000400:00000200:26.0:1616513097.959293:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.5@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 37744 00000400:00000200:26.0:1616513097.959300:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513097.959302:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513097.959310:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513097.959312:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513097.959314:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513097.959328:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661456558272 00010000:00000200:30.0:1616513097.959338:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695100 x1688661456558272/t0(0) o400->9ad9993d-fcef-89ee-ab8d-4588af352b72@10.31.133.5@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513097.959351:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661456558272, offset 192 00000400:00000200:30.0:1616513097.959357:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.5@tcp1 00000400:00000200:30.0:1616513097.959363:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.5@tcp1 routed destination 00000400:00000200:30.0:1616513097.959376:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.5@tcp1(10.31.133.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513097.959382:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513097.959387:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513097.959389:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513097.959391:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513097.959395:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.959440:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513097.959453:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513097.959459:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513097.959462:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513097.959468:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513097.959472:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef298 00000800:00000200:24.0:1616513097.959477:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513097.959480:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.990727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513097.990777:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513097.990783:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513097.990793:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513097.990802:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.69@tcp1 : PUT - for me 00000400:00000200:18.0:1616513097.990811:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.69@tcp1 of length 224 into portal 12 MB=0x5ffd5705fe700 00000400:00000200:18.0:1616513097.990818:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.69@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 37968 00000400:00000200:18.0:1616513097.990823:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513097.990826:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513097.990832:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.990835:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513097.990837:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513097.990840:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513097.990847:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667062003456 00000800:00000200:20.0:1616513097.990855:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:30.0:1616513097.990856:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464627500 x1688667062003456/t0(0) o400->e157e41a-febf-6711-b67a-5925d4da4dc3@10.31.136.69@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:20.0:1616513097.990866:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000100:00000200:30.0:1616513097.990867:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667062003456, offset 192 00000400:00000200:30.0:1616513097.990872:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.69@tcp1 00000400:00000200:20.0:1616513097.990874:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.139@tcp1 : PUT - for me 00000400:00000200:30.0:1616513097.990878:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.69@tcp1 routed destination 00000400:00000200:20.0:1616513097.990883:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.139@tcp1 of length 224 into portal 26 MB=0x5ffd9184d8f80 00000400:00000200:30.0:1616513097.990890:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.69@tcp1(10.31.136.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513097.990890:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.139@tcp1 of length 224/224 into md 0xf97d0359 [1] + 0 00000400:00000200:20.0:1616513097.990894:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be770 00000800:00000200:30.0:1616513097.990895:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513097.990896:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513097.990899:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:30.0:1616513097.990900:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513097.990902:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513097.990903:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be770 00000800:00000200:30.0:1616513097.990904:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513097.990906:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513097.990908:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513097.990908:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513097.990910:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513097.990913:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513097.990913:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513097.990923:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:0.0:1616513097.990923:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513097.990930:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:0.0:1616513097.990931:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.69@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.990933:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513097.990939:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.69@tcp1 of length 224 into portal 26 MB=0x5ffd5705fe600 00000400:00000200:14.0:1616513097.990940:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.990946:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.69@tcp1 of length 224/224 into md 0xf97d0369 [1] + 0 00000400:00000200:0.0:1616513097.990950:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf5c8 00000400:00000200:14.0:1616513097.990953:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee2a8 00000400:00000200:0.0:1616513097.990953:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513097.990956:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513097.990957:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.990960:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513097.990966:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf5c8 00000800:00000200:0.0:1616513097.990969:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.990972:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:4.0:1616513097.990974:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688682764275584 00000800:00000200:0.0:1616513097.990974:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513097.990976:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:4.0:1616513097.990985:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688682764275584/t0(0) o400->dcae5647-90df-6318-747e-8320adb1110b@10.31.129.139@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513097.990988:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667062003200 00000100:00000200:4.0:1616513097.990998:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688682764275584, offset 192 00010000:00000200:19.0:1616513097.991005:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688667062003200/t0(0) o400->171ed33b-320f-b623-b73d-4ca6668cf2b2@10.31.136.69@tcp1:54/0 lens 224/224 e 0 to 0 dl 1616513189 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:4.0:1616513097.991006:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.139@tcp1 00000400:00000200:4.0:1616513097.991013:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.139@tcp1 routed destination 00000100:00000200:19.0:1616513097.991022:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667062003200, offset 192 00000400:00000200:4.0:1616513097.991027:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.139@tcp1(10.31.129.139@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:19.0:1616513097.991029:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.69@tcp1 00000800:00000200:4.0:1616513097.991033:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:19.0:1616513097.991038:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.69@tcp1 routed destination 00000800:00000200:4.0:1616513097.991038:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513097.991041:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513097.991043:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513097.991048:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.991053:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:19.0:1616513097.991055:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.69@tcp1(10.31.136.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:4.0:1616513097.991055:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:19.0:1616513097.991062:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:19.0:1616513097.991070:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513097.991073:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513097.991075:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:19.0:1616513097.991082:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513097.991086:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:19.0:1616513097.991094:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513097.991095:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513097.991099:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513097.991102:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513097.991104:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513097.991105:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513097.991108:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513097.991112:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:18.0:1616513097.991113:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17064c8 00000400:00000200:16.0:1616513097.991115:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513097.991116:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513097.991117:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513097.991121:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513097.991127:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baff0 00000800:00000200:16.0:1616513097.991130:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513097.991250:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513097.991258:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.991264:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513097.991267:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513097.991272:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.139@tcp1 : PUT - for me 00000400:00000200:14.0:1616513097.991280:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.139@tcp1 of length 224 into portal 12 MB=0x5ffd9184d9380 00000400:00000200:14.0:1616513097.991287:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.139@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 38192 00000400:00000200:14.0:1616513097.991292:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513097.991294:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513097.991301:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513097.991304:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513097.991306:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513097.991310:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688682764276608 00010000:00000200:30.0:1616513097.991314:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c9bff980 x1688682764276608/t0(0) o400->38fc30b0-fccd-0682-9d82-79eeaf32ad96@10.31.129.139@tcp1:723/0 lens 224/224 e 0 to 0 dl 1616513103 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513097.991323:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688682764276608, offset 192 00000400:00000200:30.0:1616513097.991327:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.139@tcp1 00000400:00000200:30.0:1616513097.991332:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.139@tcp1 routed destination 00000400:00000200:30.0:1616513097.991344:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.139@tcp1(10.31.129.139@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513097.991348:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513097.991352:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513097.991354:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513097.991355:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513097.991357:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513097.991363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513097.991372:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513097.991376:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513097.991378:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513097.991382:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513097.991396:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513097.991399:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeb28 00000800:00000200:0.0:1616513097.991401:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.006363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.006419:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.006429:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.006450:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.127@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.006460:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.127@tcp1 of length 224 into portal 26 MB=0x602bed54381c0 00000400:00000200:2.0:1616513098.006467:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.127@tcp1 of length 224/224 into md 0xf97d0379 [1] + 0 00000400:00000200:2.0:1616513098.006471:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02beff0 00000800:00000200:18.0:1616513098.006473:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513098.006474:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.006477:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.006482:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beff0 00000800:00000200:18.0:1616513098.006484:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513098.006485:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513098.006489:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513098.006490:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.006492:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.127@tcp1 : PUT - for me 00000800:00000200:2.0:1616513098.006493:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513098.006501:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.127@tcp1 of length 224 into portal 12 MB=0x602bed5438240 00000400:00000200:18.0:1616513098.006508:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.127@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 38416 00000400:00000200:18.0:1616513098.006513:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.006516:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513098.006522:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.006525:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.006527:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513098.006536:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691868505276992 00010000:00000200:30.0:1616513098.006544:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621200 x1691868505276992/t0(0) o400->611d6d2d-2340-0152-991e-bc6d40fcd8ea@10.31.142.127@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:19.0:1616513098.006550:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691868505276864 00000100:00000200:30.0:1616513098.006554:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691868505276992, offset 192 00000400:00000200:30.0:1616513098.006558:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.127@tcp1 00010000:00000200:19.0:1616513098.006562:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1691868505276864/t0(0) o400->730aefce-2fd3-9a60-f6fc-f4c07dd0a0f6@10.31.142.127@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:30.0:1616513098.006564:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.127@tcp1 routed destination 00000400:00000200:30.0:1616513098.006575:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.127@tcp1(10.31.142.127@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:19.0:1616513098.006576:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691868505276864, offset 192 00000800:00000200:30.0:1616513098.006580:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:19.0:1616513098.006583:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.127@tcp1 00000800:00000200:30.0:1616513098.006584:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513098.006586:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513098.006588:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513098.006591:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:19.0:1616513098.006592:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.127@tcp1 routed destination 00000800:00000200:7.2:1616513098.006596:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:19.0:1616513098.006608:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.127@tcp1(10.31.142.127@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.006615:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:19.0:1616513098.006621:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.006624:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:19.0:1616513098.006628:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:19.0:1616513098.006634:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.006639:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:19.0:1616513098.006641:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513098.006647:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.006654:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.006657:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513098.006658:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513098.006663:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.006668:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef5c8 00000800:00000200:14.0:1616513098.006671:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513098.006690:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.006697:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.006700:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.006706:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.006712:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17067f8 00000800:00000200:20.0:1616513098.006716:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.006719:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.008540:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513098.008594:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.008603:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.008614:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.220@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.008623:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.220@tcp1 of length 224 into portal 26 MB=0x6000b6e247e80 00000400:00000200:18.0:1616513098.008630:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.220@tcp1 of length 224/224 into md 0xf97d0389 [1] + 0 00000400:00000200:18.0:1616513098.008635:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be908 00000400:00000200:18.0:1616513098.008638:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.008641:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513098.008646:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be908 00000800:00000200:18.0:1616513098.008649:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.008651:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513098.008653:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.008655:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.008666:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.008675:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.220@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.008683:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.220@tcp1 of length 224 into portal 12 MB=0x6000b6e247f00 00000400:00000200:26.0:1616513098.008689:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.220@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 38640 00000400:00000200:26.0:1616513098.008695:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.008697:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513098.008704:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.008707:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:19.0:1616513098.008708:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688898952789632 00000800:00000200:26.0:1616513098.008709:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:30.0:1616513098.008712:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688898952789760 00000800:00000200:26.0:1616513098.008712:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:30.0:1616513098.008717:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c695580 x1688898952789760/t0(0) o400->5afb2893-04e4-68b5-83a9-3cdff71873b0@10.31.136.220@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:19.0:1616513098.008719:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688898952789632/t0(0) o400->81fa5f41-368b-a5d0-0a83-df9f29f4373f@10.31.136.220@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513098.008725:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688898952789760, offset 192 00000400:00000200:30.0:1616513098.008729:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.220@tcp1 00000100:00000200:19.0:1616513098.008732:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688898952789632, offset 192 00000400:00000200:30.0:1616513098.008733:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.220@tcp1 routed destination 00000400:00000200:19.0:1616513098.008738:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.220@tcp1 00000400:00000200:30.0:1616513098.008744:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.220@tcp1(10.31.136.220@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:19.0:1616513098.008746:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.220@tcp1 routed destination 00000800:00000200:30.0:1616513098.008749:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513098.008752:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513098.008754:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513098.008756:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513098.008758:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:19.0:1616513098.008761:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.220@tcp1(10.31.136.220@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513098.008763:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:19.0:1616513098.008768:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:19.0:1616513098.008774:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.008776:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:19.0:1616513098.008778:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:19.0:1616513098.008782:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.008787:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:19.0:1616513098.008789:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513098.008814:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.008821:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.008824:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513098.008826:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513098.008830:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.008834:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eea18 00000800:00000200:14.0:1616513098.008837:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513098.008844:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.008851:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.008854:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.008861:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.008868:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706ff0 00000800:00000200:24.0:1616513098.008872:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.008875:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.008880:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513098.008883:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.008885:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.008886:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.008888:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.037255:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.037312:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.037322:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.037332:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.30@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.037343:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.30@tcp1 of length 224 into portal 26 MB=0x6041a0f169640 00000400:00000200:26.0:1616513098.037350:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.30@tcp1 of length 224/224 into md 0xf97d03b9 [1] + 0 00000400:00000200:26.0:1616513098.037355:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf298 00000400:00000200:26.0:1616513098.037359:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.037362:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513098.037367:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf298 00000800:00000200:2.0:1616513098.037367:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.037371:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.037374:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.037376:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.037432:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359829063232 00000800:00000200:7.2:1616513098.037443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:19.0:1616513098.037456:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1693359829063232/t0(0) o400->1126cb8e-63ed-ac50-05ee-1a663e765098@10.31.129.30@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.037470:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693359829063232, offset 192 00000400:00000200:19.0:1616513098.037476:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.30@tcp1 00000400:00000200:19.0:1616513098.037484:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.30@tcp1 routed destination 00000800:00000200:18.0:1616513098.037498:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:19.0:1616513098.037499:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.30@tcp1(10.31.129.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.037505:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:18.0:1616513098.037508:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:19.0:1616513098.037512:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513098.037512:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:19.0:1616513098.037515:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:19.0:1616513098.037517:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513098.037518:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.30@tcp1 : PUT - for me 00000800:00000200:19.0:1616513098.037523:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:7.2:1616513098.037527:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513098.037528:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.30@tcp1 of length 224 into portal 12 MB=0x6041a0f1699c0 00000100:00000200:19.0:1616513098.037529:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:18.0:1616513098.037535:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.30@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 38864 00000800:00000200:2.0:1616513098.037537:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:18.0:1616513098.037541:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.037544:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513098.037544:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513098.037547:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513098.037548:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513098.037553:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513098.037554:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:18.0:1616513098.037556:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513098.037558:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.037560:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706000 00000800:00000200:2.0:1616513098.037563:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513098.037571:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693359829064128 00010000:00000200:6.0:1616513098.037582:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623180 x1693359829064128/t0(0) o400->7149e3e7-0b9d-941a-56cc-5cb113b853fd@10.31.129.30@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513098.037593:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693359829064128, offset 192 00000400:00000200:6.0:1616513098.037597:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.30@tcp1 00000400:00000200:6.0:1616513098.037603:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.30@tcp1 routed destination 00000400:00000200:6.0:1616513098.037614:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.30@tcp1(10.31.129.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513098.037620:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:6.0:1616513098.037626:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513098.037628:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:6.0:1616513098.037629:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513098.037633:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.037642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.037698:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.037705:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.037707:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.037714:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.037718:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01cb0 00000800:00000200:16.0:1616513098.037722:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.037725:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.044622:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.044679:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.044688:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.044697:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.150@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.044706:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.150@tcp1 of length 224 into portal 12 MB=0x5ffe108468e00 00000400:00000200:0.0:1616513098.044713:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.150@tcp1 of length 224/224 into md 0x865be291 [1] + 70416 00000400:00000200:0.0:1616513098.044718:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.044721:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513098.044727:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.044730:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.044732:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.044735:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.044756:0:51382:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:28.0:1616513098.044792:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716855119360 00010000:00000200:28.0:1616513098.044804:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cde868480 x1688716855119360/t0(0) o400->0236b85c-a87e-8b06-cd87-e7e02df7ed73@10.31.140.150@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513098.044815:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513098.044819:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688716855119360, offset 192 00000800:00000200:14.0:1616513098.044824:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:28.0:1616513098.044827:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.150@tcp1 00000400:00000200:14.0:1616513098.044834:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.150@tcp1 : PUT - for me 00000400:00000200:28.0:1616513098.044835:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.150@tcp1 routed destination 00000400:00000200:28.0:1616513098.044850:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.150@tcp1(10.31.140.150@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.044858:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.044858:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.150@tcp1 of length 224 into portal 26 MB=0x5ffe108468c80 00000800:00000200:28.0:1616513098.044864:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513098.044865:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.150@tcp1 of length 224/224 into md 0xf97d03c9 [1] + 0 00000800:00000200:28.0:1616513098.044867:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513098.044869:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513098.044869:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513098.044874:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.044877:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf100 00000800:00000200:7.2:1616513098.044878:0:51382:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513098.044879:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.044884:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.044889:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf100 00000800:00000200:14.0:1616513098.044892:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513098.044895:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513098.044897:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.044900:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.044904:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.044906:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513098.044911:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.044914:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccff0 00000800:00000200:14.0:1616513098.044917:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.044919:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.044955:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688716855118976 00010000:00000200:19.0:1616513098.044966:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688716855118976/t0(0) o400->d4fbfac8-12c1-01d3-b257-3c27bfc3046d@10.31.140.150@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.044983:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688716855118976, offset 192 00000400:00000200:19.0:1616513098.044990:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.150@tcp1 00000400:00000200:19.0:1616513098.044997:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.150@tcp1 routed destination 00000400:00000200:19.0:1616513098.045013:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.150@tcp1(10.31.140.150@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.045020:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:19.0:1616513098.045026:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.045029:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:19.0:1616513098.045031:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:19.0:1616513098.045036:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.045041:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:19.0:1616513098.045044:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513098.045095:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.045102:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.045106:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.045112:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.045119:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707870 00000800:00000200:16.0:1616513098.045122:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.045125:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.048770:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.048826:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.048835:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.048846:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.25@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.048856:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.25@tcp1 of length 224 into portal 26 MB=0x6039434f5ae40 00000400:00000200:26.0:1616513098.048862:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.25@tcp1 of length 224/224 into md 0xf97d03d9 [1] + 0 00000400:00000200:26.0:1616513098.048867:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02beb28 00000400:00000200:26.0:1616513098.048870:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.048873:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513098.048878:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beb28 00000800:00000200:26.0:1616513098.048881:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.048883:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.048883:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513098.048890:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.048893:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.048894:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.048902:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.25@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.048911:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.25@tcp1 of length 224 into portal 12 MB=0x6039434f5b040 00000400:00000200:14.0:1616513098.048918:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.25@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 39088 00000400:00000200:14.0:1616513098.048923:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.048925:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513098.048933:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.048936:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.048938:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.048943:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692784938823232 00010000:00000200:19.0:1616513098.048954:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1692784938823232/t0(0) o400->28769519-1081-1760-2006-c3efe73e868b@10.31.140.25@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.048968:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692784938823232, offset 192 00000400:00000200:19.0:1616513098.048974:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.25@tcp1 00000400:00000200:19.0:1616513098.048983:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.25@tcp1 routed destination 00000100:00000200:6.0:1616513098.048986:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692784938823744 00010000:00000200:6.0:1616513098.048996:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebba80 x1692784938823744/t0(0) o400->6e0e1f77-24d2-9df6-791c-d0a6fbb1d1d6@10.31.140.25@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:19.0:1616513098.048998:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.25@tcp1(10.31.140.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.049005:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:6.0:1616513098.049006:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692784938823744, offset 192 00000400:00000200:6.0:1616513098.049011:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.25@tcp1 00000800:00000200:19.0:1616513098.049012:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.049015:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.049017:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:6.0:1616513098.049017:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.25@tcp1 routed destination 00000800:00000200:19.0:1616513098.049022:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.049026:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:6.0:1616513098.049028:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.25@tcp1(10.31.140.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:19.0:1616513098.049029:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:6.0:1616513098.049032:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:6.0:1616513098.049036:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513098.049038:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:6.0:1616513098.049040:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513098.049043:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.049049:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513098.049080:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.049088:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.049091:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.049091:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.049097:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.049104:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17077e8 00000800:00000200:0.0:1616513098.049106:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513098.049108:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513098.049113:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.049116:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.049122:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.049127:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00ee0 00000800:00000200:0.0:1616513098.049131:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.049134:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.049846:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513098.049857:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.049867:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513098.049868:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513098.049876:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.4@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.049885:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.4@tcp1 of length 224 into portal 12 MB=0x5ffd4f08b7d40 00000400:00000200:18.0:1616513098.049892:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.4@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 39312 00000400:00000200:18.0:1616513098.049897:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.049900:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513098.049907:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.049909:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.049911:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.049913:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:6.0:1616513098.049917:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664917376320 00010000:00000200:6.0:1616513098.049924:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464625100 x1688664917376320/t0(0) o400->fe012dbb-9ae1-1147-dca0-91fc8917097a@10.31.136.4@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513098.049947:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664917376320, offset 192 00000400:00000200:6.0:1616513098.049951:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.4@tcp1 00000400:00000200:6.0:1616513098.049955:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.4@tcp1 routed destination 00000400:00000200:6.0:1616513098.049964:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.4@tcp1(10.31.136.4@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513098.049968:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:6.0:1616513098.049971:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513098.049973:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513098.049974:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513098.049976:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.049981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513098.049985:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513098.049990:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513098.049993:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513098.049994:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.049996:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513098.049998:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.050001:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.050002:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.4@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.050004:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00990 00000800:00000200:26.0:1616513098.050007:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513098.050010:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.4@tcp1 of length 224 into portal 26 MB=0x5ffd4f08b7cc0 00000400:00000200:16.0:1616513098.050016:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.4@tcp1 of length 224/224 into md 0xf97d03e9 [1] + 0 00000400:00000200:16.0:1616513098.050021:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be4c8 00000400:00000200:16.0:1616513098.050024:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.050026:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513098.050031:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be4c8 00000800:00000200:16.0:1616513098.050034:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.050037:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.050038:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.050042:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513098.050045:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664917376192 00010000:00000200:19.0:1616513098.050050:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688664917376192/t0(0) o400->d14260b3-9a9d-390a-550e-841d643fbf0c@10.31.136.4@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.050061:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664917376192, offset 192 00000400:00000200:19.0:1616513098.050066:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.4@tcp1 00000400:00000200:19.0:1616513098.050072:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.4@tcp1 routed destination 00000400:00000200:19.0:1616513098.050086:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.4@tcp1(10.31.136.4@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.050092:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:19.0:1616513098.050097:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.050099:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:19.0:1616513098.050101:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:19.0:1616513098.050104:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:19.0:1616513098.050109:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513098.050109:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.050119:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.050123:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.050125:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.050128:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.050131:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.050136:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706d48 00000800:00000200:0.0:1616513098.050139:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.068037:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.068092:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.068102:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.068111:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.237@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.068121:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.237@tcp1 of length 224 into portal 12 MB=0x5ffdc98d3be40 00000400:00000200:24.0:1616513098.068128:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.237@tcp1 of length 224/224 into md 0x865be291 [1] + 70640 00000400:00000200:24.0:1616513098.068133:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.068136:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513098.068143:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.068146:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.068147:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.068150:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.068212:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697805454912 00010000:00000200:28.0:1616513098.068224:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceba80 x1688697805454912/t0(0) o400->048d909d-86a9-5e85-c0d2-ea0be21bd179@10.31.141.237@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.068238:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697805454912, offset 192 00000800:00000200:7.2:1616513098.068238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513098.068245:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.237@tcp1 00000400:00000200:28.0:1616513098.068253:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.237@tcp1 routed destination 00000400:00000200:28.0:1616513098.068268:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.237@tcp1(10.31.141.237@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.068274:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513098.068280:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.068283:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513098.068286:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513098.068291:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.068293:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.068303:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.068312:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.237@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.068321:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.237@tcp1 of length 224 into portal 26 MB=0x5ffdc98d3bdc0 00000400:00000200:14.0:1616513098.068328:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.237@tcp1 of length 224/224 into md 0xf97d0409 [1] + 0 00000400:00000200:14.0:1616513098.068333:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02becc0 00000400:00000200:14.0:1616513098.068336:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.068339:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.068344:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02becc0 00000800:00000200:14.0:1616513098.068347:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513098.068348:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513098.068352:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513098.068357:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513098.068358:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:18.0:1616513098.068360:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.068360:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.068367:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.068371:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd100 00000800:00000200:18.0:1616513098.068375:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.068410:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697805454784 00010000:00000200:19.0:1616513098.068420:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688697805454784/t0(0) o400->25edbe19-0591-3144-d3ed-7c6ee43ff0c7@10.31.141.237@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.068446:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697805454784, offset 192 00000400:00000200:19.0:1616513098.068452:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.237@tcp1 00000400:00000200:19.0:1616513098.068460:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.237@tcp1 routed destination 00000400:00000200:19.0:1616513098.068475:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.237@tcp1(10.31.141.237@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.068482:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:19.0:1616513098.068487:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.068490:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:19.0:1616513098.068492:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:19.0:1616513098.068497:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.068502:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:19.0:1616513098.068503:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513098.068557:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.068566:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.068569:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.068575:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513098.068576:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.068583:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707ba0 00000800:00000200:20.0:1616513098.068587:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.073479:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.073534:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.073545:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.073555:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.192@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.073565:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.192@tcp1 of length 224 into portal 26 MB=0x605893a269180 00000400:00000200:16.0:1616513098.073572:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.192@tcp1 of length 224/224 into md 0xf97d0419 [1] + 0 00000400:00000200:16.0:1616513098.073577:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf320 00000400:00000200:16.0:1616513098.073579:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.073582:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513098.073587:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf320 00000800:00000200:16.0:1616513098.073590:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.073593:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.073594:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513098.073601:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.073604:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.073605:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.073613:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.192@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.073623:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.192@tcp1 of length 224 into portal 12 MB=0x605893a269440 00000400:00000200:24.0:1616513098.073629:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.192@tcp1 of length 224/224 into md 0x865be291 [1] + 70864 00000400:00000200:24.0:1616513098.073634:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.073637:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513098.073643:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.073646:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.073648:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513098.073652:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694936804528512 00010000:00000200:19.0:1616513098.073663:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1694936804528512/t0(0) o400->7c3405cb-782e-23ae-8072-26e103827ba2@10.31.140.192@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.073678:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694936804528512, offset 192 00000400:00000200:19.0:1616513098.073684:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.192@tcp1 00000400:00000200:19.0:1616513098.073692:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.192@tcp1 routed destination 00000100:00000200:28.0:1616513098.073707:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694936804529216 00000400:00000200:19.0:1616513098.073707:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.192@tcp1(10.31.140.192@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.073714:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:28.0:1616513098.073719:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cee780 x1694936804529216/t0(0) o400->c0c3bece-bdd2-df39-b86b-e8d88e1d6d9f@10.31.140.192@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:19.0:1616513098.073720:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.073725:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.073727:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:19.0:1616513098.073732:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:28.0:1616513098.073734:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694936804529216, offset 192 00000100:00000200:19.0:1616513098.073739:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513098.073740:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.192@tcp1 00000800:00000200:7.2:1616513098.073740:0:51385:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:28.0:1616513098.073748:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.192@tcp1 routed destination 00000400:00000200:28.0:1616513098.073763:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.192@tcp1(10.31.140.192@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.073770:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.073776:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.073781:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.073783:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.073788:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.073793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513098.073795:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.073802:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.073805:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.073811:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.073817:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706330 00000800:00000200:26.0:1616513098.073820:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.073823:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513098.073846:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.073853:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.073856:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513098.073857:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513098.073863:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.073866:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd320 00000800:00000200:20.0:1616513098.073870:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:3.0:1616513098.074539:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929520, portal 4 00000100:00000200:3.0:1616513098.074545:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929520, offset 0 00000400:00000200:3.0:1616513098.074551:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:3.0:1616513098.074557:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:3.0:1616513098.074570:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513098.074576:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513098.074582:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513098.074585:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:3.0:1616513098.074587:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:3.0:1616513098.074593:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513098.074639:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513098.074656:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:0.0:1616513098.074663:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:0.0:1616513098.074666:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.074668:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:0.0:1616513098.074673:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:0.0:1616513098.074680:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bf9b00 x1693067725929520/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513098.074693:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd3188 00000800:00000200:0.0:1616513098.074699:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513098.074777:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513098.074787:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513098.074792:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513098.074795:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:16.0:1616513098.074799:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513098.074805:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627c30 00000400:00000200:16.0:1616513098.074812:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dd101 [1] + 192 00000400:00000200:16.0:1616513098.074816:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.074821:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bf9b00 x1693067725929520/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513098.074835:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513098.074837:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513098.074839:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:3.0:1616513098.074846:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bf9b00 x1693067725929520/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513098.074858:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bf9b00 x1693067725929520/t0(0) o13->boslfs02-OST003d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513098.074878:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2a18 00000800:00000200:11.2:1616513098.081017:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.081072:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.081082:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.081091:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.240@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.081100:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.240@tcp1 of length 224 into portal 26 MB=0x60182ef6afd40 00000400:00000200:24.0:1616513098.081107:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.240@tcp1 of length 224/224 into md 0xf97d0439 [1] + 0 00000400:00000200:24.0:1616513098.081112:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be220 00000400:00000200:24.0:1616513098.081114:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.081117:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513098.081122:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be220 00000800:00000200:24.0:1616513098.081125:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.081126:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.081128:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.081130:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.081135:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.081188:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:19.0:1616513098.081189:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511734406464 00000800:00000200:20.0:1616513098.081197:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00010000:00000200:19.0:1616513098.081200:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1690511734406464/t0(0) o400->a640c3e6-3b87-28d0-2969-774830db1978@10.31.130.240@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:20.0:1616513098.081213:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.240@tcp1 : PUT - for me 00000100:00000200:19.0:1616513098.081214:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511734406464, offset 192 00000400:00000200:19.0:1616513098.081221:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.240@tcp1 00000400:00000200:20.0:1616513098.081222:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.240@tcp1 of length 224 into portal 12 MB=0x60182ef6afe00 00000400:00000200:20.0:1616513098.081228:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.240@tcp1 of length 224/224 into md 0x865be291 [1] + 71088 00000400:00000200:19.0:1616513098.081228:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.240@tcp1 routed destination 00000400:00000200:20.0:1616513098.081233:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.081236:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513098.081243:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:19.0:1616513098.081244:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.240@tcp1(10.31.130.240@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513098.081246:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.081248:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:19.0:1616513098.081250:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513098.081252:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:19.0:1616513098.081257:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.081260:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.081262:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:19.0:1616513098.081267:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:19.0:1616513098.081273:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.081274:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513098.081307:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511734406656 00010000:00000200:28.0:1616513098.081320:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c664c80 x1690511734406656/t0(0) o400->4ec1ed91-0c49-00e2-348b-ced7b4b77796@10.31.130.240@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513098.081331:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:28.0:1616513098.081334:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511734406656, offset 192 00000800:00000200:18.0:1616513098.081339:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513098.081340:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.240@tcp1 00000400:00000200:18.0:1616513098.081342:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513098.081345:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513098.081348:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.240@tcp1 routed destination 00000400:00000200:18.0:1616513098.081349:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.081355:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706770 00000800:00000200:18.0:1616513098.081359:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513098.081363:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.240@tcp1(10.31.130.240@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.081370:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.081376:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.081378:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.081380:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.081385:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.081390:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.081401:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.081406:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.081408:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.081410:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.081415:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.081418:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd430 00000800:00000200:0.0:1616513098.081422:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.092293:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.092348:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.092358:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.092367:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.236@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.092376:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.236@tcp1 of length 224 into portal 26 MB=0x5ffe45c45fb00 00000400:00000200:16.0:1616513098.092383:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.236@tcp1 of length 224/224 into md 0xf97d0459 [1] + 0 00000400:00000200:16.0:1616513098.092388:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf4b8 00000400:00000200:16.0:1616513098.092392:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.092395:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513098.092400:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf4b8 00000800:00000200:20.0:1616513098.092401:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.092403:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.092406:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.092408:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513098.092424:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731149269760 00010000:00000200:19.0:1616513098.092446:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1688731149269760/t0(0) o400->c5988460-4f9f-0980-2def-5a1c5a6d009c@10.31.141.236@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.092461:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688731149269760, offset 192 00000400:00000200:19.0:1616513098.092467:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.236@tcp1 00000400:00000200:19.0:1616513098.092475:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.236@tcp1 routed destination 00000400:00000200:19.0:1616513098.092490:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.236@tcp1(10.31.141.236@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.092497:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:19.0:1616513098.092503:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.092506:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.092508:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:19.0:1616513098.092513:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.092518:0:51387:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:19.0:1616513098.092520:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513098.092574:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.092582:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.092585:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.092592:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.092598:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707d38 00000800:00000200:26.0:1616513098.092602:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.092604:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.092610:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513098.092613:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.092617:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.092619:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.092621:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513098.092716:0:51387:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513098.092770:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.092780:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513098.092781:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513098.092788:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.236@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.092797:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.236@tcp1 of length 224 into portal 12 MB=0x5ffe45c45fdc0 00000400:00000200:18.0:1616513098.092803:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.236@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 39536 00000400:00000200:18.0:1616513098.092808:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.092811:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513098.092819:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.092822:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.092824:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:6.0:1616513098.092833:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731149270464 00010000:00000200:6.0:1616513098.092841:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464627080 x1688731149270464/t0(0) o400->e4ca5721-c012-299b-e1d8-5e9f7145ed0d@10.31.141.236@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513098.092851:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688731149270464, offset 192 00000400:00000200:6.0:1616513098.092856:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.236@tcp1 00000400:00000200:6.0:1616513098.092861:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.236@tcp1 routed destination 00000400:00000200:6.0:1616513098.092872:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.236@tcp1(10.31.141.236@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513098.092877:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:6.0:1616513098.092881:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513098.092883:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:6.0:1616513098.092885:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:6.0:1616513098.092888:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.092893:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.092945:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.092952:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.092954:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.092960:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513098.092960:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513098.092964:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf01320 00000800:00000200:24.0:1616513098.092968:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.103041:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.103116:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.103125:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513098.103134:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.51@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.103143:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.51@tcp1 of length 224 into portal 26 MB=0x5ffe5c952b2c0 00000400:00000200:20.0:1616513098.103150:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.51@tcp1 of length 224/224 into md 0xf97d0469 [1] + 0 00000400:00000200:20.0:1616513098.103155:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfe48 00000400:00000200:20.0:1616513098.103158:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.103161:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513098.103166:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfe48 00000800:00000200:20.0:1616513098.103169:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.103169:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.103171:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.103176:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.103179:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.103180:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.103188:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.51@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.103197:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.51@tcp1 of length 224 into portal 12 MB=0x5ffe5c952b640 00000400:00000200:16.0:1616513098.103204:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.51@tcp1 of length 224/224 into md 0x865be291 [1] + 71312 00000400:00000200:16.0:1616513098.103209:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.103211:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513098.103218:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.103221:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.103223:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513098.103232:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737273787072 00000100:00000200:28.0:1616513098.103242:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737273787968 00010000:00000200:19.0:1616513098.103243:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688737273787072/t0(0) o400->991c7e52-e690-3788-9920-25adedbcd316@10.31.140.51@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:28.0:1616513098.103254:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d6300 x1688737273787968/t0(0) o400->a5a9e925-9fb9-771c-eb37-c8e71d2bb425@10.31.140.51@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:19.0:1616513098.103258:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688737273787072, offset 192 00000400:00000200:19.0:1616513098.103264:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.51@tcp1 00000100:00000200:28.0:1616513098.103268:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688737273787968, offset 192 00000400:00000200:19.0:1616513098.103272:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.51@tcp1 routed destination 00000400:00000200:28.0:1616513098.103274:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.51@tcp1 00000400:00000200:28.0:1616513098.103282:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.51@tcp1 routed destination 00000400:00000200:19.0:1616513098.103289:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.51@tcp1(10.31.140.51@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.103296:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:28.0:1616513098.103297:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.51@tcp1(10.31.140.51@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.103303:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:19.0:1616513098.103303:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.103306:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.103308:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513098.103309:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.103311:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:19.0:1616513098.103313:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513098.103314:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.103318:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513098.103319:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:19.0:1616513098.103320:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513098.103323:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513098.103374:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.103382:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.103385:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.103392:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513098.103392:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.103398:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17066e8 00000800:00000200:24.0:1616513098.103399:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:14.0:1616513098.103401:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513098.103402:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.103403:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513098.103404:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513098.103411:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.103415:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd540 00000800:00000200:24.0:1616513098.103419:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.110890:0:51389:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.110947:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.110956:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.110965:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.23@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.110974:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.23@tcp1 of length 224 into portal 26 MB=0x5ffd442eb4c00 00000400:00000200:26.0:1616513098.110981:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.23@tcp1 of length 224/224 into md 0xf97d0489 [1] + 0 00000400:00000200:26.0:1616513098.110986:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf430 00000400:00000200:26.0:1616513098.110988:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.110991:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513098.110996:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf430 00000800:00000200:26.0:1616513098.110999:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.111002:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513098.111003:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.111008:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.111059:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662004419584 00000800:00000200:11.2:1616513098.111067:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:19.0:1616513098.111070:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688662004419584/t0(0) o400->dd913108-0170-7bd3-e0bf-a9928de730e3@10.31.133.23@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.111083:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662004419584, offset 192 00000800:00000200:0.0:1616513098.111083:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:19.0:1616513098.111089:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.23@tcp1 00000800:00000200:0.0:1616513098.111092:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:19.0:1616513098.111096:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.23@tcp1 routed destination 00000400:00000200:0.0:1616513098.111102:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.23@tcp1 : PUT - for me 00000400:00000200:19.0:1616513098.111111:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.23@tcp1(10.31.133.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513098.111112:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.23@tcp1 of length 224 into portal 12 MB=0x5ffd442eb4c80 00000800:00000200:19.0:1616513098.111118:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513098.111119:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.23@tcp1 of length 224/224 into md 0x865be291 [1] + 71536 00000800:00000200:19.0:1616513098.111124:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:0.0:1616513098.111124:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:19.0:1616513098.111127:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:0.0:1616513098.111127:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:19.0:1616513098.111129:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:19.0:1616513098.111134:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513098.111134:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.111136:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.111137:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:7.2:1616513098.111138:0:51389:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:19.0:1616513098.111140:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513098.111140:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513098.111204:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.111212:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.111215:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:28.0:1616513098.111220:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662004419712 00000400:00000200:14.0:1616513098.111222:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.111228:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706660 00000800:00000200:14.0:1616513098.111231:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:28.0:1616513098.111232:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b98480 x1688662004419712/t0(0) o400->4dd9624c-ffb4-2fc3-b641-f719115000c2@10.31.133.23@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513098.111234:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513098.111257:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662004419712, offset 192 00000400:00000200:28.0:1616513098.111263:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.23@tcp1 00000400:00000200:28.0:1616513098.111271:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.23@tcp1 routed destination 00000400:00000200:28.0:1616513098.111286:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.23@tcp1(10.31.133.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.111293:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.111299:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.111302:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.111304:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.111309:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.111313:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.111366:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.111373:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.111376:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.111382:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.111386:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd650 00000800:00000200:20.0:1616513098.111390:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.111393:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.111503:0:51389:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513098.111516:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.111524:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513098.111526:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.111532:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.111533:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.197@tcp1 : PUT - for me 00000800:00000200:14.0:1616513098.111535:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.111539:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.197@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.111543:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.197@tcp1 of length 224 into portal 26 MB=0x5ffe15947da00 00000400:00000200:26.0:1616513098.111546:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.197@tcp1 of length 224 into portal 12 MB=0x5ffe15947dd80 00000400:00000200:18.0:1616513098.111550:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.197@tcp1 of length 224/224 into md 0xf97d04a9 [1] + 0 00000400:00000200:18.0:1616513098.111554:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bebb0 00000400:00000200:18.0:1616513098.111557:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.111559:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.197@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 39760 00000400:00000200:26.0:1616513098.111563:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.111563:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513098.111569:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bebb0 00000100:00000200:26.0:1616513098.111571:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513098.111572:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.111575:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.111577:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513098.111578:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.111579:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.111581:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.111584:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688718214158848 00010000:00000200:19.0:1616513098.111589:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688718214158848/t0(0) o400->d96470c3-0a35-7286-5727-1aa41147a4d0@10.31.142.197@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:6.0:1616513098.111592:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688718214159744 00010000:00000200:6.0:1616513098.111600:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c697500 x1688718214159744/t0(0) o400->374c80d0-215a-fc04-8cfd-fc0a0630ce33@10.31.142.197@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:19.0:1616513098.111601:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688718214158848, offset 192 00000400:00000200:19.0:1616513098.111608:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.197@tcp1 00000100:00000200:6.0:1616513098.111611:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688718214159744, offset 192 00000400:00000200:19.0:1616513098.111615:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.197@tcp1 routed destination 00000400:00000200:6.0:1616513098.111616:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.197@tcp1 00000400:00000200:6.0:1616513098.111621:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.197@tcp1 routed destination 00000400:00000200:6.0:1616513098.111635:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.197@tcp1(10.31.142.197@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:19.0:1616513098.111637:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.197@tcp1(10.31.142.197@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:6.0:1616513098.111641:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:19.0:1616513098.111643:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:6.0:1616513098.111646:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.111648:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513098.111648:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:19.0:1616513098.111650:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513098.111650:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:19.0:1616513098.111652:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513098.111653:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:19.0:1616513098.111655:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513098.111658:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:19.0:1616513098.111660:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.111660:0:51389:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513098.111669:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513098.111672:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513098.111675:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.111677:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513098.111680:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513098.111680:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513098.111683:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.111683:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.111684:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.111688:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf017e8 00000400:00000200:2.0:1616513098.111690:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:16.0:1616513098.111692:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513098.111698:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706ee0 00000800:00000200:2.0:1616513098.111702:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.112407:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.112417:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.112425:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513098.112429:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513098.112445:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.22@tcp1 : PUT - for me 00000800:00000200:7.2:1616513098.112445:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:20.0:1616513098.112454:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.22@tcp1 of length 224 into portal 12 MB=0x60182bf7bcac0 00000800:00000200:18.0:1616513098.112455:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513098.112461:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.22@tcp1 of length 224/224 into md 0x865be291 [1] + 71760 00000800:00000200:18.0:1616513098.112461:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513098.112464:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513098.112466:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.112469:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:18.0:1616513098.112469:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.22@tcp1 : PUT - for me 00000800:00000200:20.0:1616513098.112475:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513098.112475:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.22@tcp1 of length 224 into portal 26 MB=0x60182bf7bca40 00000800:00000200:20.0:1616513098.112478:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.112479:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513098.112481:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.22@tcp1 of length 224/224 into md 0xf97d04c9 [1] + 0 00000400:00000200:18.0:1616513098.112485:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfdc0 00000400:00000200:18.0:1616513098.112487:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513098.112489:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510930201280 00000100:00000200:18.0:1616513098.112489:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513098.112493:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfdc0 00000800:00000200:18.0:1616513098.112495:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00010000:00000200:28.0:1616513098.112496:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c665580 x1690510930201280/t0(0) o400->5f373f66-755f-177d-50d8-5321a152b520@10.31.133.22@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513098.112498:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.112499:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.112507:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510930201152 00000100:00000200:28.0:1616513098.112508:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510930201280, offset 192 00010000:00000200:19.0:1616513098.112512:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1690510930201152/t0(0) o400->64769791-d8e0-9f22-0f19-61228336bba1@10.31.133.22@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513098.112513:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.22@tcp1 00000400:00000200:28.0:1616513098.112519:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.22@tcp1 routed destination 00000100:00000200:19.0:1616513098.112535:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510930201152, offset 192 00000400:00000200:28.0:1616513098.112536:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.22@tcp1(10.31.133.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:19.0:1616513098.112541:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.22@tcp1 00000800:00000200:28.0:1616513098.112542:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:19.0:1616513098.112546:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.22@tcp1 routed destination 00000800:00000200:28.0:1616513098.112548:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.112550:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513098.112552:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513098.112555:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:19.0:1616513098.112559:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.22@tcp1(10.31.133.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513098.112560:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:19.0:1616513098.112564:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513098.112568:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:19.0:1616513098.112569:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513098.112571:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:19.0:1616513098.112572:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:19.0:1616513098.112573:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513098.112573:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:19.0:1616513098.112576:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513098.112576:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513098.112579:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:19.0:1616513098.112581:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:14.0:1616513098.112582:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd760 00000800:00000200:11.2:1616513098.112582:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513098.112584:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513098.112591:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.112596:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.112599:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.112599:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.112604:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.112609:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706088 00000800:00000200:0.0:1616513098.112613:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:21.2:1616513098.114656:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4ea00] (20)++ 00000800:00000200:31.0:1616513098.114712:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (21)++ 00000800:00000200:31.0:1616513098.114724:0:6475:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.205@o2ib1 00000800:00000200:7.0:1616513098.114724:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (22)-- 00000400:00000200:31.0:1616513098.114733:0:6475:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.205@o2ib1 : PUT - for me 00000400:00000200:31.0:1616513098.114741:0:6475:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.205@o2ib1 of length 224 into portal 26 MB=0x60386f0ea5b00 00000800:00000200:21.2:1616513098.114743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4ea00] (21)++ 00000400:00000200:31.0:1616513098.114748:0:6475:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.205@o2ib1 of length 224/224 into md 0xf97d04f9 [1] + 0 00000400:00000200:31.0:1616513098.114753:0:6475:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf078 00000400:00000200:31.0:1616513098.114756:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:11.0:1616513098.114757:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (22)++ 00000100:00000200:31.0:1616513098.114759:0:6475:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:11.0:1616513098.114765:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.205@o2ib1 00000400:00000200:31.0:1616513098.114766:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf078 00000800:00000200:31.0:1616513098.114770:0:6475:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4ea00] (23)++ 00000800:00000200:31.0:1616513098.114773:0:6475:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4ea00] (24)-- 00000400:00000200:11.0:1616513098.114773:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.205@o2ib1 : PUT - for me 00000800:00000200:31.0:1616513098.114775:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (23)-- 00000800:00000200:31.0:1616513098.114777:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (22)-- 00000400:00000200:11.0:1616513098.114780:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.205@o2ib1 of length 224 into portal 12 MB=0x60386f0ea5cc0 00000400:00000200:11.0:1616513098.114786:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.205@o2ib1 of length 224/224 into md 0x3657608d [1] + 95872 00000400:00000200:11.0:1616513098.114791:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:11.0:1616513098.114793:0:6473:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:11.0:1616513098.114801:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4ea00] (21)++ 00000800:00000200:11.0:1616513098.114803:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4ea00] (22)-- 00000800:00000200:11.0:1616513098.114805:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (21)-- 00000100:00000200:7.0:1616513098.114816:0:252828:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692727962655936 00010000:00000200:7.0:1616513098.114825:0:252828:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdd63d100 x1692727962655936/t0(0) o400->db89f005-a603-f7cc-3df4-30f0aead2f2c@10.242.46.205@o2ib1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:7.0:1616513098.114835:0:252828:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692727962655936, offset 192 00000400:00000200:7.0:1616513098.114839:0:252828:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.205@o2ib1 00000100:00000200:19.0:1616513098.114844:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692727962655488 00000400:00000200:7.0:1616513098.114844:0:252828:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.205@o2ib1 local destination 00000400:00000200:7.0:1616513098.114852:0:252828:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.205@o2ib1(10.242.46.205@o2ib1:10.242.46.205@o2ib1) : PUT try# 0 00010000:00000200:19.0:1616513098.114855:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1692727962655488/t0(0) o400->3759e23b-3443-c286-154b-e60843c8912c@10.242.46.205@o2ib1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.0:1616513098.114856:0:252828:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.205@o2ib1 00000800:00000200:7.0:1616513098.114860:0:252828:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91ec180] -> 10.242.46.205@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513098.114862:0:252828:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4ea00] (20)++ 00000800:00000200:7.0:1616513098.114863:0:252828:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4ea00] (21)++ 00000800:00000200:7.0:1616513098.114867:0:252828:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4ea00] (22)-- 00000100:00000200:19.0:1616513098.114869:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692727962655488, offset 192 00000800:00000200:21.2:1616513098.114871:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4ea00] (21)++ 00000400:00000200:19.0:1616513098.114875:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.205@o2ib1 00000800:00000200:7.0:1616513098.114880:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (22)++ 00000400:00000200:19.0:1616513098.114882:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.205@o2ib1 local destination 00000800:00000200:7.0:1616513098.114882:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4ea00] (23)-- 00000400:00000200:7.0:1616513098.114884:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513098.114887:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (22)-- 00000400:00000200:7.0:1616513098.114887:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.205@o2ib1: PUT: OK 00000400:00000200:7.0:1616513098.114890:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637ef68 00000400:00000200:19.0:1616513098.114894:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.205@o2ib1(10.242.46.205@o2ib1:10.242.46.205@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513098.114902:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (21)-- 00000800:00000200:19.0:1616513098.114904:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.205@o2ib1 00000800:00000200:19.0:1616513098.114909:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91ec180] -> 10.242.46.205@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.114912:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4ea00] (20)++ 00000800:00000200:19.0:1616513098.114914:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4ea00] (21)++ 00000800:00000200:19.0:1616513098.114918:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4ea00] (22)-- 00000800:00000200:21.2:1616513098.114922:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4ea00] (21)++ 00000100:00000200:19.0:1616513098.114924:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.0:1616513098.114929:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (22)++ 00000800:00000200:11.0:1616513098.114933:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4ea00] (23)-- 00000400:00000200:11.0:1616513098.114934:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:3.0:1616513098.114938:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (22)-- 00000400:00000200:11.0:1616513098.114939:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.205@o2ib1: PUT: OK 00000400:00000200:11.0:1616513098.114943:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706088 00000800:00000200:11.0:1616513098.114946:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4ea00] (21)-- 00000800:00000200:11.2:1616513098.118665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.118677:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.118685:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.118694:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.32@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.118704:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.32@tcp1 of length 224 into portal 12 MB=0x600923c77b240 00000400:00000200:24.0:1616513098.118710:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.32@tcp1 of length 224/224 into md 0x865be291 [1] + 71984 00000400:00000200:24.0:1616513098.118715:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.118718:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513098.118725:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.118728:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.118728:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.118733:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.118764:0:51390:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.118779:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.118787:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.118797:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.32@tcp1 : PUT - for me 00000100:00000200:28.0:1616513098.118800:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689477939966528 00000400:00000200:26.0:1616513098.118806:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.32@tcp1 of length 224 into portal 26 MB=0x600923c77b040 00010000:00000200:28.0:1616513098.118812:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ced580 x1689477939966528/t0(0) o400->4411ca1e-fca6-ba01-04af-073c8b4ca166@10.31.142.32@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:26.0:1616513098.118812:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.32@tcp1 of length 224/224 into md 0xf97d0509 [1] + 0 00000400:00000200:26.0:1616513098.118819:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf8f8 00000400:00000200:26.0:1616513098.118822:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.118825:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000100:00000200:28.0:1616513098.118826:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689477939966528, offset 192 00000400:00000200:26.0:1616513098.118830:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf8f8 00000800:00000200:18.0:1616513098.118831:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513098.118832:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.32@tcp1 00000800:00000200:26.0:1616513098.118833:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.118836:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.118838:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513098.118839:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.32@tcp1 routed destination 00000400:00000200:28.0:1616513098.118852:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.32@tcp1(10.31.142.32@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.118861:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513098.118866:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.118869:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513098.118871:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513098.118876:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.118880:0:51390:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:19.0:1616513098.118894:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689477939966016 00010000:00000200:19.0:1616513098.118905:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1689477939966016/t0(0) o400->f4dd15f9-aa79-3152-1de0-9c026f8523c1@10.31.142.32@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.118918:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689477939966016, offset 192 00000400:00000200:19.0:1616513098.118924:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.32@tcp1 00000400:00000200:19.0:1616513098.118932:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.32@tcp1 routed destination 00000800:00000200:2.0:1616513098.118933:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.118940:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.118943:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:19.0:1616513098.118946:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.32@tcp1(10.31.142.32@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513098.118950:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:19.0:1616513098.118953:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513098.118954:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd870 00000800:00000200:19.0:1616513098.118958:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513098.118958:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:19.0:1616513098.118961:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513098.118961:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:19.0:1616513098.118963:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:19.0:1616513098.118968:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.118972:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:19.0:1616513098.118974:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513098.119025:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.119033:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.119036:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.119043:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.119049:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706660 00000800:00000200:16.0:1616513098.119053:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.119056:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.138515:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929536, portal 4 00000100:00000200:28.0:1616513098.138520:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929536, offset 0 00000400:00000200:28.0:1616513098.138526:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:28.0:1616513098.138531:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:28.0:1616513098.138543:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.138550:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513098.138555:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.138558:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513098.138560:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513098.138565:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513098.138611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513098.138669:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513098.138682:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:24.0:1616513098.138683:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513098.138685:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.138691:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:5.2:1616513098.138693:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000100:00000200:20.0:1616513098.138697:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4557ceb600 x1693067725929536/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513098.138705:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:20.0:1616513098.138709:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcda90 00000800:00000200:20.0:1616513098.138713:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:16.0:1616513098.138715:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513098.138716:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:16.0:1616513098.138724:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513098.138733:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627c40 00000400:00000200:16.0:1616513098.138740:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dd161 [1] + 192 00000400:00000200:16.0:1616513098.138745:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.138750:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4557ceb600 x1693067725929536/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513098.138765:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513098.138768:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513098.138770:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:28.0:1616513098.138771:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4557ceb600 x1693067725929536/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513098.138782:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4557ceb600 x1693067725929536/t0(0) o13->boslfs02-OST001d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513098.138791:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd980 00000800:00000200:11.2:1616513098.139587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.139599:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.139604:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.139607:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513098.139611:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.216@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.139618:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.216@tcp1 of length 224 into portal 26 MB=0x5ffe892a68cc0 00000400:00000200:24.0:1616513098.139624:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.216@tcp1 of length 224/224 into md 0xf97d0519 [1] + 0 00000400:00000200:24.0:1616513098.139628:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bed48 00000400:00000200:24.0:1616513098.139630:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.139632:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513098.139636:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bed48 00000800:00000200:24.0:1616513098.139638:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.139640:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.139641:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513098.139726:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688749241437376 00010000:00000200:19.0:1616513098.139736:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688749241437376/t0(0) o400->92b3cf87-0b7e-d06e-9ab7-b4a9de9b743f@10.31.136.216@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.139750:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688749241437376, offset 192 00000400:00000200:19.0:1616513098.139756:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.216@tcp1 00000400:00000200:19.0:1616513098.139764:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.216@tcp1 routed destination 00000400:00000200:19.0:1616513098.139779:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.216@tcp1(10.31.136.216@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.139785:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:19.0:1616513098.139791:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.139794:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.139798:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:19.0:1616513098.139803:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:19.0:1616513098.139809:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.139809:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.139863:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.139870:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.139873:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513098.139879:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.139885:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707d38 00000800:00000200:18.0:1616513098.139888:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.139891:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.140599:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.140654:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.140664:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.140672:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.216@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.140682:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.216@tcp1 of length 224 into portal 12 MB=0x5ffe892a68e00 00000400:00000200:0.0:1616513098.140688:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.216@tcp1 of length 224/224 into md 0x865be291 [1] + 72208 00000400:00000200:0.0:1616513098.140693:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.140696:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513098.140704:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.140708:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.140708:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.140712:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.140764:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688749241437696 00010000:00000200:28.0:1616513098.140774:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9f080 x1688749241437696/t0(0) o400->6c1db612-839e-18ab-bd61-253793d28576@10.31.136.216@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.140787:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688749241437696, offset 192 00000400:00000200:28.0:1616513098.140792:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.216@tcp1 00000400:00000200:28.0:1616513098.140800:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.216@tcp1 routed destination 00000400:00000200:28.0:1616513098.140813:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.216@tcp1(10.31.136.216@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.140819:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.140824:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.140828:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.140830:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.140834:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.140839:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.140891:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.140898:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.140901:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.140907:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.140911:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd980 00000800:00000200:20.0:1616513098.140915:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.140918:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.173275:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.173333:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.173343:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.173353:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.252@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.173363:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.252@tcp1 of length 224 into portal 26 MB=0x5ffd325651780 00000400:00000200:2.0:1616513098.173370:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.252@tcp1 of length 224/224 into md 0xf97d0529 [1] + 0 00000400:00000200:2.0:1616513098.173375:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be440 00000400:00000200:2.0:1616513098.173378:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.173381:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.173386:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be440 00000800:00000200:14.0:1616513098.173389:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.173389:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.173392:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.173394:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.173461:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657214117760 00010000:00000200:19.0:1616513098.173472:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688657214117760/t0(0) o400->3561c29f-ea49-bb35-9249-7f3ed426203f@10.31.130.252@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.173485:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688657214117760, offset 192 00000400:00000200:19.0:1616513098.173491:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.252@tcp1 00000400:00000200:19.0:1616513098.173498:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.252@tcp1 routed destination 00000400:00000200:19.0:1616513098.173513:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.252@tcp1(10.31.130.252@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.173519:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:19.0:1616513098.173526:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.173528:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.173530:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:19.0:1616513098.173535:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.173539:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:19.0:1616513098.173541:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513098.173554:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.173562:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.173565:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513098.173572:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.173578:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706330 00000800:00000200:18.0:1616513098.173582:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.173585:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.175533:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.175597:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.175607:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.175616:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.252@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.175624:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.252@tcp1 of length 224 into portal 12 MB=0x5ffd325651c40 00000400:00000200:16.0:1616513098.175631:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.252@tcp1 of length 224/224 into md 0x865be291 [1] + 72432 00000400:00000200:16.0:1616513098.175636:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.175639:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513098.175645:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.175648:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.175650:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.175653:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.175712:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688657214118976 00010000:00000200:28.0:1616513098.175726:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d2880 x1688657214118976/t0(0) o400->4bd25548-a7b5-b34d-0493-7055d203c8f5@10.31.130.252@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.175741:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688657214118976, offset 192 00000400:00000200:28.0:1616513098.175748:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.252@tcp1 00000400:00000200:28.0:1616513098.175755:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.252@tcp1 routed destination 00000400:00000200:28.0:1616513098.175770:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.252@tcp1(10.31.130.252@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.175777:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.175783:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.175786:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.175788:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.175793:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.175798:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.175854:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.175861:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.175864:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.175870:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.175874:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdba0 00000800:00000200:20.0:1616513098.175878:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.175881:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.192388:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.192402:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.192411:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513098.192413:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513098.192420:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.38@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.192430:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.38@tcp1 of length 224 into portal 12 MB=0x5ffdc2ac199c0 00000400:00000200:14.0:1616513098.192444:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.38@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 39984 00000400:00000200:14.0:1616513098.192450:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.192452:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513098.192460:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.192465:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.192468:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.192480:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:6.0:1616513098.192483:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695958772160 00010000:00000200:6.0:1616513098.192495:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebde80 x1688695958772160/t0(0) o400->d652bde1-80ac-4dbc-c5f3-ef03ad2af3a8@10.31.142.38@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:6.0:1616513098.192510:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695958772160, offset 192 00000400:00000200:6.0:1616513098.192517:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.38@tcp1 00000400:00000200:6.0:1616513098.192525:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.38@tcp1 routed destination 00000800:00000200:16.0:1616513098.192534:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:6.0:1616513098.192540:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.38@tcp1(10.31.142.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513098.192544:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.192553:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.38@tcp1 : PUT - for me 00000800:00000200:6.0:1616513098.192558:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:16.0:1616513098.192562:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.38@tcp1 of length 224 into portal 26 MB=0x5ffdc2ac198c0 00000800:00000200:6.0:1616513098.192563:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:6.0:1616513098.192565:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:6.0:1616513098.192566:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513098.192569:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.38@tcp1 of length 224/224 into md 0xf97d0549 [1] + 0 00000800:00000200:6.0:1616513098.192570:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513098.192573:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf760 00000800:00000200:7.2:1616513098.192574:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513098.192576:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.192579:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513098.192584:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf760 00000800:00000200:16.0:1616513098.192588:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.192590:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.192592:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.192597:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513098.192606:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695958771904 00010000:00000200:19.0:1616513098.192616:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688695958771904/t0(0) o400->7e557054-f93c-26e2-e150-8c7cac6ef88e@10.31.142.38@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.192628:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695958771904, offset 192 00000800:00000200:18.0:1616513098.192628:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:19.0:1616513098.192633:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.38@tcp1 00000800:00000200:18.0:1616513098.192635:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.192638:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:19.0:1616513098.192641:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.38@tcp1 routed destination 00000400:00000200:18.0:1616513098.192645:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.192649:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8decf00b28 00000400:00000200:19.0:1616513098.192654:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.38@tcp1(10.31.142.38@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513098.192654:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.192659:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:19.0:1616513098.192660:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:19.0:1616513098.192666:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.192668:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:19.0:1616513098.192670:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:19.0:1616513098.192674:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.192679:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:19.0:1616513098.192680:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513098.192733:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.192741:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.192744:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.192751:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.192757:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706d48 00000800:00000200:20.0:1616513098.192761:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.192764:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.203849:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.203907:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.203917:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513098.203918:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.203924:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.203926:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.22@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.203931:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.22@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.203935:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.22@tcp1 of length 224 into portal 26 MB=0x5ffe97a73fdc0 00000400:00000200:14.0:1616513098.203938:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.22@tcp1 of length 224 into portal 12 MB=0x5ffe97a73fe40 00000400:00000200:2.0:1616513098.203942:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.22@tcp1 of length 224/224 into md 0xf97d0559 [1] + 0 00000400:00000200:2.0:1616513098.203946:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf6d8 00000400:00000200:2.0:1616513098.203949:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513098.203951:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.22@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 40208 00000400:00000200:14.0:1616513098.203955:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.203955:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.203960:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf6d8 00000100:00000200:14.0:1616513098.203962:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513098.203963:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513098.203968:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (24)++ 00000800:00000200:2.0:1616513098.203968:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513098.203970:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513098.203970:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513098.203972:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.203977:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.204025:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753130438080 00010000:00000200:19.0:1616513098.204036:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688753130438080/t0(0) o400->8cecb00c-1b60-0ea7-f966-525196956c40@10.31.142.22@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513098.204036:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688753130438208 00010000:00000200:10.0:1616513098.204049:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb8000 x1688753130438208/t0(0) o400->cdfbe6bb-e392-1226-ffdf-c9f053a6ee47@10.31.142.22@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:19.0:1616513098.204052:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688753130438080, offset 192 00000400:00000200:19.0:1616513098.204059:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.22@tcp1 00000100:00000200:10.0:1616513098.204064:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688753130438208, offset 192 00000400:00000200:19.0:1616513098.204066:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.22@tcp1 routed destination 00000400:00000200:10.0:1616513098.204073:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.22@tcp1 00000400:00000200:19.0:1616513098.204082:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.22@tcp1(10.31.142.22@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.204088:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:10.0:1616513098.204091:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.22@tcp1 routed destination 00000800:00000200:19.0:1616513098.204095:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.204098:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.204100:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:19.0:1616513098.204105:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513098.204107:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.22@tcp1(10.31.142.22@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:7.2:1616513098.204110:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:19.0:1616513098.204113:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513098.204114:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.204120:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.204122:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513098.204125:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.204130:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.204138:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513098.204164:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.204172:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.204175:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513098.204176:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.204182:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.204188:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707870 00000800:00000200:26.0:1616513098.204192:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513098.204195:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.204201:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.204204:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.204210:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.204214:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82088 00000800:00000200:0.0:1616513098.204218:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.204221:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.209718:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.209772:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.209781:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513098.209791:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.64@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.209800:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.64@tcp1 of length 224 into portal 26 MB=0x60194e6b93e80 00000400:00000200:20.0:1616513098.209808:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.64@tcp1 of length 224/224 into md 0xf97d0569 [1] + 0 00000400:00000200:20.0:1616513098.209812:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be000 00000400:00000200:20.0:1616513098.209815:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.209818:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513098.209823:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be000 00000800:00000200:24.0:1616513098.209825:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.209825:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.209828:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.209832:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.209835:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.209836:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.209844:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.64@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.209853:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.64@tcp1 of length 224 into portal 12 MB=0x60194e6b93f00 00000400:00000200:24.0:1616513098.209860:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.64@tcp1 of length 224/224 into md 0x865be291 [1] + 72656 00000400:00000200:24.0:1616513098.209867:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.209869:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513098.209876:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.209879:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.209881:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513098.209895:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588897951360 00010000:00000200:19.0:1616513098.209906:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1690588897951360/t0(0) o400->f98596b5-f1da-edcb-8a88-27eb8a2ec403@10.31.142.64@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.209919:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588897951360, offset 192 00000400:00000200:19.0:1616513098.209925:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.64@tcp1 00000400:00000200:19.0:1616513098.209933:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.64@tcp1 routed destination 00000100:00000200:28.0:1616513098.209942:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588897951488 00000400:00000200:19.0:1616513098.209947:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.64@tcp1(10.31.142.64@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00010000:00000200:28.0:1616513098.209954:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cebf00 x1690588897951488/t0(0) o400->eaec2bda-5f69-9bc7-96e7-743e9e986bde@10.31.142.64@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:19.0:1616513098.209954:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:19.0:1616513098.209960:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.209962:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.209964:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513098.209969:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588897951488, offset 192 00000800:00000200:19.0:1616513098.209969:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:28.0:1616513098.209975:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.64@tcp1 00000800:00000200:7.2:1616513098.209975:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:19.0:1616513098.209976:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513098.209984:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.64@tcp1 routed destination 00000400:00000200:28.0:1616513098.209999:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.64@tcp1(10.31.142.64@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.210005:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.210011:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.210014:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.210016:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.210021:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.210026:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513098.210031:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:16.0:1616513098.210037:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513098.210038:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.210041:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513098.210044:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.210047:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.210048:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513098.210048:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.210054:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706ff0 00000400:00000200:16.0:1616513098.210054:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513098.210057:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513098.210058:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdcb0 00000800:00000200:18.0:1616513098.210060:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513098.210062:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.210106:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.210118:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.210128:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513098.210129:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.210135:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.210137:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.210137:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.73@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.210142:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.73@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.210148:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.73@tcp1 of length 224 into portal 26 MB=0x5ffd47d71e080 00000400:00000200:24.0:1616513098.210149:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.73@tcp1 of length 224 into portal 12 MB=0x5ffd47d71e100 00000400:00000200:24.0:1616513098.210154:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.73@tcp1 of length 224/224 into md 0x865be291 [1] + 72880 00000400:00000200:0.0:1616513098.210155:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.73@tcp1 of length 224/224 into md 0xf97d0599 [1] + 0 00000400:00000200:24.0:1616513098.210158:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.210160:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513098.210160:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be3b8 00000400:00000200:0.0:1616513098.210163:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.210166:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513098.210166:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:24.0:1616513098.210168:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.210169:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.210171:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be3b8 00000800:00000200:0.0:1616513098.210174:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.210177:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513098.210178:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662986318080 00000800:00000200:0.0:1616513098.210179:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513098.210184:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662986317952 00010000:00000200:28.0:1616513098.210185:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557ceb600 x1688662986318080/t0(0) o400->2e1d77a3-4f88-2f55-cb3a-da931617096d@10.31.138.73@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:19.0:1616513098.210190:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688662986317952/t0(0) o400->34a35b29-954d-a637-375c-aff20aca361a@10.31.138.73@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.210202:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662986317952, offset 192 00000400:00000200:19.0:1616513098.210206:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.73@tcp1 00000100:00000200:28.0:1616513098.210208:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662986318080, offset 192 00000400:00000200:28.0:1616513098.210213:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.73@tcp1 00000400:00000200:19.0:1616513098.210213:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.73@tcp1 routed destination 00000400:00000200:28.0:1616513098.210220:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.73@tcp1 routed destination 00000400:00000200:19.0:1616513098.210228:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.73@tcp1(10.31.138.73@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513098.210235:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.73@tcp1(10.31.138.73@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.210235:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:19.0:1616513098.210240:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.210242:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:19.0:1616513098.210242:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.210244:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513098.210246:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.210247:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:28.0:1616513098.210249:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.210251:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:19.0:1616513098.210252:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.210252:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513098.210254:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.210259:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.210267:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.210271:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.210273:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.210275:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.210278:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.210281:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcddc0 00000800:00000200:16.0:1616513098.210283:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513098.210304:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.210311:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.210314:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513098.210316:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.210320:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.210326:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17064c8 00000800:00000200:2.0:1616513098.210330:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513098.216266:0:51400:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513098.216321:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.216331:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.216340:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.171@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.216349:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.171@tcp1 of length 224 into portal 26 MB=0x5ffe32bbec640 00000400:00000200:18.0:1616513098.216356:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.171@tcp1 of length 224/224 into md 0xf97d05a9 [1] + 0 00000400:00000200:18.0:1616513098.216360:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf540 00000400:00000200:18.0:1616513098.216363:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.216366:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513098.216371:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf540 00000800:00000200:18.0:1616513098.216374:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.216376:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.216377:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513098.216380:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.216432:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726040135232 00010000:00000200:19.0:1616513098.216456:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688726040135232/t0(0) o400->d1591251-d223-d43f-398e-2bfde4479f07@10.31.141.171@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:19.0:1616513098.216470:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688726040135232, offset 192 00000400:00000200:19.0:1616513098.216476:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.171@tcp1 00000400:00000200:19.0:1616513098.216483:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.171@tcp1 routed destination 00000400:00000200:19.0:1616513098.216498:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.171@tcp1(10.31.141.171@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.216504:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:19.0:1616513098.216510:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.216513:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.216515:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513098.216515:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:19.0:1616513098.216519:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.216529:0:51400:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000100:00000200:19.0:1616513098.216536:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513098.216570:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.216580:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513098.216585:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513098.216589:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.171@tcp1 : PUT - for me 00000800:00000200:26.0:1616513098.216590:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.216598:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:24.0:1616513098.216598:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.171@tcp1 of length 224 into portal 12 MB=0x5ffe32bbec9c0 00000400:00000200:26.0:1616513098.216601:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.216604:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.171@tcp1 of length 224/224 into md 0x865be291 [1] + 73104 00000400:00000200:26.0:1616513098.216607:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.216609:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.216612:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:26.0:1616513098.216613:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706bb0 00000800:00000200:26.0:1616513098.216617:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513098.216618:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513098.216620:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513098.216621:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.216623:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.216685:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688726040136128 00010000:00000200:28.0:1616513098.216697:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cec380 x1688726040136128/t0(0) o400->5baa57eb-6361-73ea-3509-62ac57733ca7@10.31.141.171@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.216712:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688726040136128, offset 192 00000400:00000200:28.0:1616513098.216718:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.171@tcp1 00000400:00000200:28.0:1616513098.216726:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.171@tcp1 routed destination 00000400:00000200:28.0:1616513098.216747:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.171@tcp1(10.31.141.171@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.216754:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.216760:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.216763:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.216765:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.216770:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.216774:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.216827:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.216834:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.216837:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513098.216839:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513098.216844:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.216848:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcded0 00000800:00000200:20.0:1616513098.216852:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.236903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.236959:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.236969:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.236978:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.144@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.236987:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.144@tcp1 of length 224 into portal 26 MB=0x603e4af2f7f00 00000400:00000200:14.0:1616513098.236994:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.144@tcp1 of length 224/224 into md 0xf97d05b9 [1] + 0 00000400:00000200:14.0:1616513098.236999:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02beaa0 00000400:00000200:14.0:1616513098.237001:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.237004:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.237009:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beaa0 00000800:00000200:14.0:1616513098.237012:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.237013:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.237015:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.237020:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.237023:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.237024:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513098.237031:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:18.0:1616513098.237033:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.19@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.237043:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.19@tcp1 of length 224 into portal 26 MB=0x5ffd4a5dedac0 00000400:00000200:18.0:1616513098.237050:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.19@tcp1 of length 224/224 into md 0xf97d05c9 [1] + 0 00000400:00000200:18.0:1616513098.237055:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf3a8 00000400:00000200:18.0:1616513098.237058:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.237061:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513098.237065:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf3a8 00000800:00000200:18.0:1616513098.237068:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.237071:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.237073:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513098.237078:0:241530:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693130586816256 00010000:00000200:19.0:1616513098.237088:0:241530:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1693130586816256/t0(0) o400->305e7520-9ace-edc6-db77-5dfdd9d79615@10.31.140.144@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513098.237092:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:19.0:1616513098.237100:0:241530:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693130586816256, offset 192 00000800:00000200:0.0:1616513098.237101:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:19.0:1616513098.237106:0:241530:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.144@tcp1 00000400:00000200:0.0:1616513098.237110:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.144@tcp1 : PUT - for me 00000400:00000200:19.0:1616513098.237113:0:241530:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.144@tcp1 routed destination 00000400:00000200:0.0:1616513098.237120:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.144@tcp1 of length 224 into portal 12 MB=0x603e4af2f8180 00000400:00000200:19.0:1616513098.237127:0:241530:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.144@tcp1(10.31.140.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513098.237127:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.144@tcp1 of length 224/224 into md 0x865be291 [1] + 73328 00000400:00000200:0.0:1616513098.237132:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:19.0:1616513098.237133:0:241530:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:0.0:1616513098.237135:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:19.0:1616513098.237139:0:241530:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:12.0:1616513098.237139:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663664548544 00000800:00000200:0.0:1616513098.237142:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:19.0:1616513098.237143:0:241530:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:19.0:1616513098.237145:0:241530:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513098.237145:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.237149:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:19.0:1616513098.237150:0:241530:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:12.0:1616513098.237152:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688663664548544/t0(0) o400->be5d4a90-ef59-5727-a8dc-0d5ab647784d@10.31.136.19@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513098.237152:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513098.237154:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:19.0:1616513098.237156:0:241530:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513098.237157:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513098.237161:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.237164:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.19@tcp1 : PUT - for me 00000100:00000200:12.0:1616513098.237167:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663664548544, offset 192 00000400:00000200:0.0:1616513098.237170:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.19@tcp1 of length 224 into portal 12 MB=0x5ffd4a5dedb80 00000400:00000200:12.0:1616513098.237173:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.19@tcp1 00000400:00000200:0.0:1616513098.237177:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.19@tcp1 of length 224/224 into md 0x865be291 [1] + 73552 00000400:00000200:12.0:1616513098.237181:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.19@tcp1 routed destination 00000400:00000200:0.0:1616513098.237181:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.237183:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513098.237197:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.19@tcp1(10.31.136.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513098.237202:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.237203:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513098.237204:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513098.237205:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513098.237210:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:4.0:1616513098.237212:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693130586816896 00000800:00000200:12.0:1616513098.237213:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:16.0:1616513098.237215:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663664548736 00000800:00000200:12.0:1616513098.237215:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513098.237215:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513098.237220:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513098.237222:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513098.237225:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:4.0:1616513098.237225:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b99680 x1693130586816896/t0(0) o400->0d5ee282-4f9e-e88c-9126-61dfe1048e3d@10.31.140.144@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513098.237226:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.237226:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:16.0:1616513098.237227:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b98d80 x1688663664548736/t0(0) o400->67146f9f-0b9b-79a1-ed41-6845fe8246fd@10.31.136.19@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513098.237229:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:2.0:1616513098.237232:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.237238:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17076d8 00000100:00000200:4.0:1616513098.237240:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693130586816896, offset 192 00000100:00000200:16.0:1616513098.237241:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663664548736, offset 192 00000800:00000200:2.0:1616513098.237242:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513098.237246:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.144@tcp1 00000400:00000200:16.0:1616513098.237251:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.19@tcp1 00000400:00000200:4.0:1616513098.237254:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.144@tcp1 routed destination 00000800:00000200:20.0:1616513098.237272:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513098.237276:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.19@tcp1 routed destination 00000800:00000200:20.0:1616513098.237280:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513098.237281:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.144@tcp1(10.31.140.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513098.237282:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.237285:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513098.237287:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513098.237289:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.237292:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.19@tcp1(10.31.136.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513098.237295:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bef68 00000800:00000200:16.0:1616513098.237299:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.237300:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:4.0:1616513098.237300:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.237302:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513098.237304:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513098.237305:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513098.237307:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513098.237309:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513098.237310:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.237314:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513098.237314:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513098.237318:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513098.237324:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.237331:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513098.237331:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513098.237334:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513098.237335:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513098.237336:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513098.237338:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.237341:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513098.237341:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.237343:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.237345:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb298 00000800:00000200:26.0:1616513098.237349:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513098.237349:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788e58 00000800:00000200:16.0:1616513098.237351:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.237411:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.237422:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.237429:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.237431:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513098.237450:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.18@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.237459:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.18@tcp1 of length 224 into portal 26 MB=0x5ffdbbabbeb00 00000400:00000200:24.0:1616513098.237466:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.18@tcp1 of length 224/224 into md 0xf97d05e9 [1] + 0 00000400:00000200:24.0:1616513098.237471:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bedd0 00000400:00000200:24.0:1616513098.237475:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.237478:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513098.237483:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bedd0 00000800:00000200:24.0:1616513098.237486:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.237489:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.237491:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.237499:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694079351552 00010000:00000200:12.0:1616513098.237505:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688694079351552/t0(0) o400->25464559-c4de-56e0-02b9-34a4e2adbf77@10.31.141.18@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.237517:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694079351552, offset 192 00000400:00000200:12.0:1616513098.237523:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.18@tcp1 00000400:00000200:12.0:1616513098.237529:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.18@tcp1 routed destination 00000800:00000200:7.2:1616513098.237534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.237542:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:12.0:1616513098.237542:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.18@tcp1(10.31.141.18@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513098.237548:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:12.0:1616513098.237548:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:2.0:1616513098.237551:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513098.237553:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513098.237555:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.18@tcp1 : PUT - for me 00000800:00000200:12.0:1616513098.237556:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.237557:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513098.237560:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.237562:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.18@tcp1 of length 224 into portal 12 MB=0x5ffdbbabbebc0 00000100:00000200:12.0:1616513098.237565:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.237565:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513098.237567:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.18@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 40432 00000400:00000200:14.0:1616513098.237573:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513098.237574:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000100:00000200:14.0:1616513098.237576:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513098.237577:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:18.0:1616513098.237580:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513098.237582:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.237582:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513098.237584:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.237585:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:14.0:1616513098.237585:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513098.237590:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be5d8 00000800:00000200:18.0:1616513098.237592:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513098.237646:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694079351744 00010000:00000200:10.0:1616513098.237657:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb8900 x1688694079351744/t0(0) o400->21f30af5-11d1-d076-7013-c35697dc5156@10.31.141.18@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513098.237671:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694079351744, offset 192 00000400:00000200:10.0:1616513098.237676:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.18@tcp1 00000400:00000200:10.0:1616513098.237683:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.18@tcp1 routed destination 00000400:00000200:10.0:1616513098.237697:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.18@tcp1(10.31.141.18@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.237703:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.237709:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.237711:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513098.237713:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.237718:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.237722:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.237732:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.237736:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.237738:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513098.237740:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.237743:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.237747:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83c28 00000800:00000200:0.0:1616513098.237750:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.238583:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513098.238614:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.238623:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.238629:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513098.238631:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.238637:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.168@tcp1 : PUT - for me 00000800:00000200:20.0:1616513098.238638:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513098.238646:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.168@tcp1 of length 224 into portal 12 MB=0x5ffdb77f89d80 00000800:00000200:20.0:1616513098.238647:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513098.238653:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.168@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 40656 00000400:00000200:20.0:1616513098.238656:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.168@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.238658:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.238660:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:20.0:1616513098.238665:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.168@tcp1 of length 224 into portal 26 MB=0x5ffdb77f89d00 00000800:00000200:2.0:1616513098.238666:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.238669:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513098.238671:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.168@tcp1 of length 224/224 into md 0xf97d05f9 [1] + 0 00000800:00000200:2.0:1616513098.238671:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513098.238676:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be660 00000100:00000200:10.0:1616513098.238678:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692959255936 00000400:00000200:20.0:1616513098.238679:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.238681:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00010000:00000200:10.0:1616513098.238685:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dd100 x1688692959255936/t0(0) o400->f0f75981-fab5-487c-3a0f-ae73e818d4d4@10.31.141.168@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:20.0:1616513098.238686:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be660 00000800:00000200:20.0:1616513098.238689:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.238690:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.238692:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.238695:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:10.0:1616513098.238696:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688692959255936, offset 192 00000100:00000200:12.0:1616513098.238697:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688692959255808 00010000:00000200:12.0:1616513098.238702:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688692959255808/t0(0) o400->65026146-757a-e365-a785-c60b574b4039@10.31.141.168@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:10.0:1616513098.238702:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.168@tcp1 00000400:00000200:10.0:1616513098.238707:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.168@tcp1 routed destination 00000100:00000200:12.0:1616513098.238712:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688692959255808, offset 192 00000400:00000200:12.0:1616513098.238717:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.168@tcp1 00000400:00000200:10.0:1616513098.238721:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.168@tcp1(10.31.141.168@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513098.238723:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.168@tcp1 routed destination 00000800:00000200:10.0:1616513098.238727:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513098.238731:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:12.0:1616513098.238736:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.168@tcp1(10.31.141.168@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.238737:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513098.238738:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.238741:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.238742:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513098.238746:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:7.2:1616513098.238746:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.238748:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513098.238749:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513098.238752:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513098.238757:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513098.238758:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.238767:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.238770:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.238772:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.238775:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.238777:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.238781:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be2a8 00000800:00000200:16.0:1616513098.238783:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513098.238800:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.238806:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.238809:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513098.238816:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.238820:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83b18 00000800:00000200:14.0:1616513098.238824:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.238827:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.239715:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.239726:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.239734:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.239736:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513098.239743:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.182@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.239753:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.182@tcp1 of length 224 into portal 26 MB=0x5ffd491eee300 00000400:00000200:24.0:1616513098.239760:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.182@tcp1 of length 224/224 into md 0xf97d0609 [1] + 0 00000400:00000200:24.0:1616513098.239765:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf210 00000400:00000200:24.0:1616513098.239767:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.239770:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513098.239775:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf210 00000800:00000200:24.0:1616513098.239778:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.239781:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.239782:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.239785:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663330054912 00010000:00000200:12.0:1616513098.239789:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688663330054912/t0(0) o400->91f89efb-5cb5-4466-0555-5cb2fd6ce4f7@10.31.136.182@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.239800:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663330054912, offset 192 00000400:00000200:12.0:1616513098.239804:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.182@tcp1 00000400:00000200:12.0:1616513098.239810:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.182@tcp1 routed destination 00000400:00000200:12.0:1616513098.239823:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.182@tcp1(10.31.136.182@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.239828:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513098.239832:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.239834:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.239836:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.239839:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513098.239843:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.239844:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.239897:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.239904:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.239906:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.239913:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.239919:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf210 00000800:00000200:26.0:1616513098.239923:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.239927:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.239965:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.240047:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.240056:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.240065:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.182@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.240073:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.182@tcp1 of length 224 into portal 12 MB=0x5ffd491eee440 00000400:00000200:0.0:1616513098.240080:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.182@tcp1 of length 224/224 into md 0x865be291 [1] + 73776 00000400:00000200:0.0:1616513098.240085:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.240088:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513098.240095:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.240099:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.240100:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.240104:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513098.240114:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663330055232 00010000:00000200:20.0:1616513098.240125:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9f500 x1688663330055232/t0(0) o400->41835b40-dfe9-952b-7628-637d7ddd8a73@10.31.136.182@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513098.240138:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663330055232, offset 192 00000400:00000200:20.0:1616513098.240143:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.182@tcp1 00000400:00000200:20.0:1616513098.240150:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.182@tcp1 routed destination 00000400:00000200:20.0:1616513098.240165:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.182@tcp1(10.31.136.182@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513098.240171:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.240176:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513098.240179:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.240181:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.240185:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.240190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.240200:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.240204:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.240206:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.240208:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513098.240211:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.240214:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a2a8 00000800:00000200:20.0:1616513098.240218:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.247924:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.247979:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.247988:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.247997:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.235@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.248006:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.235@tcp1 of length 224 into portal 26 MB=0x5fff31c6821c0 00000400:00000200:16.0:1616513098.248013:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.235@tcp1 of length 224/224 into md 0xf97d0639 [1] + 0 00000400:00000200:16.0:1616513098.248018:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be550 00000400:00000200:16.0:1616513098.248020:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.248023:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513098.248027:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be550 00000800:00000200:16.0:1616513098.248030:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.248033:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.248033:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513098.248040:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.248043:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.248044:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.248052:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.235@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.248061:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.235@tcp1 of length 224 into portal 12 MB=0x5fff31c682240 00000400:00000200:0.0:1616513098.248069:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.235@tcp1 of length 224/224 into md 0x865be291 [1] + 74000 00000400:00000200:0.0:1616513098.248075:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.248077:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513098.248084:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.248087:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513098.248088:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688794502275520 00000800:00000200:0.0:1616513098.248089:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:12.0:1616513098.248099:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688794502275520/t0(0) o400->7924d718-c5a0-50fe-77f8-ad685b70e9d2@10.31.141.235@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.248113:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688794502275520, offset 192 00000400:00000200:12.0:1616513098.248126:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.235@tcp1 00000400:00000200:12.0:1616513098.248133:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.235@tcp1 routed destination 00000100:00000200:20.0:1616513098.248148:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688794502275648 00000400:00000200:12.0:1616513098.248148:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.235@tcp1(10.31.141.235@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.248155:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:20.0:1616513098.248160:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9e300 x1688794502275648/t0(0) o400->15b05069-7878-ff58-df9e-106656be6ae4@10.31.141.235@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513098.248160:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.248163:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.248165:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.248170:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:20.0:1616513098.248175:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688794502275648, offset 192 00000100:00000200:12.0:1616513098.248177:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.248178:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513098.248181:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.235@tcp1 00000400:00000200:20.0:1616513098.248188:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.235@tcp1 routed destination 00000400:00000200:20.0:1616513098.248203:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.235@tcp1(10.31.141.235@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513098.248210:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.248215:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513098.248218:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.248220:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.248225:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.248230:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513098.248232:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.248239:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.248242:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.248244:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.248248:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:14.0:1616513098.248249:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.248250:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513098.248253:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513098.248254:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be660 00000400:00000200:20.0:1616513098.248256:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:14.0:1616513098.248258:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513098.248259:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a2a8 00000800:00000200:14.0:1616513098.248261:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513098.248262:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.262563:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513098.262567:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.262583:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.262592:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.262600:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.212@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.262609:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.212@tcp1 of length 224 into portal 26 MB=0x603deaa13eec0 00000400:00000200:0.0:1616513098.262615:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.212@tcp1 of length 224/224 into md 0xf97d0649 [1] + 0 00000800:00000200:26.0:1616513098.262617:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513098.262620:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfd38 00000400:00000200:0.0:1616513098.262623:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.262625:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:26.0:1616513098.262626:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513098.262629:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513098.262629:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfd38 00000800:00000200:0.0:1616513098.262633:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513098.262634:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.212@tcp1 : PUT - for me 00000800:00000200:0.0:1616513098.262635:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.262637:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.262640:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513098.262642:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.212@tcp1 of length 224 into portal 12 MB=0x603deaa13f000 00000400:00000200:26.0:1616513098.262648:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.212@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 40880 00000400:00000200:26.0:1616513098.262653:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.262655:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513098.262662:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.262665:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.262667:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513098.262701:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693104731320000 00010000:00000200:12.0:1616513098.262711:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1693104731320000/t0(0) o400->d4628aa6-2676-9e40-a06b-52c8339d5f52@10.31.141.212@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.262724:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693104731320000, offset 192 00000400:00000200:12.0:1616513098.262730:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.212@tcp1 00000100:00000200:10.0:1616513098.262735:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693104731320320 00000400:00000200:12.0:1616513098.262737:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.212@tcp1 routed destination 00010000:00000200:10.0:1616513098.262748:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c697080 x1693104731320320/t0(0) o400->f560b535-6ef9-1b38-19cd-5808b849ce9d@10.31.141.212@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513098.262751:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.212@tcp1(10.31.141.212@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.262757:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513098.262762:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:10.0:1616513098.262762:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693104731320320, offset 192 00000800:00000200:12.0:1616513098.262765:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.262767:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:10.0:1616513098.262768:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.212@tcp1 00000800:00000200:12.0:1616513098.262771:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513098.262775:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.212@tcp1 routed destination 00000800:00000200:7.2:1616513098.262776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513098.262778:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513098.262788:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:10.0:1616513098.262790:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.212@tcp1(10.31.141.212@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513098.262792:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.262793:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513098.262797:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513098.262799:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:10.0:1616513098.262802:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513098.262803:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bedd0 00000800:00000200:14.0:1616513098.262805:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513098.262805:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513098.262807:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513098.262808:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:10.0:1616513098.262812:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.262818:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.262869:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.262876:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.262878:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.262885:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.262888:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83a90 00000800:00000200:16.0:1616513098.262892:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.262895:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.277177:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.277235:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.277244:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.277253:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.36@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.277263:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.36@tcp1 of length 224 into portal 26 MB=0x5f91b3cbfba00 00000400:00000200:0.0:1616513098.277269:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.36@tcp1 of length 224/224 into md 0xf97d0669 [1] + 0 00000400:00000200:0.0:1616513098.277274:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be110 00000400:00000200:0.0:1616513098.277277:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.277280:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513098.277284:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be110 00000800:00000200:0.0:1616513098.277287:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.277288:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.277290:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.277295:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.277298:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.277299:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.277307:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.36@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.277316:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.36@tcp1 of length 224 into portal 12 MB=0x5f91b3cbfba80 00000400:00000200:24.0:1616513098.277323:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.36@tcp1 of length 224/224 into md 0x865be291 [1] + 74224 00000400:00000200:24.0:1616513098.277328:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.277330:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513098.277337:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.277340:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.277342:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.277347:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681270262184448 00010000:00000200:12.0:1616513098.277358:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1681270262184448/t0(0) o400->2be3187b-2051-b1cd-6432-6f24bfb9aaef@10.31.132.36@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.277373:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681270262184448, offset 192 00000400:00000200:12.0:1616513098.277379:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.36@tcp1 00000400:00000200:12.0:1616513098.277386:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.36@tcp1 routed destination 00000100:00000200:20.0:1616513098.277401:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681270262184576 00000400:00000200:12.0:1616513098.277402:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.36@tcp1(10.31.132.36@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.277413:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:20.0:1616513098.277414:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d4557cee300 x1681270262184576/t0(0) o400->bd8a079d-4413-be01-c7c9-f87f8597df86@10.31.132.36@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513098.277419:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.277421:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.277423:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000100:00000200:20.0:1616513098.277428:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681270262184576, offset 192 00000800:00000200:12.0:1616513098.277428:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.277435:0:51407:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513098.277455:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.36@tcp1 00000100:00000200:12.0:1616513098.277460:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513098.277462:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.36@tcp1 routed destination 00000400:00000200:20.0:1616513098.277478:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.36@tcp1(10.31.132.36@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513098.277484:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.277490:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513098.277492:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513098.277493:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.277495:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513098.277499:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513098.277501:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:2.0:1616513098.277502:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513098.277506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513098.277508:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.277513:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf3a8 00000800:00000200:2.0:1616513098.277516:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513098.277518:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513098.277519:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513098.277522:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.277524:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.277527:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513098.277530:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.277533:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a2a8 00000800:00000200:20.0:1616513098.277536:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.290270:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.290325:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.290334:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.290345:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.5@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.290353:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.5@tcp1 of length 224 into portal 12 MB=0x5ffe5e63ea080 00000400:00000200:24.0:1616513098.290359:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.5@tcp1 of length 224/224 into md 0x865be291 [1] + 74448 00000400:00000200:24.0:1616513098.290365:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.290367:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513098.290369:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513098.290374:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.290377:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.290379:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.290382:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513098.290451:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737759010944 00000800:00000200:14.0:1616513098.290452:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.290460:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00010000:00000200:20.0:1616513098.290463:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08a880 x1688737759010944/t0(0) o400->40cdb472-e267-899b-9c95-962146028423@10.31.141.5@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513098.290468:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.5@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.290476:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.5@tcp1 of length 224 into portal 26 MB=0x5ffe5e63e9f80 00000100:00000200:20.0:1616513098.290478:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688737759010944, offset 192 00000400:00000200:14.0:1616513098.290482:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.5@tcp1 of length 224/224 into md 0xf97d0679 [1] + 0 00000400:00000200:20.0:1616513098.290484:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.5@tcp1 00000400:00000200:14.0:1616513098.290486:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf870 00000400:00000200:14.0:1616513098.290488:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.290490:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513098.290492:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.5@tcp1 routed destination 00000400:00000200:14.0:1616513098.290496:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf870 00000400:00000200:20.0:1616513098.290507:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.5@tcp1(10.31.141.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513098.290509:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.290512:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513098.290514:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513098.290515:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.290516:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513098.290520:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513098.290522:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513098.290524:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513098.290529:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.290534:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513098.290557:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737759010688 00010000:00000200:12.0:1616513098.290567:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1688737759010688/t0(0) o400->edec09e4-376b-8425-0ddc-b8739792abff@10.31.141.5@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.290580:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688737759010688, offset 192 00000400:00000200:12.0:1616513098.290586:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.5@tcp1 00000800:00000200:2.0:1616513098.290587:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:12.0:1616513098.290593:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.5@tcp1 routed destination 00000800:00000200:2.0:1616513098.290594:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.290597:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.290604:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:12.0:1616513098.290606:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.5@tcp1(10.31.141.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513098.290608:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a2a8 00000800:00000200:12.0:1616513098.290612:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:2.0:1616513098.290612:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.290614:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:12.0:1616513098.290618:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.290620:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513098.290622:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513098.290626:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.290632:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513098.290633:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513098.290685:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.290692:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.290695:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.290697:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.290701:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.290706:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf540 00000800:00000200:0.0:1616513098.290709:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.305916:0:51410:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513098.305972:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.305981:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.305991:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.54@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.306001:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.54@tcp1 of length 224 into portal 26 MB=0x6018a34d9a940 00000400:00000200:18.0:1616513098.306007:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.54@tcp1 of length 224/224 into md 0xf97d0689 [1] + 0 00000400:00000200:18.0:1616513098.306012:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be880 00000400:00000200:18.0:1616513098.306015:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.306017:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513098.306022:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be880 00000800:00000200:18.0:1616513098.306025:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.306026:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.306028:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513098.306030:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513098.306087:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690542964058432 00010000:00000200:12.0:1616513098.306098:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1690542964058432/t0(0) o400->698bbd8e-a4c3-7c70-dd42-fd129f82efcc@10.31.142.54@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.306113:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690542964058432, offset 192 00000400:00000200:12.0:1616513098.306119:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.54@tcp1 00000400:00000200:12.0:1616513098.306127:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.54@tcp1 routed destination 00000400:00000200:12.0:1616513098.306142:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.54@tcp1(10.31.142.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.306148:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513098.306154:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.306157:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.306161:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.306166:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.306170:0:51410:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513098.306173:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513098.306186:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513098.306226:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.306233:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.306236:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.306242:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513098.306243:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513098.306248:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be000 00000800:00000200:2.0:1616513098.306251:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513098.306253:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:2.0:1616513098.306254:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513098.306262:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.54@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.306271:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.54@tcp1 of length 224 into portal 12 MB=0x6018a34d9ac80 00000400:00000200:24.0:1616513098.306277:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.54@tcp1 of length 224/224 into md 0x865be291 [1] + 74672 00000400:00000200:24.0:1616513098.306283:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.306285:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513098.306292:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.306295:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.306296:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.306299:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513098.306355:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690542964059264 00010000:00000200:20.0:1616513098.306367:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08ba80 x1690542964059264/t0(0) o400->d5ecc5d1-25c5-adb2-8fce-7e260a43d0b4@10.31.142.54@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513098.306381:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690542964059264, offset 192 00000400:00000200:20.0:1616513098.306388:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.54@tcp1 00000400:00000200:20.0:1616513098.306395:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.54@tcp1 routed destination 00000400:00000200:20.0:1616513098.306410:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.54@tcp1(10.31.142.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513098.306416:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.306422:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513098.306424:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.306428:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:11.2:1616513098.306443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.306450:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.306473:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.306477:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.306478:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.306484:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.306486:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aff0 00000800:00000200:20.0:1616513098.306489:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.306491:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.325264:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.325333:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.325343:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.325352:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.9@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.325361:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.9@tcp1 of length 224 into portal 26 MB=0x5f9131be7e500 00000400:00000200:24.0:1616513098.325368:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.9@tcp1 of length 224/224 into md 0xf97d06a9 [1] + 0 00000400:00000200:24.0:1616513098.325372:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bff58 00000400:00000200:24.0:1616513098.325375:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.325378:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513098.325382:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bff58 00000800:00000200:24.0:1616513098.325386:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.325388:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.325389:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.325395:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.325398:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.325399:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.325407:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.9@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.325416:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.9@tcp1 of length 224 into portal 12 MB=0x5f9131be7e580 00000400:00000200:16.0:1616513098.325424:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.9@tcp1 of length 224/224 into md 0x865be291 [1] + 74896 00000400:00000200:16.0:1616513098.325429:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.325432:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513098.325456:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.325459:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.325461:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.325465:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681235351430400 00010000:00000200:12.0:1616513098.325476:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1681235351430400/t0(0) o400->caffc1b3-e4bd-1b33-fab0-30332e296ab4@10.31.132.9@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:20.0:1616513098.325477:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681235351430528 00010000:00000200:20.0:1616513098.325490:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3f00 x1681235351430528/t0(0) o400->cdaf632c-74bc-71f6-aa23-3316482dec1d@10.31.132.9@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513098.325490:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681235351430400, offset 192 00000400:00000200:12.0:1616513098.325496:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.9@tcp1 00000100:00000200:20.0:1616513098.325504:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681235351430528, offset 192 00000400:00000200:12.0:1616513098.325504:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.9@tcp1 routed destination 00000400:00000200:20.0:1616513098.325510:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.9@tcp1 00000400:00000200:20.0:1616513098.325518:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.9@tcp1 routed destination 00000400:00000200:12.0:1616513098.325519:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.9@tcp1(10.31.132.9@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.325526:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513098.325532:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.9@tcp1(10.31.132.9@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513098.325539:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513098.325540:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.325542:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513098.325545:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.325545:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513098.325548:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513098.325549:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513098.325550:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.325555:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513098.325556:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.325556:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513098.325561:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.325576:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.325583:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.325586:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.325588:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.325593:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.325597:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aff0 00000800:00000200:0.0:1616513098.325601:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513098.325612:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.325619:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.325622:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513098.325628:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513098.325629:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.325634:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf760 00000800:00000200:14.0:1616513098.325638:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.334994:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.335049:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.335059:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513098.335068:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.246@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.335077:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.246@tcp1 of length 224 into portal 26 MB=0x6041932fd08c0 00000400:00000200:20.0:1616513098.335084:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.246@tcp1 of length 224/224 into md 0xf97d06b9 [1] + 0 00000400:00000200:20.0:1616513098.335088:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfa90 00000400:00000200:20.0:1616513098.335091:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.335094:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513098.335098:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfa90 00000800:00000200:20.0:1616513098.335101:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.335103:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.335104:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.335108:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.335162:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356136401088 00010000:00000200:12.0:1616513098.335173:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1693356136401088/t0(0) o400->44b64bbe-1546-d00c-5424-fb1f2ddaa227@10.31.128.246@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.335187:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693356136401088, offset 192 00000400:00000200:12.0:1616513098.335193:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.246@tcp1 00000400:00000200:12.0:1616513098.335201:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.246@tcp1 routed destination 00000400:00000200:12.0:1616513098.335216:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.246@tcp1(10.31.128.246@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.335223:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513098.335228:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513098.335229:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.335232:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.335234:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.335238:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513098.335245:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.335258:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513098.335282:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.335291:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.335300:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.246@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.335308:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.246@tcp1 of length 224 into portal 12 MB=0x6041932fd0cc0 00000400:00000200:24.0:1616513098.335315:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.246@tcp1 of length 224/224 into md 0x865be291 [1] + 75120 00000800:00000200:2.0:1616513098.335315:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513098.335320:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513098.335322:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000100:00000200:24.0:1616513098.335323:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:2.0:1616513098.335325:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.335329:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513098.335332:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513098.335333:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.335335:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.335337:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:2.0:1616513098.335337:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bed48 00000800:00000200:2.0:1616513098.335341:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.335344:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:20.0:1616513098.335349:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693356136402112 00010000:00000200:20.0:1616513098.335359:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b088d80 x1693356136402112/t0(0) o400->b960c3f3-e69f-8877-e129-5539aaa80801@10.31.128.246@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513098.335373:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693356136402112, offset 192 00000400:00000200:20.0:1616513098.335378:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.246@tcp1 00000400:00000200:20.0:1616513098.335385:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.246@tcp1 routed destination 00000400:00000200:20.0:1616513098.335399:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.246@tcp1(10.31.128.246@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513098.335406:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.335411:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513098.335414:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.335416:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.335420:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.335424:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.335444:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.335450:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.335453:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.335454:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.335459:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.335463:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79aff0 00000800:00000200:16.0:1616513098.335467:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.350865:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.350920:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.350929:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.350938:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.42@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.350947:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.42@tcp1 of length 224 into portal 26 MB=0x5ffd3d58f7600 00000400:00000200:0.0:1616513098.350954:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.42@tcp1 of length 224/224 into md 0xf97d06c9 [1] + 0 00000400:00000200:0.0:1616513098.350958:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfcb0 00000400:00000200:0.0:1616513098.350961:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.350964:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513098.350968:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfcb0 00000800:00000200:0.0:1616513098.350972:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.350973:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.350974:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.350977:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.351031:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660169684480 00010000:00000200:12.0:1616513098.351042:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688660169684480/t0(0) o400->7f349f95-4116-d8f0-dd53-7a857a3f6654@10.31.129.42@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.351057:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660169684480, offset 192 00000400:00000200:12.0:1616513098.351064:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.42@tcp1 00000400:00000200:12.0:1616513098.351071:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.42@tcp1 routed destination 00000400:00000200:12.0:1616513098.351086:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.42@tcp1(10.31.129.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.351093:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513098.351097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513098.351099:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.351101:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.351104:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.351109:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.351115:0:51415:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513098.351116:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513098.351150:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.351160:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513098.351168:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.42@tcp1 : PUT - for me 00000800:00000200:18.0:1616513098.351174:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513098.351177:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.42@tcp1 of length 224 into portal 12 MB=0x5ffd3d58f7940 00000800:00000200:18.0:1616513098.351181:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513098.351183:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.42@tcp1 of length 224/224 into md 0x865be291 [1] + 75344 00000400:00000200:18.0:1616513098.351184:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.351188:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513098.351190:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000100:00000200:20.0:1616513098.351191:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513098.351199:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513098.351200:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf078 00000800:00000200:20.0:1616513098.351202:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513098.351204:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513098.351205:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513098.351207:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513098.351209:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513098.351213:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513098.351215:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.351217:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513098.351219:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513098.351220:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660169685312 00000800:00000200:18.0:1616513098.351221:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513098.351231:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c662400 x1688660169685312/t0(0) o400->a73bded5-cbe5-a4f0-0c8b-135500bf9fab@10.31.129.42@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513098.351246:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660169685312, offset 192 00000400:00000200:24.0:1616513098.351252:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.42@tcp1 00000400:00000200:24.0:1616513098.351259:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.42@tcp1 routed destination 00000400:00000200:24.0:1616513098.351274:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.42@tcp1(10.31.129.42@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513098.351280:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513098.351286:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513098.351289:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.351291:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.351295:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.351300:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.351311:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.351315:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.351317:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.351320:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513098.351322:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.351325:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3980 00000800:00000200:24.0:1616513098.351328:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.352613:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513098.352643:0:51415:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513098.352666:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.352675:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.352684:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.47@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.352693:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.47@tcp1 of length 224 into portal 26 MB=0x5f909c941b840 00000800:00000200:26.0:1616513098.352699:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:16.0:1616513098.352700:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.47@tcp1 of length 224/224 into md 0xf97d06e9 [1] + 0 00000400:00000200:16.0:1616513098.352705:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02beee0 00000400:00000200:16.0:1616513098.352708:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:26.0:1616513098.352709:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:16.0:1616513098.352711:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513098.352715:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beee0 00000400:00000200:26.0:1616513098.352717:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.47@tcp1 : PUT - for me 00000800:00000200:16.0:1616513098.352717:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.352720:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.352722:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.352725:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513098.352726:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.47@tcp1 of length 224 into portal 12 MB=0x5f909c941b8c0 00000400:00000200:26.0:1616513098.352733:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.47@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 41104 00000400:00000200:26.0:1616513098.352738:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.352740:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513098.352748:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.352752:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.352753:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.352756:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513098.352782:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681195310102592 00010000:00000200:12.0:1616513098.352792:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1681195310102592/t0(0) o400->e6415f22-b002-3c80-6191-458a0f4a5fa3@10.31.131.47@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.352805:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681195310102592, offset 192 00000400:00000200:12.0:1616513098.352811:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.47@tcp1 00000100:00000200:10.0:1616513098.352816:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681195310102720 00000400:00000200:12.0:1616513098.352818:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.47@tcp1 routed destination 00010000:00000200:10.0:1616513098.352828:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd219200 x1681195310102720/t0(0) o400->8013c9c0-625f-b010-4a48-118b9a62bc0e@10.31.131.47@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513098.352832:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.47@tcp1(10.31.131.47@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.352839:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:10.0:1616513098.352842:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681195310102720, offset 192 00000800:00000200:12.0:1616513098.352844:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.352847:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:10.0:1616513098.352848:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.47@tcp1 00000800:00000200:12.0:1616513098.352849:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.352853:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513098.352856:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.47@tcp1 routed destination 00000800:00000200:7.2:1616513098.352859:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513098.352860:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:10.0:1616513098.352871:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.47@tcp1(10.31.131.47@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513098.352873:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513098.352877:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513098.352879:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.352882:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513098.352883:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.352886:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513098.352888:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513098.352889:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:10.0:1616513098.352892:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513098.352894:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bebb0 00000800:00000200:11.2:1616513098.352897:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513098.352898:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.352900:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513098.352953:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.352960:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.352963:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.352969:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.352973:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e836d8 00000800:00000200:0.0:1616513098.352978:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.352981:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.353015:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.353024:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.353029:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513098.353033:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.353037:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.69@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.353044:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.69@tcp1 of length 224 into portal 26 MB=0x60182e3e51e00 00000400:00000200:16.0:1616513098.353049:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.69@tcp1 of length 224/224 into md 0xf97d0709 [1] + 0 00000400:00000200:16.0:1616513098.353053:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf980 00000400:00000200:16.0:1616513098.353056:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.353058:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513098.353062:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf980 00000800:00000200:16.0:1616513098.353064:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.353066:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.353068:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.353073:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511541083648 00010000:00000200:12.0:1616513098.353078:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1690511541083648/t0(0) o400->6d7e264f-75e9-6e9f-1dac-6ba7f95134ab@10.31.142.69@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.353089:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511541083648, offset 192 00000400:00000200:12.0:1616513098.353093:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.69@tcp1 00000400:00000200:12.0:1616513098.353098:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.69@tcp1 routed destination 00000400:00000200:12.0:1616513098.353112:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.69@tcp1(10.31.142.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.353118:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513098.353122:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.353124:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.353126:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.353129:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513098.353133:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.353133:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.353152:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.353156:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.353158:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513098.353161:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.353164:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.353168:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfe48 00000800:00000200:26.0:1616513098.353172:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513098.353350:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.353362:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.353368:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513098.353372:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513098.353376:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.69@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.353385:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.69@tcp1 of length 224 into portal 12 MB=0x60182e3e52900 00000400:00000200:14.0:1616513098.353391:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.69@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 41328 00000400:00000200:14.0:1616513098.353396:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.353399:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513098.353405:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.353408:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.353409:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513098.353415:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511541086464 00010000:00000200:10.0:1616513098.353421:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebda00 x1690511541086464/t0(0) o400->29d415f0-375d-9e92-f100-d8e6654a3104@10.31.142.69@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513098.353453:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511541086464, offset 192 00000400:00000200:10.0:1616513098.353457:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.69@tcp1 00000400:00000200:10.0:1616513098.353463:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.69@tcp1 routed destination 00000400:00000200:10.0:1616513098.353475:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.69@tcp1(10.31.142.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.353480:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.353484:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.353486:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513098.353488:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.353491:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.353496:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.353506:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.353510:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.353513:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.353515:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.353519:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.353521:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83870 00000800:00000200:0.0:1616513098.353524:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.365856:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.365909:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.365918:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513098.365926:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.77@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.365935:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.77@tcp1 of length 224 into portal 26 MB=0x602161b082240 00000400:00000200:20.0:1616513098.365941:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.77@tcp1 of length 224/224 into md 0xf97d0729 [1] + 0 00000400:00000200:20.0:1616513098.365946:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be6e8 00000400:00000200:20.0:1616513098.365950:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.365952:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513098.365957:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be6e8 00000800:00000200:20.0:1616513098.365960:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.365961:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.365962:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.365968:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.365971:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.365972:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.365981:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.77@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.365989:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.77@tcp1 of length 224 into portal 12 MB=0x602161b082300 00000400:00000200:16.0:1616513098.365996:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.77@tcp1 of length 224/224 into md 0x865be291 [1] + 75568 00000400:00000200:16.0:1616513098.366001:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.366004:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513098.366011:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.366015:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513098.366016:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691143826317888 00000800:00000200:16.0:1616513098.366017:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:12.0:1616513098.366026:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1691143826317888/t0(0) o400->896d2b4a-ec88-4d65-02cd-f59b9ed06019@10.31.141.77@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.366039:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691143826317888, offset 192 00000400:00000200:12.0:1616513098.366045:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.77@tcp1 00000400:00000200:12.0:1616513098.366052:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.77@tcp1 routed destination 00000400:00000200:12.0:1616513098.366066:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.77@tcp1(10.31.141.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.366072:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:28.0:1616513098.366076:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691143826318080 00000800:00000200:12.0:1616513098.366077:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.366080:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.366082:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.366086:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:28.0:1616513098.366088:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d2d00 x1691143826318080/t0(0) o400->0ece1d1f-8c57-6a05-76ab-a58c058d009c@10.31.141.77@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513098.366092:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.366092:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513098.366103:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691143826318080, offset 192 00000400:00000200:28.0:1616513098.366109:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.77@tcp1 00000400:00000200:28.0:1616513098.366116:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.77@tcp1 routed destination 00000400:00000200:28.0:1616513098.366131:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.77@tcp1(10.31.141.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.366139:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.366145:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.366148:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513098.366149:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513098.366150:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.366155:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513098.366156:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.366159:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513098.366159:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513098.366165:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.366171:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be220 00000800:00000200:18.0:1616513098.366174:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.366177:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513098.366225:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.366233:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.366235:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.366236:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.366242:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.366246:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcde48 00000800:00000200:0.0:1616513098.366250:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.367179:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.367238:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.367247:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.367256:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.82@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.367266:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.82@tcp1 of length 224 into portal 26 MB=0x6059f6d828ac0 00000400:00000200:2.0:1616513098.367273:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.82@tcp1 of length 224/224 into md 0xf97d0749 [1] + 0 00000400:00000200:2.0:1616513098.367278:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf650 00000400:00000200:2.0:1616513098.367281:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.367284:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.367288:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf650 00000800:00000200:2.0:1616513098.367291:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.367293:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.367295:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.367301:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513098.367366:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695032155474624 00010000:00000200:12.0:1616513098.367375:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1695032155474624/t0(0) o400->a1042dd0-17c3-acd7-6b7e-65b3e93f8cae@10.31.138.82@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.367388:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695032155474624, offset 192 00000400:00000200:12.0:1616513098.367393:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.82@tcp1 00000400:00000200:12.0:1616513098.367400:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.82@tcp1 routed destination 00000400:00000200:12.0:1616513098.367413:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.82@tcp1(10.31.138.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.367419:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513098.367423:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.367427:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.367429:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:7.2:1616513098.367443:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:12.0:1616513098.367445:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:12.0:1616513098.367452:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513098.367496:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.367503:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.367505:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513098.367512:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.367517:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02becc0 00000800:00000200:18.0:1616513098.367521:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.367524:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513098.368847:0:51417:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.368902:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.368911:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.368920:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.82@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.368929:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.82@tcp1 of length 224 into portal 12 MB=0x6059f6d828f80 00000400:00000200:2.0:1616513098.368936:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.82@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 41552 00000400:00000200:2.0:1616513098.368941:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.368944:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513098.368957:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.368960:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.368962:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.368965:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513098.369020:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695032155475840 00010000:00000200:10.0:1616513098.369032:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9200 x1695032155475840/t0(0) o400->04bf754e-33b1-c7e2-c6b8-3bc79a2ae458@10.31.138.82@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513098.369045:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695032155475840, offset 192 00000400:00000200:10.0:1616513098.369050:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.82@tcp1 00000400:00000200:10.0:1616513098.369058:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.82@tcp1 routed destination 00000400:00000200:10.0:1616513098.369071:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.82@tcp1(10.31.138.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.369077:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.369083:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.369085:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513098.369087:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.369091:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.369097:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.369148:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.369155:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.369157:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.369163:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.369167:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e825d8 00000800:00000200:16.0:1616513098.369171:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.369174:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.377853:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.377907:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.377917:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513098.377925:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.102@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.377935:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.102@tcp1 of length 224 into portal 26 MB=0x5fff20920bb00 00000400:00000200:20.0:1616513098.377942:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.102@tcp1 of length 224/224 into md 0xf97d0769 [1] + 0 00000400:00000200:20.0:1616513098.377946:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bec38 00000400:00000200:20.0:1616513098.377949:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.377952:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513098.377956:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bec38 00000800:00000200:20.0:1616513098.377959:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.377962:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.377962:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.377967:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.378019:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688789883861760 00010000:00000200:12.0:1616513098.378029:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688789883861760/t0(0) o400->85313e54-7165-8752-c064-4eaef731d025@10.31.136.102@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.378043:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688789883861760, offset 192 00000400:00000200:12.0:1616513098.378049:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.102@tcp1 00000400:00000200:12.0:1616513098.378057:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.102@tcp1 routed destination 00000400:00000200:12.0:1616513098.378072:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.102@tcp1(10.31.136.102@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.378079:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513098.378084:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.378087:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.378089:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.378094:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.378100:0:51418:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513098.378101:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513098.378156:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.378163:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.378165:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513098.378172:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.378178:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beb28 00000800:00000200:18.0:1616513098.378181:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.378184:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.378237:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.378291:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.378311:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.378319:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.102@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.378327:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.102@tcp1 of length 224 into portal 12 MB=0x5fff20920bc00 00000400:00000200:16.0:1616513098.378333:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.102@tcp1 of length 224/224 into md 0x865be291 [1] + 75792 00000400:00000200:16.0:1616513098.378339:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.378341:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513098.378347:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.378349:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.378350:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.378353:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.378412:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688789883862016 00010000:00000200:28.0:1616513098.378424:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d7500 x1688789883862016/t0(0) o400->4e8fe3e9-514c-9da2-0094-62ce0a33ef01@10.31.136.102@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.378451:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688789883862016, offset 192 00000400:00000200:28.0:1616513098.378457:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.102@tcp1 00000400:00000200:28.0:1616513098.378465:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.102@tcp1 routed destination 00000400:00000200:28.0:1616513098.378480:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.102@tcp1(10.31.136.102@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.378486:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.378492:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.378495:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.378497:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.378501:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.378506:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.378517:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.378523:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.378526:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.378527:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.378532:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.378536:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd210 00000800:00000200:0.0:1616513098.378540:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.379792:0:51418:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.379846:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.379856:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.379864:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.234@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.379874:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.234@tcp1 of length 224 into portal 12 MB=0x603e105c76b80 00000400:00000200:14.0:1616513098.379880:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.234@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 41776 00000400:00000200:14.0:1616513098.379891:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.379893:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513098.379900:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.379900:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.379903:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.379905:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513098.379964:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114859744128 00010000:00000200:10.0:1616513098.379976:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9200 x1693114859744128/t0(0) o400->886876fa-8196-a63b-2824-662ad07527e2@10.31.136.234@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513098.379991:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693114859744128, offset 192 00000400:00000200:10.0:1616513098.379997:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.234@tcp1 00000400:00000200:10.0:1616513098.380005:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.234@tcp1 routed destination 00000800:00000200:11.2:1616513098.380018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:10.0:1616513098.380020:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.234@tcp1(10.31.136.234@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.380028:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513098.380033:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.380036:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513098.380038:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513098.380043:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.380048:0:51418:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513098.380071:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.380080:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.380089:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.234@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.380097:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.234@tcp1 of length 224 into portal 26 MB=0x603e105c76ac0 00000800:00000200:18.0:1616513098.380103:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:24.0:1616513098.380104:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.234@tcp1 of length 224/224 into md 0xf97d0779 [1] + 0 00000400:00000200:24.0:1616513098.380109:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfa08 00000800:00000200:18.0:1616513098.380109:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:24.0:1616513098.380112:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513098.380112:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.380114:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513098.380118:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfa08 00000400:00000200:18.0:1616513098.380119:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513098.380121:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513098.380123:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e826e8 00000800:00000200:24.0:1616513098.380124:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.380126:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513098.380127:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:16.0:1616513098.380128:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513098.380130:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513098.380187:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693114859743936 00010000:00000200:12.0:1616513098.380197:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1693114859743936/t0(0) o400->1ac05e7c-f825-cfd3-8362-83c25c928f89@10.31.136.234@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.380211:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693114859743936, offset 192 00000400:00000200:12.0:1616513098.380218:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.234@tcp1 00000400:00000200:12.0:1616513098.380225:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.234@tcp1 routed destination 00000400:00000200:12.0:1616513098.380240:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.234@tcp1(10.31.136.234@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.380247:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513098.380252:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.380255:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513098.380257:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513098.380262:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.380267:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513098.380269:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513098.380320:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.380327:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.380330:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.380336:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.380342:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf298 00000800:00000200:20.0:1616513098.380346:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.380348:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.389629:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.389685:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.389695:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513098.389701:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513098.389704:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.74@tcp1 : PUT - for me 00000800:00000200:14.0:1616513098.389711:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.389713:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.74@tcp1 of length 224 into portal 26 MB=0x5ffd41b5c9140 00000400:00000200:14.0:1616513098.389720:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.74@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.389720:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.74@tcp1 of length 224/224 into md 0xf97d0789 [1] + 0 00000400:00000200:2.0:1616513098.389725:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfba0 00000400:00000200:2.0:1616513098.389727:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513098.389729:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.74@tcp1 of length 224 into portal 12 MB=0x5ffd41b5c91c0 00000100:00000200:2.0:1616513098.389730:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.389734:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfba0 00000800:00000200:2.0:1616513098.389737:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513098.389739:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.74@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 42000 00000800:00000200:2.0:1616513098.389740:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513098.389741:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.389744:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513098.389744:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513098.389747:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513098.389753:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.389756:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.389758:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513098.389800:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661340754240 00010000:00000200:12.0:1616513098.389810:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688661340754240/t0(0) o400->2512e803-885f-304e-1203-2855ecbe7258@10.31.133.74@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.389823:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661340754240, offset 192 00000100:00000200:10.0:1616513098.389823:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661340754368 00000400:00000200:12.0:1616513098.389828:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.74@tcp1 00000400:00000200:12.0:1616513098.389835:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.74@tcp1 routed destination 00010000:00000200:10.0:1616513098.389835:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb8d80 x1688661340754368/t0(0) o400->70a1e146-5602-50e4-256e-6aea6e086f5b@10.31.133.74@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:12.0:1616513098.389849:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.74@tcp1(10.31.133.74@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513098.389849:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661340754368, offset 192 00000800:00000200:12.0:1616513098.389855:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:10.0:1616513098.389856:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.74@tcp1 00000800:00000200:12.0:1616513098.389861:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.389863:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:10.0:1616513098.389863:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.74@tcp1 routed destination 00000800:00000200:12.0:1616513098.389865:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.389870:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.389874:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513098.389876:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:10.0:1616513098.389878:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.74@tcp1(10.31.133.74@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.389885:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.389890:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.389893:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513098.389895:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.389900:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.389915:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513098.389945:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.389952:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.389955:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513098.389956:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.389962:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.389967:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beff0 00000800:00000200:24.0:1616513098.389969:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513098.389972:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513098.389976:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.389978:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.389985:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.389989:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82dd0 00000800:00000200:24.0:1616513098.389993:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.389996:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513098.394592:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929552, portal 4 00000100:00000200:22.0:1616513098.394598:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929552, offset 0 00000400:00000200:22.0:1616513098.394605:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:22.0:1616513098.394610:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:22.0:1616513098.394623:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.394630:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:22.0:1616513098.394636:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.394639:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:22.0:1616513098.394641:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:22.0:1616513098.394646:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513098.394702:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513098.394759:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513098.394766:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513098.394769:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.394776:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513098.394782:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d7464621b00 x1693067725929552/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513098.394806:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b188 00000800:00000200:20.0:1616513098.394810:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513098.394813:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513098.394842:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:24.0:1616513098.394895:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513098.394903:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:24.0:1616513098.394911:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:24.0:1616513098.394919:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627c50 00000400:00000200:24.0:1616513098.394925:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dd3c1 [1] + 192 00000400:00000200:24.0:1616513098.394930:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.394934:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d7464621b00 x1693067725929552/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513098.394947:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513098.394948:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513098.394951:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513098.394953:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:22.0:1616513098.394957:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d7464621b00 x1693067725929552/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:22.0:1616513098.394970:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d7464621b00 x1693067725929552/t0(0) o13->boslfs02-OST0031-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513098.394980:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a990 00000100:00000200:7.0:1616513098.395089:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929568, portal 4 00000100:00000200:7.0:1616513098.395094:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929568, offset 0 00000400:00000200:7.0:1616513098.395101:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:7.0:1616513098.395108:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:7.0:1616513098.395122:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513098.395128:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:7.0:1616513098.395135:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513098.395138:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:7.0:1616513098.395140:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:7.0:1616513098.395146:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513098.395151:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513098.395160:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513098.395164:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513098.395167:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.395170:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:20.0:1616513098.395172:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513098.395178:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cca3ee780 x1693067725929568/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513098.395188:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637edd0 00000800:00000200:20.0:1616513098.395190:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513098.395194:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513098.395204:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513098.395212:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:24.0:1616513098.395214:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:16.0:1616513098.395221:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:16.0:1616513098.395229:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627c60 00000400:00000200:16.0:1616513098.395236:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dd3d1 [1] + 192 00000400:00000200:16.0:1616513098.395241:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.395246:0:6461:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cca3ee780 x1693067725929568/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:16.0:1616513098.395261:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513098.395264:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:16.0:1616513098.395266:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:7.0:1616513098.395269:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cca3ee780 x1693067725929568/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513098.395282:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cca3ee780 x1693067725929568/t0(0) o13->boslfs02-OST000d-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513098.395293:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637ef68 00000800:00000200:25.2:1616513098.430716:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cdb3d1c00] (20)++ 00000800:00000200:25.0:1616513098.430737:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (21)++ 00000800:00000200:25.0:1616513098.430745:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.146@o2ib1 00000400:00000200:25.0:1616513098.430754:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.146@o2ib1 : PUT - for me 00000400:00000200:25.0:1616513098.430763:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.146@o2ib1 of length 224 into portal 26 MB=0x604a72a3b7200 00000400:00000200:25.0:1616513098.430770:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.242.46.146@o2ib1 of length 224/224 into md 0xf97d07a9 [1] + 0 00000400:00000200:25.0:1616513098.430774:0:6467:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfed0 00000400:00000200:25.0:1616513098.430778:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:25.0:1616513098.430781:0:6467:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:25.0:1616513098.430786:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfed0 00000800:00000200:25.0:1616513098.430790:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cdb3d1c00] (22)++ 00000800:00000200:5.0:1616513098.430790:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)++ 00000800:00000200:25.0:1616513098.430792:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cdb3d1c00] (23)-- 00000800:00000200:25.0:1616513098.430798:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (23)-- 00000800:00000200:25.0:1616513098.430801:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)-- 00000800:00000200:5.0:1616513098.430802:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.146@o2ib1 00000400:00000200:5.0:1616513098.430811:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.146@o2ib1 : PUT - for me 00000400:00000200:5.0:1616513098.430819:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.146@o2ib1 of length 224 into portal 12 MB=0x604a72a3b7280 00000400:00000200:5.0:1616513098.430825:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.46.146@o2ib1 of length 224/224 into md 0x42efdd85 [1] + 43616 00000400:00000200:5.0:1616513098.430830:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:5.0:1616513098.430833:0:6468:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:5.0:1616513098.430841:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cdb3d1c00] (21)++ 00000800:00000200:5.0:1616513098.430845:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cdb3d1c00] (22)-- 00000800:00000200:5.0:1616513098.430847:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (21)-- 00000100:00000200:12.0:1616513098.430853:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693965874852352 00010000:00000200:12.0:1616513098.430864:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1693965874852352/t0(0) o400->187808d2-e90b-a5a7-7a66-336ae372a50d@10.242.46.146@o2ib1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.430878:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693965874852352, offset 192 00000400:00000200:12.0:1616513098.430884:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.146@o2ib1 00000400:00000200:12.0:1616513098.430891:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.146@o2ib1 local destination 00000400:00000200:12.0:1616513098.430904:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.146@o2ib1(10.242.46.146@o2ib1:10.242.46.146@o2ib1) : PUT try# 0 00000100:00000200:29.0:1616513098.430909:0:170085:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693965874852480 00000800:00000200:12.0:1616513098.430911:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.146@o2ib1 00000800:00000200:12.0:1616513098.430917:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8419b46e00] -> 10.242.46.146@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.430920:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cdb3d1c00] (20)++ 00010000:00000200:29.0:1616513098.430922:0:170085:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d843f7b8480 x1693965874852480/t0(0) o400->d0eeb338-1574-8814-a38c-3edcd7fc9caf@10.242.46.146@o2ib1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513098.430922:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cdb3d1c00] (21)++ 00000800:00000200:12.0:1616513098.430927:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cdb3d1c00] (22)-- 00000800:00000200:25.2:1616513098.430934:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cdb3d1c00] (21)++ 00000100:00000200:12.0:1616513098.430935:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:29.0:1616513098.430937:0:170085:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693965874852480, offset 192 00000400:00000200:29.0:1616513098.430943:0:170085:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.146@o2ib1 00000800:00000200:25.0:1616513098.430943:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)++ 00000800:00000200:25.0:1616513098.430948:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cdb3d1c00] (23)-- 00000400:00000200:29.0:1616513098.430951:0:170085:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.146@o2ib1 local destination 00000400:00000200:25.0:1616513098.430951:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:25.0:1616513098.430957:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.146@o2ib1: PUT: OK 00000400:00000200:25.0:1616513098.430962:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be770 00000400:00000200:29.0:1616513098.430965:0:170085:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.146@o2ib1(10.242.46.146@o2ib1:10.242.46.146@o2ib1) : PUT try# 0 00000800:00000200:25.0:1616513098.430970:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)-- 00000800:00000200:29.0:1616513098.430971:0:170085:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.146@o2ib1 00000800:00000200:25.0:1616513098.430972:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (21)-- 00000800:00000200:29.0:1616513098.430978:0:170085:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8419b46e00] -> 10.242.46.146@o2ib1 (2) version: 12 00000800:00000200:29.0:1616513098.430981:0:170085:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cdb3d1c00] (20)++ 00000800:00000200:29.0:1616513098.430983:0:170085:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cdb3d1c00] (21)++ 00000800:00000200:29.0:1616513098.430990:0:170085:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cdb3d1c00] (22)-- 00000800:00000200:25.2:1616513098.430997:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cdb3d1c00] (21)++ 00000800:00000200:25.0:1616513098.431005:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)++ 00000800:00000200:25.0:1616513098.431008:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cdb3d1c00] (23)-- 00000400:00000200:25.0:1616513098.431010:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513098.431013:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (22)-- 00000400:00000200:25.0:1616513098.431014:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.146@o2ib1: PUT: OK 00000400:00000200:25.0:1616513098.431017:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd76b3760 00000800:00000200:25.0:1616513098.431020:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cdb3d1c00] (21)-- 00000800:00000200:11.2:1616513098.432503:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.432517:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.432527:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.432536:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.213@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.432546:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.213@tcp1 of length 224 into portal 26 MB=0x601958b4fa180 00000400:00000200:0.0:1616513098.432553:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.213@tcp1 of length 224/224 into md 0xf97d07c9 [1] + 0 00000400:00000200:0.0:1616513098.432557:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be7f8 00000400:00000200:0.0:1616513098.432560:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.432563:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513098.432568:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be7f8 00000800:00000200:0.0:1616513098.432571:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.432572:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.432574:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.432579:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.432582:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.432583:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513098.432591:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.213@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.432600:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.213@tcp1 of length 224 into portal 12 MB=0x601958b4fa240 00000400:00000200:20.0:1616513098.432607:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.213@tcp1 of length 224/224 into md 0x865be291 [1] + 76016 00000400:00000200:20.0:1616513098.432612:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.432615:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513098.432622:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.432626:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.432628:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.432630:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690591659270528 00010000:00000200:12.0:1616513098.432641:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1690591659270528/t0(0) o400->8f8bcb46-5e93-ef38-68b3-3fc42df6c1d5@10.31.140.213@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.432656:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690591659270528, offset 192 00000400:00000200:12.0:1616513098.432665:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.213@tcp1 00000400:00000200:12.0:1616513098.432672:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.213@tcp1 routed destination 00000100:00000200:28.0:1616513098.432686:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690591659270720 00000400:00000200:12.0:1616513098.432688:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.213@tcp1(10.31.140.213@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.432695:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:28.0:1616513098.432700:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663600 x1690591659270720/t0(0) o400->3a2d1f26-303c-317c-7ef3-9eb302efe586@10.31.140.213@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513098.432701:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.432704:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.432706:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.432711:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:28.0:1616513098.432714:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690591659270720, offset 192 00000800:00000200:7.2:1616513098.432717:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513098.432718:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513098.432720:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.213@tcp1 00000400:00000200:28.0:1616513098.432728:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.213@tcp1 routed destination 00000400:00000200:28.0:1616513098.432743:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.213@tcp1(10.31.140.213@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.432750:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.432756:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.432758:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.432761:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.432766:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.432771:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513098.432773:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.432780:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.432783:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.432785:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513098.432789:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.432795:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfc28 00000800:00000200:18.0:1616513098.432799:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513098.432825:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.432832:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.432835:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.432836:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.432842:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.432846:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc550 00000800:00000200:16.0:1616513098.432850:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.445051:0:51426:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.445106:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.445116:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.445125:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.196@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.445134:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.196@tcp1 of length 224 into portal 12 MB=0x605a072d18d80 00000400:00000200:2.0:1616513098.445141:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.196@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 42224 00000400:00000200:2.0:1616513098.445148:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.445151:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513098.445158:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.445159:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.445161:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.445164:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513098.445224:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036539506048 00000800:00000200:11.2:1616513098.445233:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:10.0:1616513098.445236:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647da880 x1695036539506048/t0(0) o400->cf20ed29-f5c3-28cb-6c41-4fc006f4b2a1@10.31.129.196@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513098.445251:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1695036539506048, offset 192 00000400:00000200:10.0:1616513098.445257:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.196@tcp1 00000400:00000200:10.0:1616513098.445264:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.196@tcp1 routed destination 00000400:00000200:10.0:1616513098.445279:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.196@tcp1(10.31.129.196@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.445286:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513098.445287:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.445291:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.445294:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513098.445296:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513098.445297:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.445301:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.445305:0:51426:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513098.445306:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.196@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.445315:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.196@tcp1 of length 224 into portal 26 MB=0x605a072d18cc0 00000400:00000200:20.0:1616513098.445321:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.196@tcp1 of length 224/224 into md 0xf97d07e9 [1] + 0 00000400:00000200:20.0:1616513098.445326:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be990 00000400:00000200:20.0:1616513098.445329:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.445331:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513098.445336:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be990 00000800:00000200:20.0:1616513098.445338:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.445342:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.445344:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.445348:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513098.445362:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.445369:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.445371:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513098.445378:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.445381:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e837e8 00000800:00000200:14.0:1616513098.445385:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.445388:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513098.445400:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1695036539505856 00010000:00000200:12.0:1616513098.445411:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1695036539505856/t0(0) o400->8543633b-3c69-12ab-65b8-db4ef9a67807@10.31.129.196@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.445425:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1695036539505856, offset 192 00000400:00000200:12.0:1616513098.445431:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.196@tcp1 00000400:00000200:12.0:1616513098.445454:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.196@tcp1 routed destination 00000400:00000200:12.0:1616513098.445469:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.196@tcp1(10.31.129.196@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.445476:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513098.445482:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.445484:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513098.445486:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513098.445491:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.445496:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513098.445499:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513098.445551:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.445558:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.445563:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.445570:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.445575:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be198 00000800:00000200:0.0:1616513098.445579:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.445582:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.445728:0:51426:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.445740:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.445748:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513098.445749:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.445755:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.445757:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.44@tcp1 : PUT - for me 00000800:00000200:14.0:1616513098.445757:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.445761:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.44@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.445771:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.44@tcp1 of length 224 into portal 26 MB=0x5ffd52a431600 00000400:00000200:2.0:1616513098.445771:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.44@tcp1 of length 224 into portal 12 MB=0x5ffd52a4316c0 00000400:00000200:2.0:1616513098.445775:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.44@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 42448 00000400:00000200:2.0:1616513098.445779:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.445782:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.44@tcp1 of length 224/224 into md 0xf97d07f9 [1] + 0 00000400:00000200:26.0:1616513098.445787:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922d078 00000400:00000200:26.0:1616513098.445789:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.445789:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513098.445793:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:26.0:1616513098.445794:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:2.0:1616513098.445795:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.445797:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.445798:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d078 00000800:00000200:26.0:1616513098.445801:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.445804:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.445806:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513098.445806:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665885710016 00000100:00000200:12.0:1616513098.445811:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665885709824 00010000:00000200:10.0:1616513098.445813:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dda00 x1688665885710016/t0(0) o400->06cdc5ef-d526-8926-57f3-95227d6c380d@10.31.138.44@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:12.0:1616513098.445816:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688665885709824/t0(0) o400->2892f2fb-e743-7755-0176-6d410445ce5d@10.31.138.44@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:10.0:1616513098.445825:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665885710016, offset 192 00000100:00000200:12.0:1616513098.445827:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665885709824, offset 192 00000400:00000200:10.0:1616513098.445830:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.44@tcp1 00000400:00000200:12.0:1616513098.445832:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.44@tcp1 00000400:00000200:10.0:1616513098.445836:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.44@tcp1 routed destination 00000400:00000200:12.0:1616513098.445838:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.44@tcp1 routed destination 00000400:00000200:10.0:1616513098.445850:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.44@tcp1(10.31.138.44@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:12.0:1616513098.445851:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.44@tcp1(10.31.138.44@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.445856:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513098.445860:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.445861:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.445863:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.445865:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.445865:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.445867:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513098.445868:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.445868:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:12.0:1616513098.445871:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513098.445872:0:51426:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513098.445875:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513098.445876:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513098.445884:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513098.445887:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.445892:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513098.445892:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:24.0:1616513098.445895:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513098.445895:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.445895:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513098.445897:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513098.445902:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.445902:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.445906:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82550 00000400:00000200:24.0:1616513098.445907:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be198 00000800:00000200:18.0:1616513098.445910:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513098.445911:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.452566:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.452579:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.452588:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513098.452591:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513098.452596:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.162@tcp1 : PUT - for me 00000800:00000200:16.0:1616513098.452601:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.452605:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.162@tcp1 of length 224 into portal 26 MB=0x5ffd474af3c40 00000400:00000200:16.0:1616513098.452610:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.162@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.452612:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.162@tcp1 of length 224/224 into md 0xf97d0819 [1] + 0 00000400:00000200:0.0:1616513098.452616:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922cc38 00000400:00000200:0.0:1616513098.452619:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.452620:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.162@tcp1 of length 224 into portal 12 MB=0x5ffd474af3d00 00000100:00000200:0.0:1616513098.452621:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513098.452625:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cc38 00000400:00000200:16.0:1616513098.452626:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.162@tcp1 of length 224/224 into md 0x865be291 [1] + 76240 00000800:00000200:0.0:1616513098.452628:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:16.0:1616513098.452631:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.452631:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513098.452632:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:16.0:1616513098.452634:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513098.452635:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.452641:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.452644:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513098.452644:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662839344192 00000800:00000200:16.0:1616513098.452646:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:12.0:1616513098.452654:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688662839344192/t0(0) o400->f11b9082-771c-f4f0-89d9-ebea821a717d@10.31.138.162@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.452668:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662839344192, offset 192 00000400:00000200:12.0:1616513098.452674:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.162@tcp1 00000400:00000200:12.0:1616513098.452681:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.162@tcp1 routed destination 00000400:00000200:12.0:1616513098.452695:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.162@tcp1(10.31.138.162@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.452701:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:28.0:1616513098.452705:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662839344384 00000800:00000200:12.0:1616513098.452706:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.452709:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.452711:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.452715:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00010000:00000200:28.0:1616513098.452717:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d0d80 x1688662839344384/t0(0) o400->2d83e85e-c3f5-06ba-4105-7a4c3297b32b@10.31.138.162@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513098.452722:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.452722:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513098.452731:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662839344384, offset 192 00000400:00000200:28.0:1616513098.452738:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.162@tcp1 00000400:00000200:28.0:1616513098.452745:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.162@tcp1 routed destination 00000400:00000200:28.0:1616513098.452760:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.162@tcp1(10.31.138.162@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.452767:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.452772:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.452775:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.452777:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513098.452778:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513098.452782:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:2.0:1616513098.452785:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:11.2:1616513098.452786:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513098.452788:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.452794:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513098.452798:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513098.452799:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfc28 00000800:00000200:2.0:1616513098.452802:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513098.452805:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513098.452805:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513098.452808:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.452809:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513098.452815:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.452819:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd8f8 00000800:00000200:24.0:1616513098.452822:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.453786:0:51427:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.453799:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.453808:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.453817:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.173@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.453827:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.173@tcp1 of length 224 into portal 26 MB=0x5ffde6bc103c0 00000400:00000200:14.0:1616513098.453834:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.173@tcp1 of length 224/224 into md 0xf97d0829 [1] + 0 00000400:00000200:14.0:1616513098.453838:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6baf68 00000400:00000200:14.0:1616513098.453841:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.453846:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.453851:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baf68 00000800:00000200:14.0:1616513098.453854:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.453857:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.453858:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.453861:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.453867:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.453867:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.453874:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.173@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.453880:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.173@tcp1 of length 224 into portal 12 MB=0x5ffde6bc10440 00000400:00000200:14.0:1616513098.453885:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.173@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 42672 00000400:00000200:14.0:1616513098.453889:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.453891:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513098.453896:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.453898:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.453900:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513098.453937:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705639187392 00010000:00000200:12.0:1616513098.453947:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688705639187392/t0(0) o400->1d199978-7f9e-c9cf-9509-838f6cd60402@10.31.132.173@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.453960:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705639187392, offset 192 00000400:00000200:12.0:1616513098.453965:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.173@tcp1 00000400:00000200:12.0:1616513098.453972:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.173@tcp1 routed destination 00000400:00000200:12.0:1616513098.453986:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.173@tcp1(10.31.132.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.453992:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:10.0:1616513098.453995:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705639187520 00000800:00000200:12.0:1616513098.453997:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.454000:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513098.454004:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:10.0:1616513098.454007:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebad00 x1688705639187520/t0(0) o400->5f2609d3-718e-63fd-8503-31a99a27cd8b@10.31.132.173@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:12.0:1616513098.454009:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.454013:0:51427:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513098.454015:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:10.0:1616513098.454021:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705639187520, offset 192 00000400:00000200:10.0:1616513098.454027:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.173@tcp1 00000400:00000200:10.0:1616513098.454035:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.173@tcp1 routed destination 00000400:00000200:10.0:1616513098.454050:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.173@tcp1(10.31.132.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.454056:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.454062:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.454064:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513098.454066:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513098.454066:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.454071:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513098.454073:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.454075:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.2:1616513098.454076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513098.454077:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.454082:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.454088:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be770 00000800:00000200:26.0:1616513098.454091:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513098.454129:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.454136:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.454139:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.454145:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.454149:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82bb0 00000800:00000200:20.0:1616513098.454153:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.454156:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.464353:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.464408:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.464417:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.464425:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.57@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.464445:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.57@tcp1 of length 224 into portal 26 MB=0x601953c6e9d00 00000400:00000200:0.0:1616513098.464452:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.57@tcp1 of length 224/224 into md 0xf97d0839 [1] + 0 00000400:00000200:0.0:1616513098.464456:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb3a8 00000400:00000200:0.0:1616513098.464459:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.464460:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513098.464462:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513098.464466:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb3a8 00000800:00000200:0.0:1616513098.464469:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513098.464471:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513098.464472:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513098.464473:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.464476:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513098.464480:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.178@tcp1 : PUT - for me 00000800:00000200:0.0:1616513098.464481:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.464490:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.178@tcp1 of length 224 into portal 26 MB=0x5ffe6505ddec0 00000400:00000200:0.0:1616513098.464492:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.178@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.464497:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.178@tcp1 of length 224/224 into md 0xf97d0859 [1] + 0 00000400:00000200:0.0:1616513098.464498:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.178@tcp1 of length 224 into portal 12 MB=0x5ffe6505ddf40 00000400:00000200:24.0:1616513098.464502:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb210 00000400:00000200:0.0:1616513098.464503:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.178@tcp1 of length 224/224 into md 0x865be291 [1] + 76464 00000400:00000200:24.0:1616513098.464504:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.464507:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513098.464507:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.464509:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513098.464513:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb210 00000800:00000200:0.0:1616513098.464514:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513098.464516:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513098.464516:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513098.464518:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513098.464519:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513098.464520:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.464522:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.464552:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590335900928 00000100:00000200:0.0:1616513098.464554:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739539443392 00010000:00000200:12.0:1616513098.464562:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1690590335900928/t0(0) o400->2f2fa441-a979-bf7a-761c-e6321abf82aa@10.31.141.57@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:0.0:1616513098.464564:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688739539443392/t0(0) o400->a04fd33a-9954-c3a2-e695-e2f27f766d7e@10.31.141.178@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513098.464576:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690590335900928, offset 192 00000100:00000200:0.0:1616513098.464576:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688739539443392, offset 192 00000100:00000200:28.0:1616513098.464581:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739539443520 00000400:00000200:12.0:1616513098.464582:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.57@tcp1 00000400:00000200:0.0:1616513098.464586:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.178@tcp1 00000400:00000200:12.0:1616513098.464588:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.57@tcp1 routed destination 00010000:00000200:28.0:1616513098.464593:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9f980 x1688739539443520/t0(0) o400->5a911e3b-f9ab-8514-2eb3-2b9a8d8a6f53@10.31.141.178@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513098.464596:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.178@tcp1 routed destination 00000400:00000200:12.0:1616513098.464602:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.57@tcp1(10.31.141.57@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513098.464608:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688739539443520, offset 192 00000800:00000200:12.0:1616513098.464608:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513098.464610:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.178@tcp1(10.31.141.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:28.0:1616513098.464614:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.178@tcp1 00000800:00000200:12.0:1616513098.464616:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513098.464616:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513098.464619:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513098.464621:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.178@tcp1 routed destination 00000800:00000200:12.0:1616513098.464621:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513098.464625:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513098.464629:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:12.0:1616513098.464631:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.464631:0:51428:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513098.464631:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.464633:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.464636:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513098.464637:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.178@tcp1(10.31.141.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513098.464641:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.464643:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:0.0:1616513098.464648:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513098.464649:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.464652:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513098.464655:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:0.0:1616513098.464656:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513098.464659:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:0.0:1616513098.464660:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.464661:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.464665:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.464666:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.464669:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cc38 00000800:00000200:0.0:1616513098.464672:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513098.464693:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513098.464699:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:2.0:1616513098.464702:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.464708:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.464713:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beff0 00000800:00000200:2.0:1616513098.464716:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.464719:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.464722:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.464723:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.464728:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.464731:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd298 00000800:00000200:2.0:1616513098.464735:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.464737:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513098.465338:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.465392:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.465402:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513098.465403:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.465411:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.57@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.465420:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.57@tcp1 of length 224 into portal 12 MB=0x601953c6ea880 00000400:00000200:26.0:1616513098.465426:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.57@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 42896 00000400:00000200:26.0:1616513098.465431:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.465454:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513098.465461:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.465464:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.465466:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513098.465523:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690590335903872 00010000:00000200:10.0:1616513098.465535:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21e300 x1690590335903872/t0(0) o400->6f2e69c8-2a15-454d-1d98-3d91b0ec28a4@10.31.141.57@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513098.465548:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690590335903872, offset 192 00000400:00000200:10.0:1616513098.465554:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.57@tcp1 00000400:00000200:10.0:1616513098.465560:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.57@tcp1 routed destination 00000400:00000200:10.0:1616513098.465576:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.57@tcp1(10.31.141.57@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.465582:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.465587:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.465590:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513098.465592:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.465602:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.465608:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.465620:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.465627:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.465629:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513098.465632:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513098.465636:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.465640:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83d38 00000800:00000200:20.0:1616513098.465644:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.516772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.516827:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.516837:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.516846:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.148@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.516855:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.148@tcp1 of length 224 into portal 12 MB=0x601ac1d2b2ac0 00000400:00000200:24.0:1616513098.516861:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.148@tcp1 of length 224/224 into md 0x865be291 [1] + 76688 00000400:00000200:24.0:1616513098.516867:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.516869:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513098.516877:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.516881:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.516882:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.516885:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.516893:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:28.0:1616513098.516943:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690688595634880 00000800:00000200:14.0:1616513098.516952:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00010000:00000200:28.0:1616513098.516956:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b089f80 x1690688595634880/t0(0) o400->e4e8a52e-fd82-26c5-6550-30a9ea490642@10.31.140.148@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513098.516962:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:28.0:1616513098.516970:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690688595634880, offset 192 00000400:00000200:14.0:1616513098.516971:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.148@tcp1 : PUT - for me 00000400:00000200:28.0:1616513098.516976:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.148@tcp1 00000400:00000200:14.0:1616513098.516981:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.148@tcp1 of length 224 into portal 26 MB=0x601ac1d2b2a40 00000400:00000200:28.0:1616513098.516983:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.148@tcp1 routed destination 00000400:00000200:14.0:1616513098.516988:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.148@tcp1 of length 224/224 into md 0xf97d0869 [1] + 0 00000400:00000200:14.0:1616513098.516993:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bbcb0 00000400:00000200:14.0:1616513098.516996:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:28.0:1616513098.516999:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.148@tcp1(10.31.140.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:14.0:1616513098.516999:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.517004:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbcb0 00000800:00000200:28.0:1616513098.517005:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:14.0:1616513098.517008:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.517009:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.517011:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:28.0:1616513098.517012:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513098.517013:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:28.0:1616513098.517014:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513098.517016:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513098.517021:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.517027:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:0.0:1616513098.517027:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690688595634752 00010000:00000200:0.0:1616513098.517048:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1690688595634752/t0(0) o400->58107f31-1d0b-a66e-c689-a92fdae27730@10.31.140.148@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:0.0:1616513098.517073:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690688595634752, offset 192 00000400:00000200:0.0:1616513098.517079:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.148@tcp1 00000400:00000200:0.0:1616513098.517087:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.148@tcp1 routed destination 00000800:00000200:2.0:1616513098.517097:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513098.517102:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.148@tcp1(10.31.140.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513098.517103:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.517105:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.517108:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513098.517111:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513098.517114:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:2.0:1616513098.517115:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd188 00000800:00000200:0.0:1616513098.517117:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513098.517119:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513098.517119:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513098.517121:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513098.517124:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.517128:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:0.0:1616513098.517131:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513098.517184:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.517192:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.517195:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.517203:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.517209:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cc38 00000800:00000200:16.0:1616513098.517213:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.517216:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.522077:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513098.522132:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.522142:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.522151:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.67@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.522161:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.67@tcp1 of length 224 into portal 12 MB=0x5ffed349cdc40 00000400:00000200:18.0:1616513098.522168:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.67@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 43120 00000400:00000200:18.0:1616513098.522173:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.522176:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513098.522184:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.522185:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.522187:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513098.522190:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513098.522204:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688769138580544 00010000:00000200:10.0:1616513098.522215:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464624380 x1688769138580544/t0(0) o400->26ea18ef-93a0-f19e-de6b-66b1ebf1f47f@10.31.142.67@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513098.522228:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688769138580544, offset 192 00000400:00000200:10.0:1616513098.522234:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.67@tcp1 00000400:00000200:10.0:1616513098.522241:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.67@tcp1 routed destination 00000400:00000200:10.0:1616513098.522255:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.67@tcp1(10.31.142.67@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.522261:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:10.0:1616513098.522266:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.522269:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513098.522271:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513098.522275:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.522281:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513098.522313:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513098.522334:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.522349:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.522352:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.522359:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.522363:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82770 00000800:00000200:2.0:1616513098.522366:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.522369:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513098.522370:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.522379:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.522388:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.67@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.522398:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.67@tcp1 of length 224 into portal 26 MB=0x5ffed349cdb80 00000400:00000200:0.0:1616513098.522404:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.67@tcp1 of length 224/224 into md 0xf97d0879 [1] + 0 00000400:00000200:0.0:1616513098.522409:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bad48 00000400:00000200:0.0:1616513098.522412:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.522415:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513098.522420:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bad48 00000800:00000200:0.0:1616513098.522423:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.522426:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.522427:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.522431:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513098.522483:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688769138580352 00010000:00000200:4.0:1616513098.522494:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688769138580352/t0(0) o400->3af62b3a-be3d-f86f-48c0-5e7df3a70a50@10.31.142.67@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513098.522509:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688769138580352, offset 192 00000400:00000200:4.0:1616513098.522515:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.67@tcp1 00000400:00000200:4.0:1616513098.522523:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.67@tcp1 routed destination 00000400:00000200:4.0:1616513098.522538:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.67@tcp1(10.31.142.67@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513098.522545:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513098.522552:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.522555:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513098.522557:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513098.522562:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.522569:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513098.522569:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:16.0:1616513098.522624:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.522631:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.522634:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.522640:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.522646:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba908 00000800:00000200:16.0:1616513098.522649:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.522652:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.535258:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.535314:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.535324:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.535333:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.156@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.535342:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.156@tcp1 of length 224 into portal 26 MB=0x5ffdc8bd11180 00000400:00000200:14.0:1616513098.535348:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.156@tcp1 of length 224/224 into md 0xf97d0889 [1] + 0 00000400:00000200:14.0:1616513098.535355:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bbb18 00000400:00000200:14.0:1616513098.535358:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.535361:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.535365:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbb18 00000800:00000200:18.0:1616513098.535368:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.535368:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.535371:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.535375:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.535378:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.535379:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.535387:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.156@tcp1 : PUT - for me 00000100:00000200:4.0:1616513098.535390:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697587175808 00000400:00000200:18.0:1616513098.535396:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.156@tcp1 of length 224 into portal 12 MB=0x5ffdc8bd11200 00010000:00000200:4.0:1616513098.535401:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688697587175808/t0(0) o400->e5c33d50-9ea0-ccaa-df0f-93aaac00778e@10.31.141.156@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513098.535402:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.156@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 43344 00000400:00000200:18.0:1616513098.535407:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.535410:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:4.0:1616513098.535415:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697587175808, offset 192 00000800:00000200:18.0:1616513098.535416:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.535419:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.535421:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513098.535421:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.156@tcp1 00000400:00000200:4.0:1616513098.535429:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.156@tcp1 routed destination 00000100:00000200:10.0:1616513098.535434:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697587175936 00010000:00000200:10.0:1616513098.535452:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464622400 x1688697587175936/t0(0) o400->ce5b5a93-c9e4-087b-44ee-78864047f552@10.31.141.156@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:4.0:1616513098.535459:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.156@tcp1(10.31.141.156@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513098.535465:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697587175936, offset 192 00000800:00000200:4.0:1616513098.535466:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:10.0:1616513098.535471:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.156@tcp1 00000800:00000200:4.0:1616513098.535472:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.535475:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513098.535477:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:10.0:1616513098.535478:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.156@tcp1 routed destination 00000800:00000200:4.0:1616513098.535482:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.535487:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513098.535489:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:10.0:1616513098.535491:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.156@tcp1(10.31.141.156@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.535497:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.535503:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.535505:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513098.535508:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.535513:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513098.535540:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.535548:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.535550:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.535551:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513098.535557:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513098.535558:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.535564:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbba0 00000800:00000200:26.0:1616513098.535568:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513098.535612:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.535619:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.535622:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.535624:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513098.535628:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.535632:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82d48 00000800:00000200:24.0:1616513098.535636:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.543118:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.543175:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.543184:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.543193:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.23@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.543202:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.23@tcp1 of length 224 into portal 12 MB=0x5ffdce33ea3c0 00000400:00000200:16.0:1616513098.543209:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.23@tcp1 of length 224/224 into md 0x865be291 [1] + 76912 00000400:00000200:16.0:1616513098.543214:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.543217:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513098.543224:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.543227:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.543229:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.543233:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.543289:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699053974464 00010000:00000200:28.0:1616513098.543301:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d3180 x1688699053974464/t0(0) o400->7070d657-4afd-fd2d-8f01-3441333e8ffb@10.31.138.23@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.543315:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699053974464, offset 192 00000400:00000200:28.0:1616513098.543322:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.23@tcp1 00000400:00000200:28.0:1616513098.543330:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.23@tcp1 routed destination 00000800:00000200:7.2:1616513098.543338:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513098.543345:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.23@tcp1(10.31.138.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.543351:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513098.543357:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.543360:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513098.543362:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:28.0:1616513098.543367:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.543393:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.543402:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.543411:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.23@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.543420:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.23@tcp1 of length 224 into portal 26 MB=0x5ffdce33ea340 00000400:00000200:2.0:1616513098.543427:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.23@tcp1 of length 224/224 into md 0xf97d08a9 [1] + 0 00000400:00000200:2.0:1616513098.543432:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bba90 00000800:00000200:18.0:1616513098.543448:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:2.0:1616513098.543451:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.543454:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513098.543455:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:18.0:1616513098.543458:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.543458:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bba90 00000800:00000200:2.0:1616513098.543461:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513098.543464:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:18.0:1616513098.543465:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:2.0:1616513098.543466:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.543469:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc000 00000800:00000200:2.0:1616513098.543469:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.543475:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513098.543478:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699053974336 00010000:00000200:4.0:1616513098.543488:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688699053974336/t0(0) o400->a21e121a-714e-e743-d9c3-c5ee59f27cac@10.31.138.23@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513098.543502:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699053974336, offset 192 00000400:00000200:4.0:1616513098.543507:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.23@tcp1 00000400:00000200:4.0:1616513098.543514:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.23@tcp1 routed destination 00000400:00000200:4.0:1616513098.543528:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.23@tcp1(10.31.138.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513098.543534:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513098.543539:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.543542:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513098.543544:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513098.543548:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.543553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513098.543556:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513098.543564:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.543568:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.543571:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.543577:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.543582:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb430 00000800:00000200:0.0:1616513098.543585:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.543587:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.556055:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.556111:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.556121:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513098.556130:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.174@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.556140:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.174@tcp1 of length 224 into portal 26 MB=0x5ffde559f4a40 00000400:00000200:20.0:1616513098.556147:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.174@tcp1 of length 224/224 into md 0xf97d08c9 [1] + 0 00000400:00000200:20.0:1616513098.556152:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb760 00000400:00000200:20.0:1616513098.556155:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.556157:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513098.556162:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb760 00000800:00000200:20.0:1616513098.556165:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.556165:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.556168:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.556170:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.556203:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:4.0:1616513098.556222:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705267878464 00010000:00000200:4.0:1616513098.556233:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688705267878464/t0(0) o400->0d6fc176-9794-bc66-d7d9-e100accbbfad@10.31.136.174@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513098.556256:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:4.0:1616513098.556262:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705267878464, offset 192 00000800:00000200:0.0:1616513098.556267:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513098.556268:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.174@tcp1 00000400:00000200:0.0:1616513098.556275:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.174@tcp1 : PUT - for me 00000400:00000200:4.0:1616513098.556276:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.174@tcp1 routed destination 00000400:00000200:0.0:1616513098.556285:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.174@tcp1 of length 224 into portal 12 MB=0x5ffde559f4bc0 00000400:00000200:4.0:1616513098.556290:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.174@tcp1(10.31.136.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513098.556292:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.174@tcp1 of length 224/224 into md 0x865be291 [1] + 77136 00000800:00000200:4.0:1616513098.556296:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:0.0:1616513098.556296:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.556299:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513098.556302:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.556304:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513098.556305:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513098.556306:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513098.556308:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.556310:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513098.556311:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513098.556312:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.556317:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513098.556317:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513098.556368:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705267878848 00000800:00000200:26.0:1616513098.556372:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.556379:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00010000:00000200:28.0:1616513098.556381:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9d100 x1688705267878848/t0(0) o400->97f46e42-79e0-5bf3-3f8e-965c096de168@10.31.136.174@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:26.0:1616513098.556381:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.556388:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.556394:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba2a8 00000100:00000200:28.0:1616513098.556397:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705267878848, offset 192 00000800:00000200:26.0:1616513098.556397:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.556400:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513098.556403:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.174@tcp1 00000400:00000200:28.0:1616513098.556411:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.174@tcp1 routed destination 00000400:00000200:28.0:1616513098.556426:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.174@tcp1(10.31.136.174@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.556432:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.556465:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.556467:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.556471:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.556476:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.556481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.556491:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.556496:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.556499:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.556501:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.556506:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.556510:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccf68 00000800:00000200:16.0:1616513098.556513:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.586067:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.586124:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.586133:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.586143:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.215@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.586152:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.215@tcp1 of length 224 into portal 26 MB=0x5ffe46fe83600 00000400:00000200:0.0:1616513098.586159:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.215@tcp1 of length 224/224 into md 0xf97d08e9 [1] + 0 00000400:00000200:0.0:1616513098.586163:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba088 00000400:00000200:0.0:1616513098.586166:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.586169:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513098.586173:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba088 00000800:00000200:24.0:1616513098.586177:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.586177:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.586179:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.586184:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.586187:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.586188:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.586197:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.215@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.586205:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.215@tcp1 of length 224 into portal 12 MB=0x5ffe46fe83900 00000400:00000200:24.0:1616513098.586212:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.215@tcp1 of length 224/224 into md 0x865be291 [1] + 77360 00000400:00000200:24.0:1616513098.586217:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.586219:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513098.586226:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.586229:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.586231:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513098.586237:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731478668800 00010000:00000200:4.0:1616513098.586249:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1688731478668800/t0(0) o400->282e6f85-35f1-7de6-6bed-cc537fbd65e3@10.31.141.215@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513098.586263:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688731478668800, offset 192 00000400:00000200:4.0:1616513098.586270:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.215@tcp1 00000400:00000200:4.0:1616513098.586277:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.215@tcp1 routed destination 00000400:00000200:4.0:1616513098.586292:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.215@tcp1(10.31.141.215@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513098.586293:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731478669568 00000800:00000200:4.0:1616513098.586298:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00010000:00000200:28.0:1616513098.586305:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b088480 x1688731478669568/t0(0) o400->536322f1-abab-46d9-5e31-68a55cef62d5@10.31.141.215@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:4.0:1616513098.586305:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.586307:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513098.586310:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513098.586314:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:28.0:1616513098.586320:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688731478669568, offset 192 00000800:00000200:7.2:1616513098.586321:0:51442:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513098.586321:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513098.586326:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.215@tcp1 00000400:00000200:28.0:1616513098.586333:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.215@tcp1 routed destination 00000400:00000200:28.0:1616513098.586348:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.215@tcp1(10.31.141.215@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.586355:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.586361:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.586364:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.586366:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.586370:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513098.586375:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513098.586375:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513098.586382:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.586385:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513098.586391:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.586397:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb540 00000800:00000200:18.0:1616513098.586400:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.586403:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513098.586429:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.586443:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.586450:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.586453:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.586460:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.586464:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcce58 00000800:00000200:16.0:1616513098.586468:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513098.586718:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929584, portal 4 00000100:00000200:30.0:1616513098.586724:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929584, offset 0 00000400:00000200:30.0:1616513098.586729:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:30.0:1616513098.586735:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:30.0:1616513098.586746:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513098.586751:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:30.0:1616513098.586757:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513098.586760:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:30.0:1616513098.586761:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:30.0:1616513098.586766:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513098.586813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.586834:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513098.586841:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513098.586844:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513098.586852:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513098.586861:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c8a2b600 x1693067725929584/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513098.586874:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef540 00000800:00000200:1.0:1616513098.586880:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513098.586883:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513098.586990:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513098.587000:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.587007:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000400:00000200:1.0:1616513098.587016:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513098.587025:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627c70 00000400:00000200:1.0:1616513098.587033:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x437002f5 [1] + 192 00000400:00000200:1.0:1616513098.587039:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513098.587043:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c8a2b600 x1693067725929584/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:25.0:1616513098.587052:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513098.587058:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513098.587061:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513098.587063:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:30.0:1616513098.587064:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c8a2b600 x1693067725929584/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513098.587074:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c8a2b600 x1693067725929584/t0(0) o13->boslfs02-OST0037-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513098.587083:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:7.2:1616513098.591274:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.591334:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.591343:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.591352:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.121@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.591362:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.121@tcp1 of length 224 into portal 26 MB=0x6012da87719c0 00000400:00000200:26.0:1616513098.591368:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.121@tcp1 of length 224/224 into md 0xf97d08f9 [1] + 0 00000400:00000200:26.0:1616513098.591373:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6badd0 00000400:00000200:26.0:1616513098.591376:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.591379:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513098.591383:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6badd0 00000800:00000200:26.0:1616513098.591386:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.591388:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.591389:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.591392:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513098.591458:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690145471797696 00010000:00000200:4.0:1616513098.591469:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1690145471797696/t0(0) o400->1a9a8190-d2a8-64c1-d4f9-d6857b868920@10.31.140.121@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513098.591483:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690145471797696, offset 192 00000400:00000200:4.0:1616513098.591491:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.121@tcp1 00000400:00000200:4.0:1616513098.591499:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.121@tcp1 routed destination 00000400:00000200:4.0:1616513098.591514:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.121@tcp1(10.31.140.121@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513098.591521:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513098.591526:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.591529:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513098.591531:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513098.591536:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.591542:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513098.591543:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513098.591594:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.591601:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.591603:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.591606:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513098.591610:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.591616:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb320 00000800:00000200:18.0:1616513098.591619:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513098.592183:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.592195:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.592203:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513098.592205:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.592212:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.121@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.592220:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.121@tcp1 of length 224 into portal 12 MB=0x6012da87720c0 00000400:00000200:2.0:1616513098.592228:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.121@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 43568 00000400:00000200:2.0:1616513098.592234:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.592236:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513098.592243:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.592246:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.592248:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513098.592260:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690145471799488 00010000:00000200:10.0:1616513098.592270:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dc380 x1690145471799488/t0(0) o400->6753570c-c4b2-d8e3-40db-fba70f8bfba1@10.31.140.121@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:10.0:1616513098.592282:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690145471799488, offset 192 00000400:00000200:10.0:1616513098.592288:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.121@tcp1 00000400:00000200:10.0:1616513098.592294:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.121@tcp1 routed destination 00000400:00000200:10.0:1616513098.592307:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.121@tcp1(10.31.140.121@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513098.592312:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513098.592317:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513098.592320:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:10.0:1616513098.592322:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:10.0:1616513098.592326:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.592372:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.592428:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.592445:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.592448:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.592455:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.592459:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83078 00000800:00000200:24.0:1616513098.592463:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.592466:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.603656:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.603671:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.603680:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.603688:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.23@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.603698:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.23@tcp1 of length 224 into portal 26 MB=0x5ffd3c2fa9740 00000400:00000200:14.0:1616513098.603704:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.23@tcp1 of length 224/224 into md 0xf97d0919 [1] + 0 00000400:00000200:14.0:1616513098.603709:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb4b8 00000400:00000200:14.0:1616513098.603711:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.603714:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.603719:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb4b8 00000800:00000200:14.0:1616513098.603722:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.603724:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.603725:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.603728:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513098.603782:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659857938240 00010000:00000200:4.0:1616513098.603793:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688659857938240/t0(0) o400->5fdf63cb-68b9-ff94-ca26-aa670b140420@10.31.136.23@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513098.603807:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688659857938240, offset 192 00000400:00000200:4.0:1616513098.603813:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.23@tcp1 00000400:00000200:4.0:1616513098.603821:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.23@tcp1 routed destination 00000400:00000200:4.0:1616513098.603836:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.23@tcp1(10.31.136.23@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513098.603842:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513098.603848:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.603851:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513098.603853:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513098.603858:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.603864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513098.603864:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513098.603918:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.603924:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.603927:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.603934:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.603939:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb5c8 00000800:00000200:26.0:1616513098.603943:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.603946:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513098.604300:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513098.604310:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.604318:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513098.604320:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513098.604328:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.23@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.604336:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.23@tcp1 of length 224 into portal 12 MB=0x5ffd3c2fa9900 00000400:00000200:18.0:1616513098.604342:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.23@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 43792 00000400:00000200:18.0:1616513098.604348:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.604351:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513098.604358:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.604361:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.604363:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:14.0:1616513098.604377:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688659857938688 00010000:00000200:14.0:1616513098.604386:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464627980 x1688659857938688/t0(0) o400->c5a503fe-eb71-4140-05a4-3987026c82d9@10.31.136.23@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:14.0:1616513098.604399:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688659857938688, offset 192 00000400:00000200:14.0:1616513098.604404:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.23@tcp1 00000400:00000200:14.0:1616513098.604410:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.23@tcp1 routed destination 00000400:00000200:14.0:1616513098.604423:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.23@tcp1(10.31.136.23@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513098.604429:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:14.0:1616513098.604434:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:14.0:1616513098.604439:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513098.604448:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513098.604452:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.604498:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.604520:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.604527:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.604530:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.604536:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.604540:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4756440 00000800:00000200:0.0:1616513098.604544:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.604547:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.625082:0:51446:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.625148:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.625167:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.625175:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.56@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.625183:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.56@tcp1 of length 224 into portal 26 MB=0x5ffdcccdd0d00 00000400:00000200:26.0:1616513098.625191:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.56@tcp1 of length 224/224 into md 0xf97d0929 [1] + 0 00000400:00000200:26.0:1616513098.625196:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba770 00000400:00000200:26.0:1616513098.625199:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.625201:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513098.625205:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba770 00000800:00000200:26.0:1616513098.625208:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.625211:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.625213:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.625220:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.625223:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.625223:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.625232:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.56@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.625240:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.56@tcp1 of length 224 into portal 12 MB=0x5ffdcccdd0dc0 00000400:00000200:2.0:1616513098.625247:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.56@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 44016 00000400:00000200:2.0:1616513098.625252:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.625255:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513098.625262:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.625265:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513098.625267:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698678480128 00000800:00000200:2.0:1616513098.625267:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:4.0:1616513098.625277:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688698678480128/t0(0) o400->b92a324d-08ef-d084-0103-19682218c73e@10.31.141.56@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513098.625291:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698678480128, offset 192 00000400:00000200:4.0:1616513098.625296:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.56@tcp1 00000400:00000200:4.0:1616513098.625303:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.56@tcp1 routed destination 00000400:00000200:4.0:1616513098.625317:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.56@tcp1(10.31.141.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513098.625323:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:18.0:1616513098.625327:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698678480320 00000800:00000200:4.0:1616513098.625330:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.625333:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513098.625335:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00010000:00000200:18.0:1616513098.625339:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8000 x1688698678480320/t0(0) o400->a14b4588-1f05-bb6d-8f2e-1ddd8ca801e1@10.31.141.56@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:4.0:1616513098.625340:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.625344:0:51446:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:4.0:1616513098.625346:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:18.0:1616513098.625359:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698678480320, offset 192 00000400:00000200:18.0:1616513098.625365:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.56@tcp1 00000400:00000200:18.0:1616513098.625373:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.56@tcp1 routed destination 00000400:00000200:18.0:1616513098.625388:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.56@tcp1(10.31.141.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513098.625394:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513098.625402:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:18.0:1616513098.625404:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513098.625406:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513098.625411:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513098.625424:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.625428:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.625430:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513098.625432:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513098.625448:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.625453:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba440 00000800:00000200:11.2:1616513098.625453:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513098.625456:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513098.625511:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.625518:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.625521:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.625528:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.625531:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d747a6cc000 00000800:00000200:24.0:1616513098.625535:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.625538:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.625546:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513098.625548:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.625550:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.625552:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.625554:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513098.634580:0:6484:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929600, portal 4 00000100:00000200:24.0:1616513098.634585:0:6484:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929600, offset 0 00000400:00000200:24.0:1616513098.634592:0:6484:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:24.0:1616513098.634598:0:6484:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:24.0:1616513098.634610:0:6484:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513098.634617:0:6484:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:24.0:1616513098.634623:0:6484:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513098.634626:0:6484:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:24.0:1616513098.634628:0:6484:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:24.0:1616513098.634633:0:6484:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513098.634685:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513098.634742:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513098.634749:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513098.634752:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.634759:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513098.634765:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cc6b9a400 x1693067725929600/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513098.634777:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3ed0 00000800:00000200:2.0:1616513098.634781:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513098.634784:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513098.634857:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513098.634914:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513098.634924:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:26.0:1616513098.634932:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513098.634940:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627c80 00000400:00000200:26.0:1616513098.634947:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d0f29 [1] + 192 00000400:00000200:26.0:1616513098.634952:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.634957:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cc6b9a400 x1693067725929600/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513098.634968:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513098.634972:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513098.634975:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513098.634977:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:24.0:1616513098.634980:0:6484:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cc6b9a400 x1693067725929600/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:24.0:1616513098.634993:0:6484:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cc6b9a400 x1693067725929600/t0(0) o13->boslfs02-OST0000-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:24.0:1616513098.635003:0:6484:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3980 00000800:00000200:7.2:1616513098.639447:0:51447:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.639504:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.639514:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.639522:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.208@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.639531:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.208@tcp1 of length 224 into portal 26 MB=0x60182c2b01640 00000400:00000200:2.0:1616513098.639538:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.208@tcp1 of length 224/224 into md 0xf97d0939 [1] + 0 00000400:00000200:2.0:1616513098.639543:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb7e8 00000400:00000200:2.0:1616513098.639548:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.639550:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.639555:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb7e8 00000800:00000200:14.0:1616513098.639558:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.639558:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.639561:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.639562:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513098.639578:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510983960128 00010000:00000200:4.0:1616513098.639589:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1690510983960128/t0(0) o400->82e2c496-b6dc-8f3a-8351-f8e7773f8735@10.31.140.208@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513098.639604:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510983960128, offset 192 00000400:00000200:4.0:1616513098.639610:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.208@tcp1 00000400:00000200:4.0:1616513098.639617:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.208@tcp1 routed destination 00000800:00000200:7.2:1616513098.639624:0:51447:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:4.0:1616513098.639633:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.208@tcp1(10.31.140.208@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513098.639639:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513098.639645:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.639648:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513098.639650:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513098.639655:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:4.0:1616513098.639661:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513098.639678:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.639688:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.639696:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.208@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.639705:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.208@tcp1 of length 224 into portal 12 MB=0x60182c2b01a80 00000400:00000200:18.0:1616513098.639711:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.208@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 44240 00000400:00000200:18.0:1616513098.639716:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.639719:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513098.639726:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513098.639730:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513098.639731:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.639734:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.639742:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.639745:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.639746:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.639752:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.639763:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba990 00000800:00000200:26.0:1616513098.639767:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513098.639800:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510983961216 00010000:00000200:22.0:1616513098.639812:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621680 x1690510983961216/t0(0) o400->e810b75a-4c89-d8b6-2238-41f0a54b4c40@10.31.140.208@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.639826:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510983961216, offset 192 00000400:00000200:22.0:1616513098.639833:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.208@tcp1 00000400:00000200:22.0:1616513098.639840:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.208@tcp1 routed destination 00000400:00000200:22.0:1616513098.639855:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.208@tcp1(10.31.140.208@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.639862:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.639867:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.639872:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.639874:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.639879:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.639922:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.639978:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.639985:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.639988:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.639994:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.639998:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a990 00000800:00000200:16.0:1616513098.640003:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.640006:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.640558:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.640615:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.640625:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.640636:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.122@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.640647:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.122@tcp1 of length 224 into portal 26 MB=0x5ffd4c576f5c0 00000400:00000200:0.0:1616513098.640654:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.122@tcp1 of length 224/224 into md 0xf97d0949 [1] + 0 00000400:00000200:0.0:1616513098.640659:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6baee0 00000400:00000200:0.0:1616513098.640661:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.640664:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513098.640669:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.640669:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6baee0 00000800:00000200:0.0:1616513098.640672:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.640675:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.640677:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:4.0:1616513098.640679:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664194610624 00010000:00000200:4.0:1616513098.640685:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688664194610624/t0(0) o400->53b212cb-be48-5c47-ca4e-ce74e1fa93d2@10.31.132.122@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:4.0:1616513098.640696:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664194610624, offset 192 00000400:00000200:4.0:1616513098.640701:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.122@tcp1 00000400:00000200:4.0:1616513098.640707:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.122@tcp1 routed destination 00000400:00000200:4.0:1616513098.640720:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.122@tcp1(10.31.132.122@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513098.640726:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513098.640730:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513098.640733:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513098.640734:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513098.640737:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:4.0:1616513098.640743:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.640744:0:51447:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.640799:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.640817:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.640819:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513098.640821:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.640825:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.640830:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb980 00000800:00000200:2.0:1616513098.640834:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.641390:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.641455:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.641464:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.641472:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.122@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.641482:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.122@tcp1 of length 224 into portal 12 MB=0x5ffd4c576f700 00000400:00000200:16.0:1616513098.641488:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.122@tcp1 of length 224/224 into md 0x865be291 [1] + 77584 00000400:00000200:16.0:1616513098.641493:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.641496:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513098.641503:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.641506:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.641508:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.641512:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.641568:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664194610944 00010000:00000200:28.0:1616513098.641580:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d2400 x1688664194610944/t0(0) o400->8225a013-3bb4-9062-3c44-b802c5701c41@10.31.132.122@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.641595:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664194610944, offset 192 00000400:00000200:28.0:1616513098.641601:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.122@tcp1 00000400:00000200:28.0:1616513098.641609:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.122@tcp1 routed destination 00000400:00000200:28.0:1616513098.641624:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.122@tcp1(10.31.132.122@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.641630:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.641636:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.641638:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.641641:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.641645:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.641651:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.641662:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.641668:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.641670:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.641673:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513098.641677:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.641683:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd078 00000800:00000200:20.0:1616513098.641687:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.647451:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513098.647495:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513098.647504:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513098.647509:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513098.647513:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:14.0:1616513098.647518:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513098.647521:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.50@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.647527:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.50@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.647530:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.50@tcp1 of length 224 into portal 26 MB=0x5ffd470605200 00000400:00000200:24.0:1616513098.647536:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.50@tcp1 of length 224/224 into md 0xf97d0979 [1] + 0 00000400:00000200:14.0:1616513098.647537:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.50@tcp1 of length 224 into portal 12 MB=0x5ffd470605400 00000400:00000200:24.0:1616513098.647541:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba330 00000400:00000200:24.0:1616513098.647543:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:14.0:1616513098.647546:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.50@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 44464 00000100:00000200:24.0:1616513098.647551:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.647551:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.647556:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba330 00000800:00000200:16.0:1616513098.647558:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:14.0:1616513098.647558:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513098.647559:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.647562:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.647563:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513098.647565:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.647567:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.647569:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.647572:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513098.647573:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662767063552 00010000:00000200:8.0:1616513098.647581:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688662767063552/t0(0) o400->64c2ac63-e432-4285-062a-0252b9f1e24d@10.31.136.50@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.647593:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662767063552, offset 192 00000400:00000200:8.0:1616513098.647598:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.50@tcp1 00000400:00000200:8.0:1616513098.647604:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.50@tcp1 routed destination 00000400:00000200:8.0:1616513098.647615:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.50@tcp1(10.31.136.50@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.647620:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.647624:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.647627:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000100:00000200:22.0:1616513098.647628:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662767064064 00000800:00000200:8.0:1616513098.647628:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.647632:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.647637:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.647637:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00010000:00000200:22.0:1616513098.647641:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebc800 x1688662767064064/t0(0) o400->1f89d7ca-bb20-c3d1-ebba-3f5a93a07007@10.31.136.50@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.647655:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662767064064, offset 192 00000400:00000200:22.0:1616513098.647662:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.50@tcp1 00000400:00000200:22.0:1616513098.647669:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.50@tcp1 routed destination 00000400:00000200:22.0:1616513098.647684:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.50@tcp1(10.31.136.50@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.647691:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:18.0:1616513098.647693:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513098.647696:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.647699:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513098.647699:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513098.647701:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513098.647702:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513098.647706:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513098.647708:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:11.2:1616513098.647711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513098.647713:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:18.0:1616513098.647716:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.647719:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513098.647766:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.647772:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.647775:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.647781:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.647785:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a330 00000800:00000200:0.0:1616513098.647789:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.647792:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513098.650511:0:6485:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929616, portal 4 00000100:00000200:12.0:1616513098.650516:0:6485:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929616, offset 0 00000400:00000200:12.0:1616513098.650523:0:6485:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:12.0:1616513098.650529:0:6485:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:12.0:1616513098.650542:0:6485:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513098.650549:0:6485:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:12.0:1616513098.650555:0:6485:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513098.650559:0:6485:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:12.0:1616513098.650561:0:6485:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:12.0:1616513098.650566:0:6485:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513098.650614:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513098.650635:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513098.650642:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513098.650645:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.0:1616513098.650646:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513098.650652:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513098.650660:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdeb8d580 x1693067725929616/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513098.650672:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfa08 00000800:00000200:27.0:1616513098.650677:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513098.650801:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513098.650813:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513098.650819:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:11.0:1616513098.650820:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513098.650827:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513098.650836:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627c90 00000400:00000200:27.0:1616513098.650843:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716763d [1] + 192 00000400:00000200:27.0:1616513098.650848:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513098.650853:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdeb8d580 x1693067725929616/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513098.650877:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513098.650880:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513098.650882:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:12.0:1616513098.650887:0:6485:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdeb8d580 x1693067725929616/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:12.0:1616513098.650900:0:6485:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdeb8d580 x1693067725929616/t0(0) o13->boslfs02-OST002a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:12.0:1616513098.650911:0:6485:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf298 00000100:00000200:1.0:1616513098.667925:0:6493:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929632, portal 4 00000100:00000200:1.0:1616513098.667932:0:6493:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929632, offset 0 00000400:00000200:1.0:1616513098.667939:0:6493:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:1.0:1616513098.667946:0:6493:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:1.0:1616513098.667959:0:6493:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:1.0:1616513098.667966:0:6493:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:1.0:1616513098.667973:0:6493:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:1.0:1616513098.667976:0:6493:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513098.667979:0:6493:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.667984:0:6493:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513098.667991:0:6493:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:5.0:1616513098.668044:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:5.0:1616513098.668052:0:6467:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:5.0:1616513098.668054:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:1.0:1616513098.668057:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:5.0:1616513098.668060:0:6467:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:5.0:1616513098.668066:0:6467:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1393180 x1693067725929632/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513098.668078:0:6467:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84ff0 00000800:00000200:5.0:1616513098.668082:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513098.668205:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513098.668214:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.668221:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513098.668223:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513098.668229:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513098.668236:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627ca0 00000400:00000200:1.0:1616513098.668242:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700305 [1] + 192 00000400:00000200:1.0:1616513098.668247:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513098.668252:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1393180 x1693067725929632/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513098.668266:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.668268:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513098.668270:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:5.0:1616513098.668273:0:6493:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1393180 x1693067725929632/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:5.0:1616513098.668283:0:6493:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1393180 x1693067725929632/t0(0) o13->boslfs02-OST003b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:5.0:1616513098.668292:0:6493:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8efbe84b28 00000800:00000200:11.2:1616513098.668879:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.668935:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.668944:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.668954:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.146@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.668963:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.146@tcp1 of length 224 into portal 26 MB=0x5ffd713c7a8c0 00000400:00000200:16.0:1616513098.668970:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.146@tcp1 of length 224/224 into md 0xf97d0989 [1] + 0 00000400:00000200:16.0:1616513098.668976:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c17075c8 00000400:00000200:16.0:1616513098.668978:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.668981:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513098.668986:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17075c8 00000800:00000200:24.0:1616513098.668988:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.668989:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.668992:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.668994:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513098.669000:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674098456768 00010000:00000200:8.0:1616513098.669007:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688674098456768/t0(0) o400->57694c40-5c60-0b87-bd14-8ddff6a20c1e@10.31.131.146@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.669019:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688674098456768, offset 192 00000400:00000200:8.0:1616513098.669024:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.146@tcp1 00000400:00000200:8.0:1616513098.669030:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.146@tcp1 routed destination 00000400:00000200:8.0:1616513098.669043:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.146@tcp1(10.31.131.146@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.669049:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.669054:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.669056:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.669058:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.669062:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.669067:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.669069:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.669126:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.669133:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.669135:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513098.669142:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.669148:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:14.0:1616513098.669152:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.669155:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.669161:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.669169:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.146@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.669177:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.146@tcp1 of length 224 into portal 12 MB=0x5ffd713c7ad80 00000800:00000200:26.0:1616513098.669179:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513098.669183:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.146@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 44688 00000400:00000200:14.0:1616513098.669188:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.669191:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513098.669197:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.669200:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.669202:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513098.669262:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688674098457984 00010000:00000200:22.0:1616513098.669274:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebec00 x1688674098457984/t0(0) o400->ee69d52d-cf2e-fc5c-f932-f392dc584489@10.31.131.146@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.669289:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688674098457984, offset 192 00000400:00000200:22.0:1616513098.669295:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.146@tcp1 00000400:00000200:22.0:1616513098.669302:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.146@tcp1 routed destination 00000400:00000200:22.0:1616513098.669317:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.146@tcp1(10.31.131.146@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.669323:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.669329:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.669332:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.669334:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.669338:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.669344:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.669399:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.669406:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.669409:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.669416:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.669420:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b540 00000800:00000200:0.0:1616513098.669424:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.669426:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:3.0:1616513098.682615:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929648, portal 4 00000100:00000200:3.0:1616513098.682620:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929648, offset 0 00000400:00000200:3.0:1616513098.682627:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:3.0:1616513098.682633:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:3.0:1616513098.682645:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513098.682652:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:3.0:1616513098.682658:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513098.682661:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:3.0:1616513098.682663:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:3.0:1616513098.682668:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513098.682714:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.682731:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513098.682737:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513098.682739:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513098.682746:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513098.682753:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be42400 x1693067725929648/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513098.682765:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2f68 00000800:00000200:1.0:1616513098.682769:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513098.682771:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:7.2:1616513098.682885:0:51451:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:1.2:1616513098.682885:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513098.682894:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.682902:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513098.682905:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513098.682910:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513098.682918:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627cb0 00000400:00000200:1.0:1616513098.682925:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700325 [1] + 192 00000400:00000200:1.0:1616513098.682930:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513098.682934:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be42400 x1693067725929648/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513098.682942:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:1.0:1616513098.682947:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.682950:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:2.0:1616513098.682952:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:1.0:1616513098.682952:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:3.0:1616513098.682956:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be42400 x1693067725929648/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513098.682961:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.38@tcp1 : PUT - for me 00000100:00000200:3.0:1616513098.682970:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be42400 x1693067725929648/t0(0) o13->boslfs02-OST002f-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513098.682972:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.38@tcp1 of length 224 into portal 26 MB=0x5ffd52791bf00 00000400:00000200:2.0:1616513098.682978:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.38@tcp1 of length 224/224 into md 0xf97d09b9 [1] + 0 00000400:00000200:3.0:1616513098.682980:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2a18 00000400:00000200:2.0:1616513098.682983:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707f58 00000400:00000200:2.0:1616513098.682986:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.682989:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.682993:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707f58 00000800:00000200:2.0:1616513098.682996:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.682998:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.682999:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.683004:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:8.0:1616513098.683005:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665840533248 00000800:00000200:26.0:1616513098.683008:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513098.683008:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:8.0:1616513098.683011:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688665840533248/t0(0) o400->09c29f0e-6cff-0dd0-0182-a2428a88023c@10.31.136.38@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513098.683017:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.38@tcp1 : PUT - for me 00000100:00000200:8.0:1616513098.683021:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665840533248, offset 192 00000400:00000200:26.0:1616513098.683026:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.38@tcp1 of length 224 into portal 12 MB=0x5ffd52791bf80 00000400:00000200:8.0:1616513098.683026:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.38@tcp1 00000400:00000200:8.0:1616513098.683032:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.38@tcp1 routed destination 00000400:00000200:26.0:1616513098.683033:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.38@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 44912 00000400:00000200:26.0:1616513098.683041:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.683044:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513098.683046:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.38@tcp1(10.31.136.38@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513098.683051:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.683052:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513098.683054:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.683056:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513098.683056:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.683059:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.683061:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.683064:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.683068:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.683068:0:51451:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.683074:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.683091:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.683094:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513098.683095:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513098.683100:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.683106:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:14.0:1616513098.683110:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513098.683119:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665840533376 00010000:00000200:22.0:1616513098.683131:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21f080 x1688665840533376/t0(0) o400->bf7372e2-8c3e-3888-a400-12e651929b40@10.31.136.38@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.683157:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665840533376, offset 192 00000400:00000200:22.0:1616513098.683163:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.38@tcp1 00000400:00000200:22.0:1616513098.683170:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.38@tcp1 routed destination 00000400:00000200:22.0:1616513098.683185:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.38@tcp1(10.31.136.38@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.683192:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.683197:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.683200:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.683202:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.683207:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.683214:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.683269:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.683276:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.683279:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.683285:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.683289:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a198 00000800:00000200:24.0:1616513098.683293:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.683296:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.684937:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.684992:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.685001:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.685010:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.161@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.685018:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.161@tcp1 of length 224 into portal 26 MB=0x60216e008e140 00000400:00000200:26.0:1616513098.685025:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.161@tcp1 of length 224/224 into md 0xf97d09d9 [1] + 0 00000400:00000200:26.0:1616513098.685030:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1706550 00000400:00000200:26.0:1616513098.685032:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.685035:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513098.685039:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706550 00000800:00000200:26.0:1616513098.685043:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.685045:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.685047:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000100:00000200:8.0:1616513098.685049:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691147131478336 00000800:00000200:26.0:1616513098.685054:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00010000:00000200:8.0:1616513098.685054:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1691147131478336/t0(0) o400->52019eb0-93a9-dff7-58cb-50fa97bdac0b@10.31.140.161@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513098.685057:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.685058:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513098.685063:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691147131478336, offset 192 00000400:00000200:2.0:1616513098.685066:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.161@tcp1 : PUT - for me 00000400:00000200:8.0:1616513098.685068:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.161@tcp1 00000400:00000200:8.0:1616513098.685074:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.161@tcp1 routed destination 00000400:00000200:2.0:1616513098.685076:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.161@tcp1 of length 224 into portal 12 MB=0x60216e008e200 00000400:00000200:2.0:1616513098.685082:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.161@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 45136 00000400:00000200:8.0:1616513098.685087:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.161@tcp1(10.31.140.161@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513098.685087:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.685090:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513098.685092:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.685096:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.685098:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.685098:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.685100:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:8.0:1616513098.685103:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513098.685103:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513098.685105:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.685107:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.685107:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.685167:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:22.0:1616513098.685168:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691147131478528 00000800:00000200:18.0:1616513098.685174:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.685177:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513098.685178:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:22.0:1616513098.685180:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647db180 x1691147131478528/t0(0) o400->7890681e-8c96-740d-a5de-919fbcdc2cb9@10.31.140.161@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:18.0:1616513098.685184:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.685189:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000100:00000200:22.0:1616513098.685194:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691147131478528, offset 192 00000800:00000200:18.0:1616513098.685195:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:22.0:1616513098.685201:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.161@tcp1 00000400:00000200:22.0:1616513098.685208:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.161@tcp1 routed destination 00000400:00000200:22.0:1616513098.685233:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.161@tcp1(10.31.140.161@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.685239:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.685245:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.685247:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.685249:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.685253:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.685260:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.685315:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.685322:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.685325:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.685331:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.685335:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b7e8 00000800:00000200:0.0:1616513098.685339:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.685342:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513098.714709:0:6486:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929664, portal 4 00000100:00000200:8.0:1616513098.714713:0:6486:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929664, offset 0 00000400:00000200:8.0:1616513098.714719:0:6486:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:8.0:1616513098.714725:0:6486:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:8.0:1616513098.714738:0:6486:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.714744:0:6486:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:8.0:1616513098.714750:0:6486:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.714753:0:6486:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000100:00000200:0.0:1616513098.714753:0:6487:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929680, portal 4 00000800:00000200:8.0:1616513098.714755:0:6486:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:0.0:1616513098.714758:0:6487:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929680, offset 0 00000800:00000200:8.0:1616513098.714762:0:6486:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:0.0:1616513098.714767:0:6487:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:0.0:1616513098.714773:0:6487:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:0.0:1616513098.714786:0:6487:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513098.714793:0:6487:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:0.0:1616513098.714803:0:6487:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:27.2:1616513098.714806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:0.0:1616513098.714806:0:6487:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:0.0:1616513098.714808:0:6487:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:0.0:1616513098.714814:0:6487:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (24)-- 00000800:00000200:27.0:1616513098.714825:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000800:00000200:27.0:1616513098.714830:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:27.0:1616513098.714833:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513098.714839:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513098.714845:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d56affc1200 x1693067725929664/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513098.714857:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:27.0:1616513098.714861:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513098.714863:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513098.714867:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513098.714868:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:11.0:1616513098.714871:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513098.714873:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513098.714878:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d465b08d100 x1693067725929680/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513098.714887:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c5d8 00000800:00000200:27.0:1616513098.714889:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513098.714991:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513098.714999:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513098.715004:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000800:00000200:11.0:1616513098.715006:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513098.715012:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:11.0:1616513098.715012:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513098.715015:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513098.715019:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627cc0 00000400:00000200:11.0:1616513098.715019:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:11.0:1616513098.715026:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627cd0 00000400:00000200:27.0:1616513098.715027:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716765d [1] + 192 00000400:00000200:27.0:1616513098.715031:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:11.0:1616513098.715033:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716766d [1] + 192 00000400:00000200:11.0:1616513098.715037:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513098.715039:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d56affc1200 x1693067725929664/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513098.715052:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000100:00000200:11.0:1616513098.715053:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d465b08d100 x1693067725929680/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513098.715055:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513098.715056:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:11.0:1616513098.715066:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:11.0:1616513098.715069:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:11.0:1616513098.715070:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:8.0:1616513098.715080:0:6486:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d56affc1200 x1693067725929664/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:8.0:1616513098.715092:0:6486:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d56affc1200 x1693067725929664/t0(0) o13->boslfs02-OST0006-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513098.715102:0:6486:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000100:00000200:0.0:1616513098.715107:0:6487:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d465b08d100 x1693067725929680/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:0.0:1616513098.715120:0:6487:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d465b08d100 x1693067725929680/t0(0) o13->boslfs02-OST0002-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513098.715131:0:6487:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d760 00000800:00000200:11.2:1616513098.721282:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.721338:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.721348:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.721357:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.197@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.721368:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.197@tcp1 of length 224 into portal 26 MB=0x604a4872140c0 00000400:00000200:24.0:1616513098.721375:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.197@tcp1 of length 224/224 into md 0xf97d09e9 [1] + 0 00000400:00000200:24.0:1616513098.721380:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c17078f8 00000400:00000200:24.0:1616513098.721383:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.721386:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513098.721391:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17078f8 00000800:00000200:16.0:1616513098.721392:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.721394:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.721397:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.721399:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.721408:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:8.0:1616513098.721411:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954548515008 00010000:00000200:8.0:1616513098.721422:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1693954548515008/t0(0) o400->e1ef48ff-3b22-544a-9924-432904be2d55@10.31.140.197@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.721441:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954548515008, offset 192 00000400:00000200:8.0:1616513098.721446:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.197@tcp1 00000400:00000200:8.0:1616513098.721454:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.197@tcp1 routed destination 00000800:00000200:0.0:1616513098.721462:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513098.721469:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.197@tcp1(10.31.140.197@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513098.721472:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513098.721475:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.721481:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:0.0:1616513098.721481:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.197@tcp1 : PUT - for me 00000800:00000200:8.0:1616513098.721484:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.721486:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513098.721490:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.197@tcp1 of length 224 into portal 12 MB=0x604a487214200 00000800:00000200:8.0:1616513098.721491:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.721497:0:51455:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513098.721497:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.197@tcp1 of length 224/224 into md 0x865be291 [1] + 77808 00000100:00000200:8.0:1616513098.721498:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:0.0:1616513098.721502:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.721505:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513098.721512:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.721515:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.721517:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.721522:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513098.721572:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.721579:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.721582:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:28.0:1616513098.721588:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954548515328 00000400:00000200:14.0:1616513098.721588:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.721593:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:14.0:1616513098.721597:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.721600:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:28.0:1616513098.721602:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b99200 x1693954548515328/t0(0) o400->18969db2-c6cb-bc68-11a4-bd047e7f51e1@10.31.140.197@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.721616:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954548515328, offset 192 00000400:00000200:28.0:1616513098.721625:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.197@tcp1 00000400:00000200:28.0:1616513098.721632:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.197@tcp1 routed destination 00000400:00000200:28.0:1616513098.721648:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.197@tcp1(10.31.140.197@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.721654:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.721660:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.721662:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.721665:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.721670:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.721675:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.721685:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.721690:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.721695:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.721695:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.721702:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.721706:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccc38 00000800:00000200:16.0:1616513098.721710:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:30.0:1616513098.730630:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929696, portal 4 00000100:00000200:30.0:1616513098.730635:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929696, offset 0 00000400:00000200:30.0:1616513098.730642:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:30.0:1616513098.730647:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:30.0:1616513098.730659:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513098.730665:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:30.0:1616513098.730670:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513098.730673:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:30.0:1616513098.730675:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:30.0:1616513098.730680:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513098.730727:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513098.730786:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513098.730793:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513098.730796:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.730803:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513098.730809:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bdc800 x1693067725929696/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513098.730820:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee198 00000800:00000200:26.0:1616513098.730824:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513098.730827:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513098.730902:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513098.730956:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513098.730966:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513098.730975:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513098.730983:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627ce0 00000400:00000200:14.0:1616513098.730989:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d0ff9 [1] + 192 00000400:00000200:14.0:1616513098.730995:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.731000:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bdc800 x1693067725929696/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513098.731012:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513098.731015:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513098.731018:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:30.0:1616513098.731019:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bdc800 x1693067725929696/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513098.731020:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:30.0:1616513098.731031:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bdc800 x1693067725929696/t0(0) o13->boslfs02-OST000c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513098.731049:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:7.2:1616513098.748936:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.748995:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.749004:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.749013:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.130@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.749022:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.130@tcp1 of length 224 into portal 26 MB=0x5ffd8a0e18f00 00000400:00000200:26.0:1616513098.749029:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.130@tcp1 of length 224/224 into md 0xf97d0a09 [1] + 0 00000400:00000200:26.0:1616513098.749037:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707650 00000400:00000200:26.0:1616513098.749039:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.749042:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513098.749047:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707650 00000800:00000200:18.0:1616513098.749049:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.749050:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.749053:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513098.749056:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.749059:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.749060:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513098.749066:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688680760708864 00000400:00000200:18.0:1616513098.749069:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.130@tcp1 : PUT - for me 00010000:00000200:8.0:1616513098.749075:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1688680760708864/t0(0) o400->d50e971a-8720-8290-8b01-b43edd2f62c5@10.31.143.130@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513098.749077:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.130@tcp1 of length 224 into portal 12 MB=0x5ffd8a0e18f80 00000400:00000200:18.0:1616513098.749084:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.130@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 45360 00000100:00000200:8.0:1616513098.749088:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688680760708864, offset 192 00000400:00000200:18.0:1616513098.749089:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.749092:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513098.749094:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.130@tcp1 00000800:00000200:18.0:1616513098.749099:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513098.749100:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.130@tcp1 routed destination 00000800:00000200:18.0:1616513098.749103:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.749104:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513098.749114:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.130@tcp1(10.31.143.130@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.749120:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.749125:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.749128:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.749129:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.749134:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.749141:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.749141:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513098.749169:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688680760708992 00010000:00000200:22.0:1616513098.749182:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621f80 x1688680760708992/t0(0) o400->ca744503-5b8e-c18d-7b79-2984c42e5fbb@10.31.143.130@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:14.0:1616513098.749194:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:22.0:1616513098.749197:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688680760708992, offset 192 00000800:00000200:14.0:1616513098.749202:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513098.749203:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.130@tcp1 00000400:00000200:14.0:1616513098.749204:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513098.749205:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513098.749210:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.130@tcp1 routed destination 00000400:00000200:14.0:1616513098.749211:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.749217:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000800:00000200:14.0:1616513098.749220:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:22.0:1616513098.749226:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.130@tcp1(10.31.143.130@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.749234:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.749240:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.749243:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.749245:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.749250:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.749258:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.749314:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.749320:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.749323:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.749330:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.749334:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b3a8 00000800:00000200:20.0:1616513098.749338:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.749341:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.754796:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513098.754822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513098.754852:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.754861:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513098.754867:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.754870:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.60@tcp1 : PUT - for me 00000800:00000200:24.0:1616513098.754880:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513098.754880:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.60@tcp1 of length 224 into portal 12 MB=0x6037bcd0b5bc0 00000400:00000200:2.0:1616513098.754886:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.60@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 45584 00000800:00000200:24.0:1616513098.754889:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513098.754891:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.754894:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513098.754898:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.60@tcp1 : PUT - for me 00000800:00000200:2.0:1616513098.754900:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.754903:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.754904:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513098.754908:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.60@tcp1 of length 224 into portal 26 MB=0x6037bcd0b5b40 00000400:00000200:24.0:1616513098.754915:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.60@tcp1 of length 224/224 into md 0xf97d0a29 [1] + 0 00000400:00000200:24.0:1616513098.754920:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707078 00000400:00000200:24.0:1616513098.754922:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.754925:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513098.754930:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707078 00000800:00000200:24.0:1616513098.754933:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.754934:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.754935:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.754938:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513098.754942:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692680116198208 00010000:00000200:8.0:1616513098.754949:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1692680116198208/t0(0) o400->2d6e1b29-637e-3f91-eb96-0bc3891d3725@10.31.140.60@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.754960:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692680116198208, offset 192 00000400:00000200:8.0:1616513098.754965:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.60@tcp1 00000100:00000200:22.0:1616513098.754968:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692680116198336 00000400:00000200:8.0:1616513098.754971:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.60@tcp1 routed destination 00010000:00000200:22.0:1616513098.754980:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dba80 x1692680116198336/t0(0) o400->84bae4ee-0338-1641-9a20-ee4cf017e286@10.31.140.60@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:8.0:1616513098.754984:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.60@tcp1(10.31.140.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.754989:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.754993:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:22.0:1616513098.754994:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692680116198336, offset 192 00000800:00000200:8.0:1616513098.754996:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.754997:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.755000:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513098.755001:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.60@tcp1 00000100:00000200:8.0:1616513098.755005:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.755005:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:22.0:1616513098.755008:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.60@tcp1 routed destination 00000400:00000200:22.0:1616513098.755024:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.60@tcp1(10.31.140.60@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.755030:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.755036:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.755038:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.755041:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.755045:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.755050:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513098.755058:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.755065:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.755068:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.755074:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.755080:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2000 00000800:00000200:26.0:1616513098.755084:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.755087:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513098.755102:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.755109:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.755112:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.755118:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.755122:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ab28 00000800:00000200:20.0:1616513098.755127:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.755130:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.755954:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.755965:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.755973:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513098.755975:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.755982:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.64@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.755991:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.64@tcp1 of length 224 into portal 12 MB=0x60218611c9580 00000400:00000200:16.0:1616513098.755998:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.64@tcp1 of length 224/224 into md 0x865be291 [1] + 78032 00000400:00000200:16.0:1616513098.756003:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.756006:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:7.2:1616513098.756010:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513098.756012:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.756015:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.756017:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513098.756083:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.756093:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:28.0:1616513098.756098:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691153591997824 00000400:00000200:18.0:1616513098.756102:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.64@tcp1 : PUT - for me 00010000:00000200:28.0:1616513098.756110:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d7980 x1691153591997824/t0(0) o400->8b9210f6-25f2-1dc1-9ed1-72a40c7b8c75@10.31.140.64@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:18.0:1616513098.756110:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.64@tcp1 of length 224 into portal 26 MB=0x60218611c9500 00000400:00000200:18.0:1616513098.756117:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.64@tcp1 of length 224/224 into md 0xf97d0a49 [1] + 0 00000400:00000200:18.0:1616513098.756122:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707ed0 00000400:00000200:18.0:1616513098.756124:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513098.756125:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691153591997824, offset 192 00000100:00000200:18.0:1616513098.756127:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:28.0:1616513098.756132:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.64@tcp1 00000400:00000200:18.0:1616513098.756132:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707ed0 00000800:00000200:18.0:1616513098.756135:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.756137:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:28.0:1616513098.756139:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.64@tcp1 routed destination 00000800:00000200:18.0:1616513098.756139:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.756141:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691153591997696 00000800:00000200:18.0:1616513098.756142:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:8.0:1616513098.756146:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1691153591997696/t0(0) o400->32d9e64d-a6e7-b37f-fa4d-80df1e80fa71@10.31.140.64@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513098.756154:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.64@tcp1(10.31.140.64@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:8.0:1616513098.756156:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691153591997696, offset 192 00000800:00000200:28.0:1616513098.756161:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513098.756161:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.64@tcp1 00000400:00000200:8.0:1616513098.756167:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.64@tcp1 routed destination 00000400:00000200:8.0:1616513098.756180:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.64@tcp1(10.31.140.64@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.756185:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.756186:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.756189:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.756190:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.756191:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.756192:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513098.756193:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.756196:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513098.756196:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513098.756200:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513098.756201:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513098.756201:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.756210:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:20.0:1616513098.756210:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513098.756214:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513098.756214:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513098.756215:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.756216:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.756220:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513098.756220:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513098.756221:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.756221:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.756228:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccb28 00000800:00000200:26.0:1616513098.756231:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513098.756232:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3c28 00000800:00000200:20.0:1616513098.756234:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.780469:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513098.780526:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.780535:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.780545:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.9@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.780554:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.9@tcp1 of length 224 into portal 26 MB=0x6001bd6998c80 00000400:00000200:18.0:1616513098.780561:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.9@tcp1 of length 224/224 into md 0xf97d0a59 [1] + 0 00000400:00000200:18.0:1616513098.780565:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1706e58 00000400:00000200:18.0:1616513098.780568:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.780573:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513098.780578:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706e58 00000800:00000200:2.0:1616513098.780580:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.780581:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.780584:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.780586:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513098.780598:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969424768128 00010000:00000200:8.0:1616513098.780609:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688969424768128/t0(0) o400->a9192ddf-56aa-7680-e2ad-49adf518e12b@10.31.133.9@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.780631:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688969424768128, offset 192 00000400:00000200:8.0:1616513098.780637:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.9@tcp1 00000400:00000200:8.0:1616513098.780644:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.9@tcp1 routed destination 00000400:00000200:8.0:1616513098.780659:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.9@tcp1(10.31.133.9@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.780665:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.780670:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.780673:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.780675:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.780679:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.780684:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513098.780686:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513098.780736:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.780743:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.780745:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513098.780752:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.780758:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:14.0:1616513098.780762:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.780765:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513098.780861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.780873:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.780881:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513098.780883:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.780890:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.9@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.780899:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.9@tcp1 of length 224 into portal 12 MB=0x6001bd6998d80 00000400:00000200:2.0:1616513098.780905:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.9@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 45808 00000400:00000200:2.0:1616513098.780910:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.780913:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513098.780920:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.780923:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.780925:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513098.780986:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688969424768384 00010000:00000200:22.0:1616513098.780999:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9680 x1688969424768384/t0(0) o400->df2f0c42-c4d9-b462-9c31-ddbab08f1595@10.31.133.9@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.781014:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688969424768384, offset 192 00000400:00000200:22.0:1616513098.781020:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.9@tcp1 00000400:00000200:22.0:1616513098.781027:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.9@tcp1 routed destination 00000400:00000200:22.0:1616513098.781042:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.9@tcp1(10.31.133.9@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.781048:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.781054:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.781057:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.781059:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.781064:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.781071:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.781124:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.781131:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.781133:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.781139:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.781143:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a088 00000800:00000200:24.0:1616513098.781147:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.781150:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.788775:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.788826:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.788834:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.788842:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.70@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.788852:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.70@tcp1 of length 224 into portal 26 MB=0x5ffec33b0ad40 00000400:00000200:14.0:1616513098.788858:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.70@tcp1 of length 224/224 into md 0xf97d0a69 [1] + 0 00000400:00000200:14.0:1616513098.788863:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707430 00000400:00000200:14.0:1616513098.788865:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.788868:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.788872:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707430 00000800:00000200:14.0:1616513098.788875:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.788878:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513098.788878:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.788885:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.788888:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.788889:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513098.788889:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688764828134720 00000400:00000200:26.0:1616513098.788898:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.70@tcp1 : PUT - for me 00010000:00000200:8.0:1616513098.788898:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1688764828134720/t0(0) o400->434263dc-2032-f05f-aa10-0d2ced4332fb@10.31.142.70@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513098.788907:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.70@tcp1 of length 224 into portal 12 MB=0x5ffec33b0adc0 00000100:00000200:8.0:1616513098.788911:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688764828134720, offset 192 00000400:00000200:26.0:1616513098.788914:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.70@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 46032 00000400:00000200:8.0:1616513098.788916:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.70@tcp1 00000400:00000200:26.0:1616513098.788919:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.788921:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513098.788923:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.70@tcp1 routed destination 00000800:00000200:26.0:1616513098.788928:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.788931:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.788932:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513098.788937:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.70@tcp1(10.31.142.70@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.788943:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.788948:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.788951:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.788953:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.788957:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.788963:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.788963:0:51462:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:22.0:1616513098.788991:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688764828134848 00010000:00000200:22.0:1616513098.789003:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd219b00 x1688764828134848/t0(0) o400->d122d2bc-e5b0-6982-a08d-8c69e440d822@10.31.142.70@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:2.0:1616513098.789016:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:22.0:1616513098.789018:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688764828134848, offset 192 00000400:00000200:22.0:1616513098.789024:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.70@tcp1 00000800:00000200:2.0:1616513098.789024:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.789026:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.789027:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513098.789032:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.70@tcp1 routed destination 00000400:00000200:2.0:1616513098.789033:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.789039:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3980 00000800:00000200:2.0:1616513098.789043:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:22.0:1616513098.789048:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.70@tcp1(10.31.142.70@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.789054:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.789060:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.789062:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.789065:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.789069:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.789076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.789131:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.789143:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.789146:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.789153:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.789157:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a440 00000800:00000200:0.0:1616513098.789161:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.789164:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.798281:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513098.798338:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.798347:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.798356:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.132@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.798365:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.132@tcp1 of length 224 into portal 26 MB=0x5ffd4b5d898c0 00000400:00000200:24.0:1616513098.798372:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.132@tcp1 of length 224/224 into md 0xf97d0a79 [1] + 0 00000400:00000200:24.0:1616513098.798377:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c17063b8 00000400:00000200:24.0:1616513098.798380:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.798382:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513098.798387:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17063b8 00000800:00000200:24.0:1616513098.798390:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.798392:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.798393:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513098.798397:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663932573888 00000800:00000200:24.0:1616513098.798399:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.798402:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:8.0:1616513098.798402:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688663932573888/t0(0) o400->ac5da34e-b51b-71f0-977e-f40ea78874e5@10.31.136.132@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:16.0:1616513098.798403:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.798411:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.132@tcp1 : PUT - for me 00000100:00000200:8.0:1616513098.798412:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663932573888, offset 192 00000400:00000200:8.0:1616513098.798417:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.132@tcp1 00000400:00000200:16.0:1616513098.798420:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.132@tcp1 of length 224 into portal 12 MB=0x5ffd4b5d899c0 00000400:00000200:8.0:1616513098.798422:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.132@tcp1 routed destination 00000400:00000200:16.0:1616513098.798427:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.132@tcp1 of length 224/224 into md 0x865be291 [1] + 78256 00000400:00000200:16.0:1616513098.798432:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513098.798444:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.132@tcp1(10.31.136.132@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513098.798448:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513098.798450:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.798454:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513098.798455:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513098.798456:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.798457:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513098.798458:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.798460:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513098.798460:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.798464:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.798474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:28.0:1616513098.798519:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663932574144 00010000:00000200:28.0:1616513098.798531:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d1b00 x1688663932574144/t0(0) o400->99f974cb-bd01-8f75-e591-23fe9e8d15be@10.31.136.132@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:18.0:1616513098.798532:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.798538:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.798541:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:28.0:1616513098.798546:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663932574144, offset 192 00000400:00000200:18.0:1616513098.798548:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:28.0:1616513098.798552:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.132@tcp1 00000400:00000200:18.0:1616513098.798557:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3cb0 00000400:00000200:28.0:1616513098.798559:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.132@tcp1 routed destination 00000800:00000200:18.0:1616513098.798560:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.798563:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513098.798574:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.132@tcp1(10.31.136.132@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.798581:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.798587:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.798591:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.798593:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.798598:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.798603:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.798656:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.798664:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.798666:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.798668:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513098.798673:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.798677:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccd48 00000800:00000200:20.0:1616513098.798680:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.804214:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.804269:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.804278:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.804286:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.128@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.804295:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.128@tcp1 of length 224 into portal 26 MB=0x5f95f13d586c0 00000400:00000200:14.0:1616513098.804301:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.128@tcp1 of length 224/224 into md 0xf97d0a89 [1] + 0 00000400:00000200:14.0:1616513098.804306:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707b18 00000400:00000200:14.0:1616513098.804308:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.804311:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.804315:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707b18 00000800:00000200:14.0:1616513098.804318:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.804320:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.804322:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.804324:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513098.804326:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681561633523392 00010000:00000200:8.0:1616513098.804331:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1681561633523392/t0(0) o400->8e41ec44-29bc-a4f8-cee3-4940eadfd1cf@10.31.143.128@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513098.804335:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513098.804340:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681561633523392, offset 192 00000400:00000200:8.0:1616513098.804345:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.128@tcp1 00000400:00000200:2.0:1616513098.804346:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.128@tcp1 : PUT - for me 00000400:00000200:8.0:1616513098.804350:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.128@tcp1 routed destination 00000400:00000200:2.0:1616513098.804355:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.128@tcp1 of length 224 into portal 12 MB=0x5f95f13d58740 00000400:00000200:2.0:1616513098.804361:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.128@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 46256 00000400:00000200:8.0:1616513098.804362:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.128@tcp1(10.31.143.128@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:2.0:1616513098.804366:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:8.0:1616513098.804367:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:2.0:1616513098.804369:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513098.804371:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.804373:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513098.804374:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513098.804376:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:8.0:1616513098.804377:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513098.804379:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:8.0:1616513098.804380:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513098.804381:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.804383:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.804387:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.804389:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513098.804391:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.804394:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.804399:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf22a8 00000800:00000200:2.0:1616513098.804402:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513098.804456:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681561633523520 00010000:00000200:22.0:1616513098.804469:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d9f80 x1681561633523520/t0(0) o400->ea4f45db-ae47-1407-b27c-638ff9a77d77@10.31.143.128@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.804483:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681561633523520, offset 192 00000400:00000200:22.0:1616513098.804491:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.128@tcp1 00000400:00000200:22.0:1616513098.804498:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.128@tcp1 routed destination 00000400:00000200:22.0:1616513098.804513:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.128@tcp1(10.31.143.128@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.804520:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:11.2:1616513098.804520:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.804525:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.804528:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.804530:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513098.804535:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.804593:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.804603:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.804612:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.8@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.804621:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.8@tcp1 of length 224 into portal 26 MB=0x5ffd4a198b740 00000400:00000200:16.0:1616513098.804628:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.8@tcp1 of length 224/224 into md 0xf97d0aa9 [1] + 0 00000400:00000200:16.0:1616513098.804633:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707188 00000400:00000200:16.0:1616513098.804635:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.804640:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:16.0:1616513098.804645:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707188 00000800:00000200:16.0:1616513098.804648:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513098.804648:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:16.0:1616513098.804651:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:8.0:1616513098.804655:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663592843072 00000800:00000200:16.0:1616513098.804658:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00010000:00000200:8.0:1616513098.804659:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688663592843072/t0(0) o400->8830ccf4-e4ab-e0f6-f5dd-967052ffa43d@10.31.138.8@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513098.804659:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:16.0:1616513098.804661:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.804662:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513098.804668:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663592843072, offset 192 00000400:00000200:0.0:1616513098.804669:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513098.804673:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.8@tcp1 00000400:00000200:0.0:1616513098.804673:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b100 00000400:00000200:8.0:1616513098.804678:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.8@tcp1 routed destination 00000800:00000200:0.0:1616513098.804683:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:8.0:1616513098.804690:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.8@tcp1(10.31.138.8@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.804695:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.804699:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.804701:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.804703:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.804705:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.804709:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.804711:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.804721:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.804725:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.804727:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513098.804729:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513098.804732:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.804738:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3430 00000800:00000200:14.0:1616513098.804740:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.804813:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.804867:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.804877:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513098.804878:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513098.804885:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.8@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.804894:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.8@tcp1 of length 224 into portal 12 MB=0x5ffd4a198b880 00000400:00000200:20.0:1616513098.804900:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.8@tcp1 of length 224/224 into md 0x865be291 [1] + 78480 00000400:00000200:20.0:1616513098.804908:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.804911:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513098.804917:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.804920:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.804922:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.804993:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663592843392 00010000:00000200:28.0:1616513098.805006:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663f00 x1688663592843392/t0(0) o400->767b926d-a84f-e17d-7f17-528eef9ebc66@10.31.138.8@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.805020:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663592843392, offset 192 00000400:00000200:28.0:1616513098.805026:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.8@tcp1 00000400:00000200:28.0:1616513098.805034:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.8@tcp1 routed destination 00000400:00000200:28.0:1616513098.805049:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.8@tcp1(10.31.138.8@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.805055:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.805061:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.805064:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.805066:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.805071:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.805076:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.805086:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.805089:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.805091:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.805096:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513098.805096:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.805099:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcca18 00000800:00000200:0.0:1616513098.805101:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.808839:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.808893:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.808913:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.808921:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.5@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.808929:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.5@tcp1 of length 224 into portal 26 MB=0x5ffd59e34fa80 00000400:00000200:2.0:1616513098.808935:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.130.5@tcp1 of length 224/224 into md 0xf97d0ab9 [1] + 0 00000400:00000200:2.0:1616513098.808940:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707760 00000400:00000200:2.0:1616513098.808942:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.808945:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.808948:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707760 00000800:00000200:2.0:1616513098.808951:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.808954:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.808955:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.808958:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513098.808968:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667830942336 00010000:00000200:8.0:1616513098.808978:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1688667830942336/t0(0) o400->e457ea0b-f4ef-a816-d54e-c2d0e6771747@10.31.130.5@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.808991:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667830942336, offset 192 00000400:00000200:8.0:1616513098.808996:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.5@tcp1 00000400:00000200:8.0:1616513098.809004:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.5@tcp1 routed destination 00000400:00000200:8.0:1616513098.809017:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.5@tcp1(10.31.130.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.809023:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.809029:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.809031:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.809033:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.809037:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.809042:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513098.809043:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513098.809096:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.809103:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.809105:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513098.809112:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.809118:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2aa0 00000800:00000200:18.0:1616513098.809121:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.809124:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.809331:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.809391:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.809400:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.809409:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.130.5@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.809417:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.130.5@tcp1 of length 224 into portal 12 MB=0x5ffd59e34fac0 00000400:00000200:16.0:1616513098.809424:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.130.5@tcp1 of length 224/224 into md 0x865be291 [1] + 78704 00000400:00000200:16.0:1616513098.809429:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.809432:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513098.809452:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.809454:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.809457:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.809459:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.809518:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667830942400 00010000:00000200:28.0:1616513098.809530:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d4800 x1688667830942400/t0(0) o400->1f279942-b786-ebcf-ab75-bc35f72f25ea@10.31.130.5@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.809545:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667830942400, offset 192 00000400:00000200:28.0:1616513098.809551:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.130.5@tcp1 00000400:00000200:28.0:1616513098.809559:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.130.5@tcp1 routed destination 00000400:00000200:28.0:1616513098.809574:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.130.5@tcp1(10.31.130.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.809580:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.809586:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.809588:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.809591:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.809596:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.809601:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.809654:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.809661:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.809664:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.809670:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.809674:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc4c8 00000800:00000200:24.0:1616513098.809678:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.809681:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:0.0:1616513098.810474:0:6480:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929712, portal 4 00000100:00000200:0.0:1616513098.810479:0:6480:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929712, offset 0 00000400:00000200:0.0:1616513098.810486:0:6480:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:0.0:1616513098.810492:0:6480:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:0.0:1616513098.810505:0:6480:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513098.810512:0:6480:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:0.0:1616513098.810517:0:6480:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513098.810520:0:6480:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:0.0:1616513098.810523:0:6480:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:0.0:1616513098.810528:0:6480:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513098.810578:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513098.810635:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513098.810642:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513098.810644:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.810650:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513098.810656:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdeb8f980 x1693067725929712/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513098.810667:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cbb0 00000800:00000200:2.0:1616513098.810673:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513098.810676:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513098.810681:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:26.0:1616513098.810688:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:2.0:1616513098.810688:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:2.0:1616513098.810695:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627cf0 00000400:00000200:2.0:1616513098.810701:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d10f9 [1] + 192 00000400:00000200:2.0:1616513098.810705:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.810709:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdeb8f980 x1693067725929712/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513098.810722:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513098.810724:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513098.810726:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:0.0:1616513098.810729:0:6480:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdeb8f980 x1693067725929712/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:0.0:1616513098.810742:0:6480:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdeb8f980 x1693067725929712/t0(0) o13->boslfs02-OST001c-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513098.810751:0:6480:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922d760 00000100:00000200:3.0:1616513098.826650:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929728, portal 4 00000100:00000200:3.0:1616513098.826656:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929728, offset 0 00000400:00000200:3.0:1616513098.826663:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:3.0:1616513098.826669:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:3.0:1616513098.826683:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513098.826690:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:3.0:1616513098.826697:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513098.826700:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:3.0:1616513098.826702:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:3.0:1616513098.826708:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513098.826754:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513098.826772:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513098.826777:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513098.826781:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513098.826787:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513098.826793:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bfe780 x1693067725929728/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513098.826804:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd3a90 00000800:00000200:27.0:1616513098.826808:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513098.826811:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513098.826925:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513098.826932:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513098.826938:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:27.0:1616513098.826945:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513098.826953:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627d00 00000400:00000200:27.0:1616513098.826959:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716767d [1] + 192 00000400:00000200:27.0:1616513098.826964:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513098.826968:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bfe780 x1693067725929728/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513098.826980:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:11.0:1616513098.826981:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513098.826983:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513098.826985:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:3.0:1616513098.826987:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bfe780 x1693067725929728/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513098.827000:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bfe780 x1693067725929728/t0(0) o13->boslfs02-OST0022-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513098.827012:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2a18 00000800:00000200:11.2:1616513098.847201:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.847258:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.847267:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513098.847277:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.42@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.847287:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.42@tcp1 of length 224 into portal 26 MB=0x5ffdc1244c540 00000400:00000200:20.0:1616513098.847293:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.42@tcp1 of length 224/224 into md 0xf97d0ad9 [1] + 0 00000400:00000200:20.0:1616513098.847298:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707cb0 00000400:00000200:20.0:1616513098.847301:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.847304:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513098.847308:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707cb0 00000800:00000200:20.0:1616513098.847312:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.847314:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513098.847316:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.847319:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513098.847326:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695547938112 00010000:00000200:8.0:1616513098.847334:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688695547938112/t0(0) o400->702a1d24-1e68-91c5-21ee-2a9eab32ea08@10.31.140.42@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.847346:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688695547938112, offset 192 00000400:00000200:8.0:1616513098.847352:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.42@tcp1 00000400:00000200:8.0:1616513098.847359:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.42@tcp1 routed destination 00000400:00000200:8.0:1616513098.847372:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.42@tcp1(10.31.140.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.847378:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.847384:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.847386:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.847388:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:7.2:1616513098.847390:0:51467:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513098.847392:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.847398:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513098.847405:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.847414:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.847423:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.42@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.847432:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.42@tcp1 of length 224 into portal 12 MB=0x5ffdc1244c800 00000400:00000200:14.0:1616513098.847445:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.42@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 46480 00000400:00000200:14.0:1616513098.847450:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.847453:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513098.847458:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513098.847460:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513098.847463:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:14.0:1616513098.847465:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513098.847466:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000400:00000200:26.0:1616513098.847469:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.847469:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.847476:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.847481:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf27f8 00000800:00000200:26.0:1616513098.847485:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513098.847525:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688695547938816 00010000:00000200:22.0:1616513098.847538:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eba880 x1688695547938816/t0(0) o400->997bf0b0-693f-3dc3-bb75-4eb3e6ef64a7@10.31.140.42@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.847552:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688695547938816, offset 192 00000400:00000200:22.0:1616513098.847559:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.42@tcp1 00000400:00000200:22.0:1616513098.847566:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.42@tcp1 routed destination 00000400:00000200:22.0:1616513098.847583:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.42@tcp1(10.31.140.42@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.847590:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.847595:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.847598:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.847600:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.847605:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.847611:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.847665:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.847673:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.847675:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513098.847682:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.847688:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a220 00000800:00000200:24.0:1616513098.847692:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513098.847695:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:31.0:1616513098.858588:0:6508:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929744, portal 4 00000100:00000200:31.0:1616513098.858596:0:6508:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929744, offset 0 00000100:00000200:16.0:1616513098.858597:0:6481:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929760, portal 4 00000400:00000200:31.0:1616513098.858603:0:6508:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000100:00000200:16.0:1616513098.858603:0:6481:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929760, offset 0 00000400:00000200:31.0:1616513098.858609:0:6508:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:16.0:1616513098.858610:0:6481:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:16.0:1616513098.858616:0:6481:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:31.0:1616513098.858631:0:6508:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513098.858631:0:6481:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513098.858639:0:6481:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:31.0:1616513098.858640:0:6508:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:16.0:1616513098.858646:0:6481:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513098.858647:0:6508:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513098.858649:0:6481:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:31.0:1616513098.858650:0:6508:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:16.0:1616513098.858651:0:6481:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:31.0:1616513098.858653:0:6508:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:16.0:1616513098.858657:0:6481:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:31.0:1616513098.858658:0:6508:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:25.2:1616513098.858703:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:5.2:1616513098.858723:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:2.0:1616513098.858761:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513098.858769:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513098.858772:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.858779:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:20.0:1616513098.858782:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000100:00000200:2.0:1616513098.858785:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d4536052400 x1693067725929760/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513098.858790:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513098.858793:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513098.858794:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:2.0:1616513098.858797:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788660 00000400:00000200:20.0:1616513098.858800:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000800:00000200:2.0:1616513098.858800:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513098.858803:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:20.0:1616513098.858806:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456eb600 x1693067725929744/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513098.858818:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c3298 00000800:00000200:20.0:1616513098.858823:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513098.858878:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:25.2:1616513098.858887:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:24.0:1616513098.858944:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:26.0:1616513098.858950:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:24.0:1616513098.858954:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:0.0:1616513098.858957:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:26.0:1616513098.858960:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:24.0:1616513098.858963:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513098.858968:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:24.0:1616513098.858971:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627d10 00000400:00000200:26.0:1616513098.858976:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627d20 00000400:00000200:24.0:1616513098.858978:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dd721 [1] + 192 00000400:00000200:26.0:1616513098.858983:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d1129 [1] + 192 00000400:00000200:24.0:1616513098.858983:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:26.0:1616513098.858988:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513098.858988:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456eb600 x1693067725929744/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513098.858993:0:6470:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d4536052400 x1693067725929760/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:24.0:1616513098.859004:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:18.0:1616513098.859005:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:24.0:1616513098.859007:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:26.0:1616513098.859008:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:24.0:1616513098.859009:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:26.0:1616513098.859010:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513098.859012:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:31.0:1616513098.859014:0:6508:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456eb600 x1693067725929744/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513098.859016:0:6481:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d4536052400 x1693067725929760/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:31.0:1616513098.859028:0:6508:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456eb600 x1693067725929744/t0(0) o13->boslfs02-OST0011-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:16.0:1616513098.859029:0:6481:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d4536052400 x1693067725929760/t0(0) o13->boslfs02-OST0018-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:31.0:1616513098.859039:0:6508:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84e33c2330 00000400:00000200:16.0:1616513098.859049:0:6481:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788e58 00000800:00000200:7.2:1616513098.862780:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.862836:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.862845:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.862854:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.138@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.862863:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.138@tcp1 of length 224 into portal 26 MB=0x5ffd76743a7c0 00000400:00000200:2.0:1616513098.862869:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.138@tcp1 of length 224/224 into md 0xf97d0ae9 [1] + 0 00000400:00000200:2.0:1616513098.862874:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707298 00000400:00000200:2.0:1616513098.862877:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.862880:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.862884:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707298 00000800:00000200:2.0:1616513098.862888:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.862889:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.862890:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.862894:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513098.862899:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675499091904 00010000:00000200:8.0:1616513098.862906:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1688675499091904/t0(0) o400->b4f6af33-02f6-423c-b903-8f786c0588cc@10.31.129.138@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.862917:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688675499091904, offset 192 00000400:00000200:8.0:1616513098.862922:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.138@tcp1 00000400:00000200:8.0:1616513098.862928:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.138@tcp1 routed destination 00000400:00000200:8.0:1616513098.862941:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.138@tcp1(10.31.129.138@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.862947:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.862951:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:7.2:1616513098.862952:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.862954:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.862956:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513098.862959:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:8.0:1616513098.862964:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513098.863005:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.863014:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.863023:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.138@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.863031:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.138@tcp1 of length 224 into portal 12 MB=0x5ffd76743ab40 00000400:00000200:18.0:1616513098.863037:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.138@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 46704 00000400:00000200:18.0:1616513098.863043:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.863046:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513098.863053:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513098.863056:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513098.863058:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.863060:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.863068:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513098.863071:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.863072:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.863078:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513098.863084:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ed0 00000800:00000200:26.0:1616513098.863087:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513098.863117:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688675499092800 00010000:00000200:22.0:1616513098.863129:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464623600 x1688675499092800/t0(0) o400->c374d4ac-5166-8050-3d4c-115501838d30@10.31.129.138@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.863144:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688675499092800, offset 192 00000400:00000200:22.0:1616513098.863150:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.138@tcp1 00000400:00000200:22.0:1616513098.863157:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.138@tcp1 routed destination 00000400:00000200:22.0:1616513098.863172:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.138@tcp1(10.31.129.138@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.863179:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.863184:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.863187:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.863189:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.863194:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.863201:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.863256:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.863263:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.863265:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513098.863272:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.863276:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b210 00000800:00000200:16.0:1616513098.863280:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.863283:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:31.0:1616513098.871548:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513098.871555:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:11.2:1616513098.874915:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513098.874971:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.874980:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.874989:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.90@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.874998:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.90@tcp1 of length 224 into portal 26 MB=0x6012e1984ce00 00000400:00000200:0.0:1616513098.875005:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.90@tcp1 of length 224/224 into md 0xf97d0af9 [1] + 0 00000400:00000200:0.0:1616513098.875012:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1706ee0 00000400:00000200:0.0:1616513098.875015:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.875018:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513098.875023:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706ee0 00000800:00000200:24.0:1616513098.875025:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.875026:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.875028:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.875030:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513098.875040:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690147368521216 00010000:00000200:8.0:1616513098.875049:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1690147368521216/t0(0) o400->d5c509e1-ce00-68fc-f695-e0d6aba9c177@10.31.140.90@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.875062:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690147368521216, offset 192 00000400:00000200:8.0:1616513098.875068:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.90@tcp1 00000400:00000200:8.0:1616513098.875075:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.90@tcp1 routed destination 00000400:00000200:8.0:1616513098.875089:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.90@tcp1(10.31.140.90@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.875095:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.875100:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.875103:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.875105:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.875110:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.875116:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.875117:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.875171:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.875178:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.875181:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.875188:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.875193:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf37e8 00000800:00000200:2.0:1616513098.875197:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.875200:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513098.875592:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513098.875648:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.875657:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513098.875666:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.90@tcp1 : PUT - for me 00000400:00000200:16.0:1616513098.875674:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.90@tcp1 of length 224 into portal 12 MB=0x6012e1984d140 00000400:00000200:16.0:1616513098.875681:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.90@tcp1 of length 224/224 into md 0x865be291 [1] + 78928 00000400:00000200:16.0:1616513098.875686:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513098.875689:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513098.875696:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.875699:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.875700:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:16.0:1616513098.875703:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.875761:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690147368522048 00010000:00000200:28.0:1616513098.875773:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5e80 x1690147368522048/t0(0) o400->1b66155a-fd43-daef-0848-317550d3090e@10.31.140.90@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.875798:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690147368522048, offset 192 00000400:00000200:28.0:1616513098.875805:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.90@tcp1 00000400:00000200:28.0:1616513098.875812:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.90@tcp1 routed destination 00000400:00000200:28.0:1616513098.875827:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.90@tcp1(10.31.140.90@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.875834:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.875839:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.875842:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.875845:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.875849:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.875854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.875865:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513098.875870:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513098.875873:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513098.875875:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513098.875879:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513098.875883:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc908 00000800:00000200:24.0:1616513098.875887:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:19.0:1616513098.890658:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929776, portal 4 00000100:00000200:19.0:1616513098.890664:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929776, offset 0 00000400:00000200:19.0:1616513098.890671:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:19.0:1616513098.890676:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:19.0:1616513098.890689:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513098.890696:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:19.0:1616513098.890703:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513098.890706:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:19.0:1616513098.890708:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:19.0:1616513098.890713:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513098.890760:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.890778:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513098.890783:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513098.890786:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513098.890792:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513098.890798:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded3a80 x1693067725929776/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513098.890809:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706110 00000800:00000200:1.0:1616513098.890814:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513098.890816:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513098.890941:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513098.890951:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.890957:0:6466:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513098.890960:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513098.890965:0:6466:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513098.890972:0:6466:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627d30 00000400:00000200:1.0:1616513098.890978:0:6466:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700335 [1] + 192 00000400:00000200:1.0:1616513098.890984:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513098.890988:0:6466:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded3a80 x1693067725929776/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513098.891000:0:6466:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.891003:0:6466:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513098.891004:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:19.0:1616513098.891008:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded3a80 x1693067725929776/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513098.891022:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded3a80 x1693067725929776/t0(0) o13->boslfs02-OST0023-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513098.891031:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707980 00000800:00000200:7.2:1616513098.901928:0:51472:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513098.901985:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513098.902000:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.902010:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.174@tcp1 : PUT - for me 00000400:00000200:14.0:1616513098.902019:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.174@tcp1 of length 224 into portal 26 MB=0x60085ccef4a40 00000400:00000200:14.0:1616513098.902026:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.174@tcp1 of length 224/224 into md 0xf97d0b09 [1] + 0 00000400:00000200:14.0:1616513098.902031:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c17066e8 00000400:00000200:14.0:1616513098.902034:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.902037:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513098.902041:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17066e8 00000800:00000200:14.0:1616513098.902045:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.902047:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.902047:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513098.902051:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513098.902059:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689424529148480 00010000:00000200:8.0:1616513098.902067:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1689424529148480/t0(0) o400->d1642000-c431-5b58-a2bb-c612f4c373c4@10.31.132.174@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.902079:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689424529148480, offset 192 00000400:00000200:8.0:1616513098.902085:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.174@tcp1 00000400:00000200:8.0:1616513098.902092:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.174@tcp1 routed destination 00000400:00000200:8.0:1616513098.902106:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.174@tcp1(10.31.132.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.902112:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.902118:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.902120:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.902122:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.902127:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.902131:0:51472:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513098.902132:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513098.902186:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.902193:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.902196:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.902202:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.902208:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2a18 00000800:00000200:2.0:1616513098.902212:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.902215:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513098.902476:0:51472:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.902487:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.902495:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513098.902497:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513098.902504:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.174@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.902513:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.174@tcp1 of length 224 into portal 12 MB=0x60085ccef4b00 00000400:00000200:26.0:1616513098.902520:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.174@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 46928 00000400:00000200:26.0:1616513098.902526:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.902528:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513098.902536:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.902540:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513098.902542:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513098.902602:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689424529148672 00010000:00000200:22.0:1616513098.902614:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21e780 x1689424529148672/t0(0) o400->5fb29952-d8f6-62af-3029-e25262e444ea@10.31.132.174@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.902629:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689424529148672, offset 192 00000400:00000200:22.0:1616513098.902636:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.174@tcp1 00000400:00000200:22.0:1616513098.902643:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.174@tcp1 routed destination 00000400:00000200:22.0:1616513098.902658:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.174@tcp1(10.31.132.174@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.902664:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.902670:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.902673:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.902683:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.902688:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.902695:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.902752:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.902758:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.902761:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.902768:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.902772:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a7f8 00000800:00000200:20.0:1616513098.902776:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.902781:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.909542:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513098.909599:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.909609:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513098.909617:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.144@tcp1 : PUT - for me 00000400:00000200:2.0:1616513098.909627:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.144@tcp1 of length 224 into portal 26 MB=0x5ffd79dae7900 00000400:00000200:2.0:1616513098.909634:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.144@tcp1 of length 224/224 into md 0xf97d0b19 [1] + 0 00000400:00000200:2.0:1616513098.909639:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1706770 00000400:00000200:2.0:1616513098.909641:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513098.909644:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.909649:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706770 00000800:00000200:2.0:1616513098.909652:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513098.909653:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513098.909655:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513098.909659:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513098.909665:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676412061952 00010000:00000200:8.0:1616513098.909673:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688676412061952/t0(0) o400->fe3df388-a833-fe65-c2ab-b17ce360503e@10.31.143.144@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.909685:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688676412061952, offset 192 00000400:00000200:8.0:1616513098.909690:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.144@tcp1 00000400:00000200:8.0:1616513098.909697:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.144@tcp1 routed destination 00000400:00000200:8.0:1616513098.909711:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.144@tcp1(10.31.143.144@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.909717:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.909722:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.909724:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.909726:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.909730:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.909736:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.909736:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513098.909752:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513098.909789:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.909796:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513098.909798:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513098.909804:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513098.909810:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:0.0:1616513098.909811:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513098.909815:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.909817:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513098.909821:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513098.909830:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.144@tcp1 : PUT - for me 00000400:00000200:0.0:1616513098.909838:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.144@tcp1 of length 224 into portal 12 MB=0x5ffd79dae7980 00000400:00000200:0.0:1616513098.909845:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.144@tcp1 of length 224/224 into md 0x865be291 [1] + 79152 00000400:00000200:0.0:1616513098.909850:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513098.909853:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513098.909863:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.909866:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.909868:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513098.909872:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513098.909928:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676412062080 00010000:00000200:28.0:1616513098.909941:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9de80 x1688676412062080/t0(0) o400->360eb5e2-f87d-1622-5b32-6a31506e6c4c@10.31.143.144@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.909955:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688676412062080, offset 192 00000400:00000200:28.0:1616513098.909962:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.144@tcp1 00000400:00000200:28.0:1616513098.909969:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.144@tcp1 routed destination 00000400:00000200:28.0:1616513098.909984:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.144@tcp1(10.31.143.144@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.909991:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.909997:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.909999:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.910002:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.910007:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.910012:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.910064:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.910071:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.910074:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.910080:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.910084:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc7f8 00000800:00000200:20.0:1616513098.910088:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.910091:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513098.943627:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513098.943683:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.943693:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.943702:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.20@tcp1 : PUT - for me 00000400:00000200:18.0:1616513098.943712:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.20@tcp1 of length 224 into portal 26 MB=0x5ffd3dc6b0000 00000400:00000200:18.0:1616513098.943719:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.20@tcp1 of length 224/224 into md 0xf97d0b29 [1] + 0 00000400:00000200:18.0:1616513098.943724:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707ba0 00000400:00000200:18.0:1616513098.943727:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.943730:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513098.943735:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707ba0 00000800:00000200:18.0:1616513098.943738:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.943739:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.943741:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513098.943743:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513098.943756:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660284735488 00010000:00000200:8.0:1616513098.943767:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688660284735488/t0(0) o400->687c5edb-df0f-87ef-6bd4-8f4d2381186e@10.31.138.20@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513098.943781:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660284735488, offset 192 00000400:00000200:8.0:1616513098.943787:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.20@tcp1 00000400:00000200:8.0:1616513098.943794:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.20@tcp1 routed destination 00000400:00000200:8.0:1616513098.943809:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.20@tcp1(10.31.138.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.943819:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.943825:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.943827:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.943829:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.943835:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513098.943838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513098.943840:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513098.943841:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513098.943895:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513098.943904:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513098.943913:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.20@tcp1 : PUT - for me 00000400:00000200:24.0:1616513098.943922:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.20@tcp1 of length 224 into portal 12 MB=0x5ffd3dc6b0080 00000400:00000200:24.0:1616513098.943928:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.20@tcp1 of length 224/224 into md 0x865be291 [1] + 79376 00000400:00000200:24.0:1616513098.943934:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.943934:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000100:00000200:24.0:1616513098.943936:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513098.943941:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513098.943943:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:14.0:1616513098.943944:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513098.943946:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513098.943948:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513098.943951:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:24.0:1616513098.943953:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:14.0:1616513098.943956:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:14.0:1616513098.943960:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513098.943963:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513098.944024:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660284735616 00010000:00000200:28.0:1616513098.944038:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08e780 x1688660284735616/t0(0) o400->725c47c9-41a2-df06-427a-b9ce286a9897@10.31.138.20@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513098.944052:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660284735616, offset 192 00000400:00000200:28.0:1616513098.944059:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.20@tcp1 00000400:00000200:28.0:1616513098.944067:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.20@tcp1 routed destination 00000400:00000200:28.0:1616513098.944082:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.20@tcp1(10.31.138.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513098.944091:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513098.944096:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513098.944099:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513098.944101:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513098.944107:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.944112:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.944167:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.944174:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513098.944176:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513098.944183:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513098.944187:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc3b8 00000800:00000200:20.0:1616513098.944191:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.944194:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:7.0:1616513098.954478:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929792, portal 4 00000100:00000200:7.0:1616513098.954482:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929792, offset 0 00000400:00000200:7.0:1616513098.954488:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:7.0:1616513098.954493:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:7.0:1616513098.954503:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513098.954509:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:7.0:1616513098.954514:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513098.954516:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:7.0:1616513098.954518:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:7.0:1616513098.954522:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000100:00000200:7.0:1616513098.954531:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929808, portal 4 00000100:00000200:7.0:1616513098.954534:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929808, offset 0 00000400:00000200:7.0:1616513098.954538:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:7.0:1616513098.954542:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:7.0:1616513098.954551:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513098.954556:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:7.0:1616513098.954560:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513098.954562:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:7.0:1616513098.954563:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:7.0:1616513098.954566:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:1.2:1616513098.954568:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.954586:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513098.954592:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513098.954597:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513098.954605:0:6465:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513098.954611:0:6465:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d841ded7500 x1693067725929792/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:25.2:1616513098.954612:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000400:00000200:1.0:1616513098.954623:0:6465:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637e6e8 00000800:00000200:1.0:1616513098.954628:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513098.954630:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:2.0:1616513098.954670:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:2.0:1616513098.954684:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:2.0:1616513098.954688:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513098.954694:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:2.0:1616513098.954701:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd604380 x1693067725929808/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:2.0:1616513098.954725:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637f8f8 00000800:00000200:2.0:1616513098.954729:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513098.954732:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:2.0:1616513098.954738:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:1.2:1616513098.954741:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000400:00000200:2.0:1616513098.954745:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:1.0:1616513098.954750:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000400:00000200:2.0:1616513098.954753:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627d50 00000800:00000200:1.0:1616513098.954756:0:6468:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513098.954759:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:2.0:1616513098.954759:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d11a9 [1] + 192 00000400:00000200:2.0:1616513098.954765:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:1.0:1616513098.954765:0:6468:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000100:00000200:2.0:1616513098.954769:0:6469:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd604380 x1693067725929808/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513098.954773:0:6468:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627d40 00000400:00000200:1.0:1616513098.954780:0:6468:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700345 [1] + 192 00000800:00000200:2.0:1616513098.954782:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513098.954784:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:2.0:1616513098.954785:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (23)-- 00000400:00000200:1.0:1616513098.954785:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513098.954788:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:7.0:1616513098.954789:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd604380 x1693067725929808/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:1.0:1616513098.954789:0:6468:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d841ded7500 x1693067725929792/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513098.954800:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd604380 x1693067725929808/t0(0) o13->boslfs02-OST0038-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513098.954802:0:6468:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513098.954804:0:6468:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513098.954806:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000400:00000200:7.0:1616513098.954808:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637e5d8 00000100:00000200:7.0:1616513098.954819:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d841ded7500 x1693067725929792/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513098.954827:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d841ded7500 x1693067725929792/t0(0) o13->boslfs02-OST002b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513098.954833:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637ef68 00000800:00000200:7.2:1616513098.973474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513098.973530:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513098.973539:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513098.973549:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.226@tcp1 : PUT - for me 00000400:00000200:26.0:1616513098.973560:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.226@tcp1 of length 224 into portal 26 MB=0x5fff2115955c0 00000400:00000200:26.0:1616513098.973567:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.226@tcp1 of length 224/224 into md 0xf97d0b49 [1] + 0 00000400:00000200:26.0:1616513098.973572:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c17077e8 00000400:00000200:26.0:1616513098.973575:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513098.973578:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513098.973582:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17077e8 00000800:00000200:18.0:1616513098.973584:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.973586:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513098.973588:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:26.0:1616513098.973591:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513098.973594:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513098.973594:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513098.973603:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.226@tcp1 : PUT - for me 00000100:00000200:8.0:1616513098.973606:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688790021789120 00000400:00000200:18.0:1616513098.973612:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.226@tcp1 of length 224 into portal 12 MB=0x5fff211595640 00010000:00000200:8.0:1616513098.973618:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1688790021789120/t0(0) o400->4c0e5693-ef2c-2642-c313-726d84b06f29@10.31.136.226@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513098.973619:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.226@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 47152 00000400:00000200:18.0:1616513098.973624:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.973627:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:8.0:1616513098.973632:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688790021789120, offset 192 00000800:00000200:18.0:1616513098.973635:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.973638:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513098.973639:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.226@tcp1 00000800:00000200:18.0:1616513098.973640:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513098.973647:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.226@tcp1 routed destination 00000400:00000200:8.0:1616513098.973679:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.226@tcp1(10.31.136.226@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.973685:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513098.973691:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.973693:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.973696:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.973700:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513098.973705:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513098.973707:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513098.973714:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688790021789248 00010000:00000200:22.0:1616513098.973728:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7464621b00 x1688790021789248/t0(0) o400->478c2eb9-719c-7216-6c90-db0ef1b32dc1@10.31.136.226@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.973742:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688790021789248, offset 192 00000400:00000200:22.0:1616513098.973749:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.226@tcp1 00000400:00000200:22.0:1616513098.973756:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.226@tcp1 routed destination 00000800:00000200:2.0:1616513098.973762:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.973769:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.973771:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513098.973772:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.226@tcp1(10.31.136.226@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513098.973773:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513098.973778:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:2.0:1616513098.973778:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513098.973783:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:22.0:1616513098.973784:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:2.0:1616513098.973786:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:22.0:1616513098.973787:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.973789:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.973794:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.973800:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513098.973856:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.973862:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.973865:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.973872:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.973876:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b8f8 00000800:00000200:0.0:1616513098.973880:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.973883:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513098.988970:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513098.989026:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513098.989036:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513098.989045:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.118@tcp1 : PUT - for me 00000400:00000200:20.0:1616513098.989055:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.118@tcp1 of length 224 into portal 26 MB=0x5ffd4d80b5640 00000400:00000200:20.0:1616513098.989062:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.118@tcp1 of length 224/224 into md 0xf97d0b69 [1] + 0 00000400:00000200:20.0:1616513098.989067:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1706000 00000400:00000200:20.0:1616513098.989070:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513098.989073:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513098.989078:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706000 00000800:00000200:20.0:1616513098.989081:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513098.989084:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513098.989086:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513098.989090:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513098.989094:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664506324544 00000800:00000200:7.2:1616513098.989096:0:51481:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00010000:00000200:8.0:1616513098.989102:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688664506324544/t0(0) o400->918a6a4a-23ec-9ae2-4f25-82dd461fe578@10.31.136.118@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513098.989110:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513098.989114:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664506324544, offset 192 00000800:00000200:14.0:1616513098.989119:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:8.0:1616513098.989119:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.118@tcp1 00000400:00000200:8.0:1616513098.989126:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.118@tcp1 routed destination 00000400:00000200:14.0:1616513098.989128:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.118@tcp1 : PUT - for me 00000400:00000200:8.0:1616513098.989139:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.118@tcp1(10.31.136.118@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.989145:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:14.0:1616513098.989148:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.118@tcp1 of length 224 into portal 12 MB=0x5ffd4d80b5780 00000800:00000200:8.0:1616513098.989150:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.989153:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:14.0:1616513098.989155:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.118@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 47376 00000800:00000200:8.0:1616513098.989155:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:8.0:1616513098.989159:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (24)-- 00000400:00000200:14.0:1616513098.989160:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513098.989163:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:8.0:1616513098.989165:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513098.989167:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513098.989170:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513098.989173:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (25)-- 00000800:00000200:18.0:1616513098.989175:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513098.989175:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000400:00000200:18.0:1616513098.989178:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513098.989178:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513098.989184:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513098.989190:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2cc0 00000800:00000200:18.0:1616513098.989194:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513098.989237:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664506324864 00010000:00000200:22.0:1616513098.989250:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebcc80 x1688664506324864/t0(0) o400->98c44682-9a09-c00a-346f-8cc12502839c@10.31.136.118@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513098.989265:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664506324864, offset 192 00000400:00000200:22.0:1616513098.989271:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.118@tcp1 00000400:00000200:22.0:1616513098.989279:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.118@tcp1 routed destination 00000400:00000200:22.0:1616513098.989294:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.118@tcp1(10.31.136.118@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513098.989301:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513098.989306:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513098.989309:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513098.989311:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.989316:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.989321:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513098.989359:0:51481:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513098.989374:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513098.989381:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513098.989384:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513098.989391:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513098.989395:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a770 00000800:00000200:0.0:1616513098.989399:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513098.989402:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513098.989415:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513098.989425:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513098.989426:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513098.989451:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.52@tcp1 : PUT - for me 00000800:00000200:7.2:1616513098.989452:0:51481:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:2.0:1616513098.989460:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.52@tcp1 of length 224 into portal 26 MB=0x5ffd4960a3a80 00000800:00000200:18.0:1616513098.989462:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513098.989467:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.52@tcp1 of length 224/224 into md 0xf97d0b79 [1] + 0 00000800:00000200:18.0:1616513098.989468:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513098.989471:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513098.989472:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c17067f8 00000400:00000200:2.0:1616513098.989475:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:18.0:1616513098.989476:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.52@tcp1 : PUT - for me 00000100:00000200:2.0:1616513098.989477:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513098.989482:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c17067f8 00000400:00000200:18.0:1616513098.989484:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.52@tcp1 of length 224 into portal 12 MB=0x5ffd4960a3b80 00000800:00000200:2.0:1616513098.989485:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513098.989488:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513098.989490:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.52@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 47600 00000800:00000200:2.0:1616513098.989490:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513098.989491:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663398955648 00010000:00000200:8.0:1616513098.989496:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1688663398955648/t0(0) o400->ba5c95c1-6a46-454b-5cbb-813a46f96b0e@10.31.138.52@tcp1:55/0 lens 224/224 e 0 to 0 dl 1616513190 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:18.0:1616513098.989498:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513098.989500:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:8.0:1616513098.989506:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663398955648, offset 192 00000800:00000200:18.0:1616513098.989508:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513098.989510:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513098.989510:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.52@tcp1 00000800:00000200:18.0:1616513098.989512:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513098.989516:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.52@tcp1 routed destination 00000100:00000200:22.0:1616513098.989519:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663398955904 00010000:00000200:22.0:1616513098.989526:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bad00 x1688663398955904/t0(0) o400->ffeb2481-c5e2-da5a-610c-8e653e554dde@10.31.138.52@tcp1:724/0 lens 224/224 e 0 to 0 dl 1616513104 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:8.0:1616513098.989528:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.52@tcp1(10.31.138.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513098.989533:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:22.0:1616513098.989537:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663398955904, offset 192 00000800:00000200:8.0:1616513098.989537:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513098.989539:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513098.989540:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513098.989543:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513098.989544:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.52@tcp1 00000100:00000200:8.0:1616513098.989547:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513098.989548:0:51481:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:22.0:1616513098.989550:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.52@tcp1 routed destination 00000800:00000200:14.0:1616513098.989557:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513098.989561:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513098.989563:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.52@tcp1(10.31.138.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513098.989563:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513098.989565:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513098.989569:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513098.989569:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513098.989573:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:14.0:1616513098.989573:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ee0 00000800:00000200:22.0:1616513098.989576:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513098.989576:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:22.0:1616513098.989578:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513098.989581:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513098.989586:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513098.989597:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513098.989602:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513098.989605:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513098.989606:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513098.989611:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513098.989615:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b430 00000800:00000200:16.0:1616513098.989619:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.009508:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513099.009565:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.009575:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513099.009581:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513099.009584:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.56@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.009593:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.56@tcp1 of length 224 into portal 26 MB=0x5ffdd40c6de00 00000400:00000200:2.0:1616513099.009600:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.56@tcp1 of length 224/224 into md 0xf97d0b89 [1] + 0 00000400:00000200:2.0:1616513099.009605:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707a08 00000400:00000200:2.0:1616513099.009608:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.009611:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513099.009615:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707a08 00000800:00000200:2.0:1616513099.009618:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.009621:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.009623:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513099.009644:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700623183360 00010000:00000200:8.0:1616513099.009655:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688700623183360/t0(0) o400->a492b999-d0e8-f6e8-c6bc-aafb7821c5b4@10.31.142.56@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513099.009669:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700623183360, offset 192 00000400:00000200:8.0:1616513099.009675:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.56@tcp1 00000400:00000200:8.0:1616513099.009683:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.56@tcp1 routed destination 00000400:00000200:8.0:1616513099.009698:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.56@tcp1(10.31.142.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.009705:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.009711:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.009714:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.009716:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.009721:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.009726:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513099.009727:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.009731:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513099.009781:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.009788:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.009790:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.009797:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513099.009799:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513099.009803:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2440 00000800:00000200:26.0:1616513099.009806:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.009809:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513099.009809:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.009820:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.56@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.009829:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.56@tcp1 of length 224 into portal 12 MB=0x5ffdd40c6e180 00000400:00000200:0.0:1616513099.009836:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.56@tcp1 of length 224/224 into md 0x865be291 [1] + 79600 00000400:00000200:0.0:1616513099.009841:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.009844:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.009851:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.009854:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.009856:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.009859:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.009872:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700623184256 00010000:00000200:28.0:1616513099.009885:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9ba80 x1688700623184256/t0(0) o400->3312979b-a1cb-7a83-61a0-8d49f63b4080@10.31.142.56@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.009900:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700623184256, offset 192 00000400:00000200:28.0:1616513099.009907:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.56@tcp1 00000400:00000200:28.0:1616513099.009914:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.56@tcp1 routed destination 00000400:00000200:28.0:1616513099.009930:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.56@tcp1(10.31.142.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.009936:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513099.009942:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.009945:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513099.009947:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513099.009951:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.009967:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.010021:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.010033:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513099.010035:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.010036:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.010043:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.010047:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc088 00000800:00000200:20.0:1616513099.010051:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.028825:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.028881:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.028890:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.028900:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.60@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.028909:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.60@tcp1 of length 224 into portal 26 MB=0x601845daf9600 00000400:00000200:0.0:1616513099.028915:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.60@tcp1 of length 224/224 into md 0xf97d0b99 [1] + 0 00000400:00000200:0.0:1616513099.028920:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1707e48 00000400:00000200:0.0:1616513099.028923:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.028926:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.028931:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707e48 00000800:00000200:24.0:1616513099.028934:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.028934:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.028936:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.028938:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.028949:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517879363072 00010000:00000200:8.0:1616513099.028957:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1690517879363072/t0(0) o400->1bad8a4f-587a-dfbe-6ff6-c3bf56504386@10.31.141.60@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513099.028971:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690517879363072, offset 192 00000400:00000200:8.0:1616513099.028977:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.60@tcp1 00000400:00000200:8.0:1616513099.028983:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.60@tcp1 routed destination 00000400:00000200:8.0:1616513099.028997:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.60@tcp1(10.31.141.60@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.029003:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.029009:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.029011:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.029013:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.029017:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513099.029023:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513099.029026:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.029090:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.029104:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.029107:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.029114:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.029119:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3650 00000800:00000200:18.0:1616513099.029123:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.029126:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.029498:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.029510:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.029520:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:16.0:1616513099.029529:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.60@tcp1 : PUT - for me 00000400:00000200:16.0:1616513099.029537:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.60@tcp1 of length 224 into portal 12 MB=0x601845dafa200 00000400:00000200:16.0:1616513099.029544:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.60@tcp1 of length 224/224 into md 0x865be291 [1] + 79824 00000400:00000200:16.0:1616513099.029552:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:16.0:1616513099.029554:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513099.029561:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.029564:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513099.029564:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513099.029568:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.029623:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690517879366144 00010000:00000200:28.0:1616513099.029635:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2d5a00 x1690517879366144/t0(0) o400->0ce8e909-f2f5-e4e3-9fbf-da642260110e@10.31.141.60@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.029649:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690517879366144, offset 192 00000400:00000200:28.0:1616513099.029655:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.60@tcp1 00000400:00000200:28.0:1616513099.029661:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.60@tcp1 routed destination 00000400:00000200:28.0:1616513099.029675:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.60@tcp1(10.31.141.60@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.029682:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513099.029687:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.029689:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513099.029691:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513099.029696:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.029700:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.029711:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.029717:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.029719:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.029722:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.029726:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513099.029730:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc2a8 00000800:00000200:24.0:1616513099.029734:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.029976:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513099.029987:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.029995:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.029996:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.030004:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.104.147@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.030014:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.104.147@tcp1 of length 440 into portal 12 MB=0x603d0322d1100 00000400:00000200:20.0:1616513099.030021:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.242.104.147@tcp1 of length 440/440 into md 0x865be291 [1] + 80048 00000400:00000200:20.0:1616513099.030026:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.030029:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513099.030036:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.030038:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.030040:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.030047:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693042590159104 00010000:00000200:28.0:1616513099.030125:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c660000 x1693042590159104/t0(0) o41->2cb7a2ed-8e80-193f-5295-82d3280d5852@10.242.104.147@tcp1:725/0 lens 440/368 e 0 to 0 dl 1616513105 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:28.0:1616513099.030136:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 10, xid 1693042590159104, offset 192 00000400:00000200:28.0:1616513099.030141:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.104.147@tcp1 00000400:00000200:28.0:1616513099.030147:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.104.147@tcp1 routed destination 00000400:00000200:28.0:1616513099.030159:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.104.147@tcp1(10.242.104.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.030165:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.030170:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.030172:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.030174:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.030177:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.030182:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.030236:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.030242:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.030245:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.030252:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.030256:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc5d8 00000800:00000200:26.0:1616513099.030260:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.030263:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.033881:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.033937:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.033947:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.033955:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.52@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.033965:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.52@tcp1 of length 224 into portal 26 MB=0x5f90848448e40 00000400:00000200:0.0:1616513099.033971:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.52@tcp1 of length 224/224 into md 0xf97d0ba9 [1] + 0 00000400:00000200:0.0:1616513099.033976:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da4c1706aa0 00000400:00000200:0.0:1616513099.033979:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.033981:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.033986:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706aa0 00000800:00000200:0.0:1616513099.033989:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.033992:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.033992:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513099.033997:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681188851060288 00000800:00000200:0.0:1616513099.033999:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:8.0:1616513099.034002:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1681188851060288/t0(0) o400->1c2ef9ca-137e-5fb1-721d-69e588985574@10.31.131.52@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:0.0:1616513099.034002:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.034003:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513099.034012:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.52@tcp1 : PUT - for me 00000100:00000200:8.0:1616513099.034012:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681188851060288, offset 192 00000400:00000200:8.0:1616513099.034016:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.52@tcp1 00000400:00000200:24.0:1616513099.034021:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.52@tcp1 of length 224 into portal 12 MB=0x5f90848448ec0 00000400:00000200:8.0:1616513099.034022:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.52@tcp1 routed destination 00000400:00000200:24.0:1616513099.034027:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.52@tcp1 of length 224/224 into md 0x865be291 [1] + 80488 00000400:00000200:24.0:1616513099.034032:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.034034:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.52@tcp1(10.31.131.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513099.034035:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513099.034039:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.034043:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.034045:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.034046:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.034047:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.034048:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:8.0:1616513099.034048:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.034051:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513099.034055:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.034056:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.034068:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.034076:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.034079:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.034081:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.034086:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.034091:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2990 00000800:00000200:20.0:1616513099.034095:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.034106:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681188851060416 00010000:00000200:28.0:1616513099.034118:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b08d100 x1681188851060416/t0(0) o400->4f072141-083e-9cbc-5fa9-a8aaf90e25dd@10.31.131.52@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.034131:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681188851060416, offset 192 00000400:00000200:28.0:1616513099.034137:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.52@tcp1 00000400:00000200:28.0:1616513099.034144:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.52@tcp1 routed destination 00000400:00000200:28.0:1616513099.034158:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.52@tcp1(10.31.131.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.034164:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.034170:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.034172:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.034174:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.034179:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.034185:0:51486:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.034240:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.034246:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.034249:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.034256:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.034260:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc6e8 00000800:00000200:18.0:1616513099.034264:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.034266:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.034272:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513099.034274:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.034277:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.034278:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.034281:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.042105:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.042161:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.042171:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.042179:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.20@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.042190:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.20@tcp1 of length 224 into portal 26 MB=0x5ffe641cde980 00000400:00000200:14.0:1616513099.042196:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.20@tcp1 of length 224/224 into md 0xf97d0bb9 [1] + 0 00000400:00000200:14.0:1616513099.042206:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be330 00000400:00000200:14.0:1616513099.042209:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.042212:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513099.042215:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513099.042216:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be330 00000800:00000200:14.0:1616513099.042219:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.042222:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.042224:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513099.042228:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739295127936 00010000:00000200:8.0:1616513099.042234:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688739295127936/t0(0) o400->a0a1c4ce-0379-2198-9194-c4e96c4dea3d@10.31.142.20@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513099.042244:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688739295127936, offset 192 00000800:00000200:7.2:1616513099.042244:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513099.042249:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.20@tcp1 00000400:00000200:8.0:1616513099.042255:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.20@tcp1 routed destination 00000400:00000200:8.0:1616513099.042268:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.20@tcp1(10.31.142.20@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.042273:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.042277:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.042280:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.042281:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.042284:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513099.042288:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.042291:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513099.042303:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.042313:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513099.042321:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.20@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.042330:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.20@tcp1 of length 224 into portal 12 MB=0x5ffe641cdecc0 00000400:00000200:2.0:1616513099.042337:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.20@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 47824 00000400:00000200:2.0:1616513099.042342:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.042345:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513099.042354:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:2.0:1616513099.042356:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.042359:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:24.0:1616513099.042360:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:2.0:1616513099.042361:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513099.042363:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513099.042364:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:24.0:1616513099.042370:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513099.042375:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2908 00000800:00000200:24.0:1616513099.042378:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.042381:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513099.042426:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688739295128768 00010000:00000200:22.0:1616513099.042454:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647df080 x1688739295128768/t0(0) o400->70e0eb3d-5110-1706-c249-dccd632f56ab@10.31.142.20@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.042469:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688739295128768, offset 192 00000400:00000200:22.0:1616513099.042476:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.20@tcp1 00000400:00000200:22.0:1616513099.042483:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.20@tcp1 routed destination 00000400:00000200:22.0:1616513099.042498:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.20@tcp1(10.31.142.20@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.042504:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.042510:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.042513:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.042515:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.042520:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.042525:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.042535:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.042540:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.042543:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513099.042545:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.042549:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.042553:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bdc0 00000800:00000200:18.0:1616513099.042557:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.044541:0:51487:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513099.044596:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.044606:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513099.044614:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.166@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.044623:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.166@tcp1 of length 224 into portal 26 MB=0x5ffde05ce88c0 00000400:00000200:2.0:1616513099.044630:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.166@tcp1 of length 224/224 into md 0xf97d0bc9 [1] + 0 00000400:00000200:2.0:1616513099.044635:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf7e8 00000400:00000200:2.0:1616513099.044637:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.044640:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513099.044644:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf7e8 00000800:00000200:2.0:1616513099.044647:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.044650:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.044652:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000100:00000200:8.0:1616513099.044654:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703928797376 00000800:00000200:2.0:1616513099.044657:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00010000:00000200:8.0:1616513099.044659:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688703928797376/t0(0) o400->6f9c5ab3-ad73-9fc6-2e1b-b41612523527@10.31.132.166@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:2.0:1616513099.044660:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.044661:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513099.044669:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688703928797376, offset 192 00000400:00000200:26.0:1616513099.044671:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.166@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.044674:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.166@tcp1 00000400:00000200:8.0:1616513099.044679:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.166@tcp1 routed destination 00000400:00000200:26.0:1616513099.044680:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.166@tcp1 of length 224 into portal 12 MB=0x5ffde05ce8940 00000400:00000200:26.0:1616513099.044686:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.166@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 48048 00000400:00000200:26.0:1616513099.044692:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.044692:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.166@tcp1(10.31.132.166@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:26.0:1616513099.044694:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513099.044698:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:26.0:1616513099.044701:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.044702:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513099.044704:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513099.044704:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513099.044705:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513099.044705:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.044708:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513099.044712:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.044713:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513099.044766:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688703928797504 00000800:00000200:0.0:1616513099.044769:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.044776:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:22.0:1616513099.044779:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd218d80 x1688703928797504/t0(0) o400->abf4a986-ad0f-c73e-b326-48f2b7b57cf1@10.31.132.166@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:0.0:1616513099.044779:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.044786:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.044791:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000100:00000200:22.0:1616513099.044793:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688703928797504, offset 192 00000800:00000200:0.0:1616513099.044795:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.044798:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513099.044800:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.166@tcp1 00000400:00000200:22.0:1616513099.044807:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.166@tcp1 routed destination 00000400:00000200:22.0:1616513099.044822:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.166@tcp1(10.31.132.166@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.044829:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.044834:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.044839:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.044841:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.044846:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.044851:0:51487:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.044912:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.044919:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.044922:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513099.044923:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.044930:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.044934:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4af68 00000800:00000200:18.0:1616513099.044938:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.054855:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.054918:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.054937:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513099.054946:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.239@tcp1 : PUT - for me 00000400:00000200:24.0:1616513099.054954:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.239@tcp1 of length 224 into portal 26 MB=0x5ffd43f04b800 00000400:00000200:24.0:1616513099.054960:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.239@tcp1 of length 224/224 into md 0xf97d0bf9 [1] + 0 00000400:00000200:24.0:1616513099.054964:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bea18 00000400:00000200:24.0:1616513099.054967:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513099.054971:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513099.054976:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bea18 00000800:00000200:0.0:1616513099.054977:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.054978:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.054981:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.054983:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.054987:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661938976768 00010000:00000200:8.0:1616513099.054993:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688661938976768/t0(0) o400->0d25ae35-695f-608d-d2f6-c39bef81ea34@10.31.132.239@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513099.055003:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661938976768, offset 192 00000400:00000200:8.0:1616513099.055007:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.239@tcp1 00000400:00000200:8.0:1616513099.055013:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.239@tcp1 routed destination 00000400:00000200:8.0:1616513099.055024:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.239@tcp1(10.31.132.239@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.055029:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.055034:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.055037:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.055038:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.055041:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513099.055045:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.055046:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.055057:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.055062:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.055065:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.055072:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.055077:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3d38 00000800:00000200:0.0:1616513099.055081:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.055083:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.055282:0:51488:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.055339:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.055349:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513099.055357:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.239@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.055365:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.239@tcp1 of length 224 into portal 12 MB=0x5ffd43f04b9c0 00000400:00000200:26.0:1616513099.055372:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.239@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 48272 00000400:00000200:26.0:1616513099.055377:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.055379:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513099.055386:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.055389:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.055391:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.055393:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.055463:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661938977216 00010000:00000200:22.0:1616513099.055475:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21da00 x1688661938977216/t0(0) o400->f8828f70-e209-4c13-88b2-24d7ab6d0d93@10.31.132.239@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.055489:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661938977216, offset 192 00000400:00000200:22.0:1616513099.055496:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.239@tcp1 00000400:00000200:22.0:1616513099.055503:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.239@tcp1 routed destination 00000400:00000200:22.0:1616513099.055518:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.239@tcp1(10.31.132.239@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.055535:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.055540:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.055543:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.055545:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.055549:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.055555:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.055609:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.055616:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513099.055619:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513099.055625:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.055629:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:2.0:1616513099.055633:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.055636:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.059139:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.059192:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.059200:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513099.059209:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.116@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.059218:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.116@tcp1 of length 224 into portal 26 MB=0x5ffde0bbd0340 00000400:00000200:26.0:1616513099.059224:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.116@tcp1 of length 224/224 into md 0xf97d0c09 [1] + 0 00000400:00000200:26.0:1616513099.059228:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be2a8 00000400:00000200:26.0:1616513099.059230:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.059233:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513099.059237:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be2a8 00000800:00000200:26.0:1616513099.059240:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.059242:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.059244:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.059246:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513099.059252:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704028312384 00010000:00000200:8.0:1616513099.059260:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688704028312384/t0(0) o400->b9094fbc-0f19-afcb-d2f4-64cc633f3c54@10.31.142.116@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513099.059272:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704028312384, offset 192 00000400:00000200:8.0:1616513099.059277:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.116@tcp1 00000400:00000200:8.0:1616513099.059283:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.116@tcp1 routed destination 00000400:00000200:8.0:1616513099.059296:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.116@tcp1(10.31.142.116@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.059301:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.059306:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.059308:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.059310:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.059314:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513099.059319:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.059320:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.059374:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.059381:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.059384:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.059391:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.059396:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3d38 00000800:00000200:0.0:1616513099.059400:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.059403:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.061257:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513099.061314:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.061323:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513099.061330:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.116@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.061338:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.116@tcp1 of length 224 into portal 12 MB=0x5ffde0bbd0780 00000400:00000200:2.0:1616513099.061344:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.116@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 48496 00000400:00000200:2.0:1616513099.061349:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.061352:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513099.061357:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.061360:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513099.061362:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.061365:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.061418:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704028313472 00010000:00000200:22.0:1616513099.061429:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647df980 x1688704028313472/t0(0) o400->7041f681-1641-5775-ea34-30746c810903@10.31.142.116@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.061454:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704028313472, offset 192 00000400:00000200:22.0:1616513099.061460:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.116@tcp1 00000400:00000200:22.0:1616513099.061467:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.116@tcp1 routed destination 00000400:00000200:22.0:1616513099.061481:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.116@tcp1(10.31.142.116@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.061487:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.061492:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.061495:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.061497:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.061501:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.061506:0:51489:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.061559:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.061567:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.061570:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.061576:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.061580:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ad48 00000800:00000200:26.0:1616513099.061583:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.061586:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.066541:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929824, portal 4 00000100:00000200:30.0:1616513099.066547:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929824, offset 0 00000400:00000200:30.0:1616513099.066554:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:30.0:1616513099.066560:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:30.0:1616513099.066573:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.066579:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:30.0:1616513099.066586:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.066589:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:30.0:1616513099.066591:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:30.0:1616513099.066596:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513099.066642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.066660:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513099.066666:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513099.066668:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513099.066674:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513099.066680:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74dd21f980 x1693067725929824/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513099.066691:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee770 00000800:00000200:27.0:1616513099.066695:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513099.066697:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513099.066814:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513099.066823:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.066828:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513099.066832:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513099.066835:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513099.066842:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627d60 00000400:00000200:27.0:1616513099.066848:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371676dd [1] + 192 00000400:00000200:27.0:1616513099.066857:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513099.066861:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74dd21f980 x1693067725929824/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513099.066874:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.066883:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513099.066885:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:30.0:1616513099.066892:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74dd21f980 x1693067725929824/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513099.066914:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74dd21f980 x1693067725929824/t0(0) o13->boslfs02-OST0036-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513099.066924:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:7.2:1616513099.075555:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.075612:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.075621:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.075631:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.53@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.075641:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.53@tcp1 of length 224 into portal 26 MB=0x5ffd493f69f80 00000400:00000200:18.0:1616513099.075647:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.53@tcp1 of length 224/224 into md 0xf97d0c19 [1] + 0 00000400:00000200:18.0:1616513099.075652:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be5d8 00000400:00000200:18.0:1616513099.075655:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.075658:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.075662:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be5d8 00000800:00000200:2.0:1616513099.075664:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.075665:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.075668:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.075670:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513099.075680:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663364116352 00010000:00000200:8.0:1616513099.075688:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688663364116352/t0(0) o400->344d2eb8-2e9c-193f-7eb3-4c0e1288eb45@10.31.138.53@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513099.075701:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663364116352, offset 192 00000400:00000200:8.0:1616513099.075706:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.53@tcp1 00000400:00000200:8.0:1616513099.075713:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.53@tcp1 routed destination 00000400:00000200:8.0:1616513099.075727:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.53@tcp1(10.31.138.53@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.075733:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.075738:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.075740:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.075742:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.075746:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513099.075752:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.075753:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.075818:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.075824:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.075827:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.075834:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.075840:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2908 00000800:00000200:0.0:1616513099.075843:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.075846:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.076946:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.076958:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.076963:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.076971:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.53@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.076980:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.53@tcp1 of length 224 into portal 12 MB=0x5ffd493f6a0c0 00000400:00000200:0.0:1616513099.076988:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.53@tcp1 of length 224/224 into md 0x865be291 [1] + 80712 00000400:00000200:0.0:1616513099.076993:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.076996:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.077002:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.077004:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.077006:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.077009:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.077088:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663364116672 00010000:00000200:28.0:1616513099.077100:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b99f80 x1688663364116672/t0(0) o400->5e9d9516-d614-a2da-0ca3-d8ae8869ed09@10.31.138.53@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.077115:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663364116672, offset 192 00000400:00000200:28.0:1616513099.077121:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.53@tcp1 00000400:00000200:28.0:1616513099.077129:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.53@tcp1 routed destination 00000400:00000200:28.0:1616513099.077145:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.53@tcp1(10.31.138.53@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.077152:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.077157:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.077160:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.077162:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.077167:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.077172:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.077226:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.077233:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.077236:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.077243:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.077247:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc198 00000800:00000200:26.0:1616513099.077251:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.077254:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.080650:0:51491:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513099.080705:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.080714:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513099.080723:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.167@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.080734:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.167@tcp1 of length 224 into portal 12 MB=0x5ffd4a5483f80 00000400:00000200:2.0:1616513099.080740:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.167@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 48720 00000400:00000200:2.0:1616513099.080745:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.080748:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513099.080754:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.080757:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.080759:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.080763:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.080802:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513099.080818:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663654678400 00010000:00000200:22.0:1616513099.080831:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647de780 x1688663654678400/t0(0) o400->43c56fe8-2735-9cd0-1ef4-767701a88639@10.31.136.167@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.080856:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663654678400, offset 192 00000400:00000200:22.0:1616513099.080863:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.167@tcp1 00000800:00000200:0.0:1616513099.080868:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513099.080870:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.167@tcp1 routed destination 00000800:00000200:0.0:1616513099.080877:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:22.0:1616513099.080885:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.167@tcp1(10.31.136.167@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.080892:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.080894:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.167@tcp1 : PUT - for me 00000800:00000200:22.0:1616513099.080898:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.080900:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513099.080902:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:0.0:1616513099.080904:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.167@tcp1 of length 224 into portal 26 MB=0x5ffd4a5483f00 00000800:00000200:22.0:1616513099.080907:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:0.0:1616513099.080911:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.167@tcp1 of length 224/224 into md 0xf97d0c39 [1] + 0 00000400:00000200:0.0:1616513099.080928:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bef68 00000800:00000200:20.0:1616513099.080929:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:0.0:1616513099.080930:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.080933:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:20.0:1616513099.080938:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:0.0:1616513099.080938:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bef68 00000400:00000200:20.0:1616513099.080940:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.080940:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513099.080943:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513099.080945:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.080947:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:8.0:1616513099.080947:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663654678272 00000800:00000200:0.0:1616513099.080948:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.080951:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4af68 00010000:00000200:8.0:1616513099.080952:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1688663654678272/t0(0) o400->b7db667a-5779-a8b2-3462-ce9fb9df9d00@10.31.136.167@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513099.080955:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.080962:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663654678272, offset 192 00000400:00000200:8.0:1616513099.080966:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.167@tcp1 00000400:00000200:8.0:1616513099.080972:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.167@tcp1 routed destination 00000400:00000200:8.0:1616513099.080984:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.167@tcp1(10.31.136.167@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.080989:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.080993:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.080995:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.080997:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.081000:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:8.0:1616513099.081004:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513099.081004:0:51491:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.081057:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.081064:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.081067:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.081073:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.081079:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3650 00000800:00000200:26.0:1616513099.081083:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.081086:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.085725:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.085780:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.085790:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513099.085798:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.124@tcp1 : PUT - for me 00000400:00000200:24.0:1616513099.085808:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.124@tcp1 of length 224 into portal 26 MB=0x604a3623a2b40 00000400:00000200:24.0:1616513099.085815:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.124@tcp1 of length 224/224 into md 0xf97d0c49 [1] + 0 00000400:00000200:24.0:1616513099.085820:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02beaa0 00000400:00000200:24.0:1616513099.085822:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513099.085825:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513099.085830:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beaa0 00000800:00000200:24.0:1616513099.085833:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.085835:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.085836:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513099.085839:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.085845:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:8.0:1616513099.085847:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949634423616 00010000:00000200:8.0:1616513099.085855:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1693949634423616/t0(0) o400->fb56d8a0-f7ce-e68b-5d79-991ef6371ded@10.31.142.124@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513099.085867:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693949634423616, offset 192 00000400:00000200:8.0:1616513099.085873:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.124@tcp1 00000400:00000200:8.0:1616513099.085880:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.124@tcp1 routed destination 00000400:00000200:8.0:1616513099.085894:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.124@tcp1(10.31.142.124@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.085916:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.085921:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.085924:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.085925:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.085929:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513099.085933:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:8.0:1616513099.085935:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513099.085942:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513099.085951:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.124@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.085960:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.124@tcp1 of length 224 into portal 12 MB=0x604a3623a2e00 00000400:00000200:20.0:1616513099.085967:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.124@tcp1 of length 224/224 into md 0x865be291 [1] + 80936 00000400:00000200:20.0:1616513099.085972:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.085975:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513099.085981:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:20.0:1616513099.085984:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513099.085986:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.085990:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.085999:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.086002:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.086003:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513099.086008:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.086014:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ee0 00000800:00000200:4.0:1616513099.086018:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.086044:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693949634424320 00010000:00000200:28.0:1616513099.086056:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c662880 x1693949634424320/t0(0) o400->93e86fd7-1a42-3643-39f0-0299740c53f6@10.31.142.124@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.086071:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693949634424320, offset 192 00000400:00000200:28.0:1616513099.086077:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.124@tcp1 00000400:00000200:28.0:1616513099.086085:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.124@tcp1 routed destination 00000400:00000200:28.0:1616513099.086100:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.124@tcp1(10.31.142.124@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.086106:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.086112:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.086115:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.086117:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.086122:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.086128:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.086182:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.086189:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.086191:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.086198:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.086202:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc198 00000800:00000200:18.0:1616513099.086206:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.086209:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.101543:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.101557:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.101566:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513099.101572:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.101575:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.218@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.101586:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.218@tcp1 of length 224 into portal 26 MB=0x604f5f6338700 00000400:00000200:26.0:1616513099.101592:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.218@tcp1 of length 224/224 into md 0xf97d0c59 [1] + 0 00000400:00000200:26.0:1616513099.101597:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be3b8 00000400:00000200:26.0:1616513099.101600:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.101603:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513099.101607:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be3b8 00000800:00000200:26.0:1616513099.101611:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.101613:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.101615:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:8.0:1616513099.101625:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304304334592 00010000:00000200:8.0:1616513099.101634:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1694304304334592/t0(0) o400->4d864bc3-b06d-b62c-b828-d7c17c3a1560@10.31.140.218@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:8.0:1616513099.101647:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694304304334592, offset 192 00000400:00000200:8.0:1616513099.101653:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.218@tcp1 00000400:00000200:8.0:1616513099.101660:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.218@tcp1 routed destination 00000400:00000200:8.0:1616513099.101674:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.218@tcp1(10.31.140.218@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.101701:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.101706:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.101708:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.101710:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.101714:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513099.101720:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.101721:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.101744:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.101751:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.101753:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513099.101760:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513099.101766:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2c38 00000800:00000200:24.0:1616513099.101769:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.101772:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.101892:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.101948:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.101957:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513099.101963:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513099.101966:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.218@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.101975:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.218@tcp1 of length 224 into portal 12 MB=0x604f5f6338840 00000400:00000200:0.0:1616513099.101981:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.218@tcp1 of length 224/224 into md 0x865be291 [1] + 81160 00000400:00000200:0.0:1616513099.101986:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.101989:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.101996:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.101999:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.102000:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.102059:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694304304334912 00010000:00000200:28.0:1616513099.102072:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9d580 x1694304304334912/t0(0) o400->bb661ba6-79b0-e903-d77e-005947123f16@10.31.140.218@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.102086:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694304304334912, offset 192 00000400:00000200:28.0:1616513099.102092:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.218@tcp1 00000400:00000200:28.0:1616513099.102100:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.218@tcp1 routed destination 00000400:00000200:28.0:1616513099.102115:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.218@tcp1(10.31.140.218@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.102122:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.102127:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.102130:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.102134:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.102139:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.102144:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.102160:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.102167:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.102169:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.102176:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.102180:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc6e8 00000800:00000200:18.0:1616513099.102184:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.102187:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.106111:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.106165:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.106174:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.106183:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.42@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.106192:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.42@tcp1 of length 224 into portal 26 MB=0x5f90fb91ee700 00000400:00000200:14.0:1616513099.106199:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.42@tcp1 of length 224/224 into md 0xf97d0c69 [1] + 0 00000400:00000200:14.0:1616513099.106203:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf6d8 00000400:00000200:14.0:1616513099.106206:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.106209:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513099.106213:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf6d8 00000800:00000200:14.0:1616513099.106216:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.106219:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513099.106219:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.106226:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000100:00000200:8.0:1616513099.106228:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681220809189120 00000800:00000200:14.0:1616513099.106229:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.106230:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:8.0:1616513099.106237:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1681220809189120/t0(0) o400->55ed0d37-8354-68fd-749a-338719d28815@10.31.132.42@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:26.0:1616513099.106240:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.42@tcp1 : PUT - for me 00000100:00000200:8.0:1616513099.106248:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681220809189120, offset 192 00000400:00000200:26.0:1616513099.106249:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.42@tcp1 of length 224 into portal 12 MB=0x5f90fb91ee780 00000400:00000200:8.0:1616513099.106253:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.42@tcp1 00000400:00000200:26.0:1616513099.106255:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.42@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 48944 00000400:00000200:26.0:1616513099.106260:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.106260:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.42@tcp1 routed destination 00000100:00000200:26.0:1616513099.106263:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513099.106269:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.106272:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513099.106272:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.42@tcp1(10.31.132.42@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513099.106274:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513099.106278:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.106283:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.106285:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.106287:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.106291:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513099.106296:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.106298:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513099.106333:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681220809189248 00010000:00000200:22.0:1616513099.106345:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21a880 x1681220809189248/t0(0) o400->a48896ee-5d20-43f5-a7cc-7150037a4ac0@10.31.132.42@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513099.106352:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.106359:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:22.0:1616513099.106360:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681220809189248, offset 192 00000400:00000200:24.0:1616513099.106362:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513099.106366:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.42@tcp1 00000400:00000200:24.0:1616513099.106368:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513099.106373:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.42@tcp1 routed destination 00000400:00000200:24.0:1616513099.106374:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3100 00000800:00000200:24.0:1616513099.106378:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.106381:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513099.106388:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.42@tcp1(10.31.132.42@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.106395:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.106401:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.106403:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.106405:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.106410:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.106415:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.106469:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.106477:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513099.106480:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513099.106481:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513099.106486:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.106490:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bdc0 00000800:00000200:2.0:1616513099.106494:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.109904:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513099.109957:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.109967:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.109976:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.212@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.109985:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.212@tcp1 of length 224 into portal 26 MB=0x601a9d3b94140 00000400:00000200:4.0:1616513099.109992:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.212@tcp1 of length 224/224 into md 0xf97d0c79 [1] + 0 00000400:00000200:4.0:1616513099.109997:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be440 00000400:00000200:4.0:1616513099.110000:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.110002:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513099.110007:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be440 00000800:00000200:4.0:1616513099.110011:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.110012:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.110013:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:8.0:1616513099.110016:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690678773498176 00000800:00000200:4.0:1616513099.110021:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00010000:00000200:8.0:1616513099.110022:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1690678773498176/t0(0) o400->ff5a996d-7e98-150a-7f05-4a5e349b9210@10.31.140.212@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:4.0:1616513099.110024:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.110025:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000100:00000200:8.0:1616513099.110032:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690678773498176, offset 192 00000400:00000200:0.0:1616513099.110033:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.212@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.110037:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.212@tcp1 00000400:00000200:8.0:1616513099.110043:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.212@tcp1 routed destination 00000400:00000200:8.0:1616513099.110057:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.212@tcp1(10.31.140.212@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.110065:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.110066:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.212@tcp1 of length 224 into portal 12 MB=0x601a9d3b941c0 00000800:00000200:8.0:1616513099.110069:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.110072:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.110073:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:0.0:1616513099.110073:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.212@tcp1 of length 224/224 into md 0x865be291 [1] + 81384 00000800:00000200:8.0:1616513099.110076:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.110078:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.110080:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.110081:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:0.0:1616513099.110081:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.110087:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513099.110091:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513099.110092:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.110095:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.110100:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.110102:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513099.110104:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513099.110107:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.110112:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf37e8 00000800:00000200:0.0:1616513099.110114:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.110170:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690678773498304 00010000:00000200:28.0:1616513099.110182:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9c800 x1690678773498304/t0(0) o400->eb88c265-3c15-831d-f5db-deb76cc8ad6a@10.31.140.212@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.110197:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690678773498304, offset 192 00000400:00000200:28.0:1616513099.110203:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.212@tcp1 00000400:00000200:28.0:1616513099.110211:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.212@tcp1 routed destination 00000400:00000200:28.0:1616513099.110226:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.212@tcp1(10.31.140.212@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.110232:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.110238:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.110241:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.110243:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.110247:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.110253:0:51494:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.110328:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.110335:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.110337:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.110344:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.110350:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc5d8 00000800:00000200:18.0:1616513099.110354:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.110357:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513099.114521:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929840, portal 4 00000100:00000200:28.0:1616513099.114526:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929840, offset 0 00000400:00000200:28.0:1616513099.114532:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:28.0:1616513099.114537:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:28.0:1616513099.114549:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.114555:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:28.0:1616513099.114561:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.114564:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:28.0:1616513099.114566:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:28.0:1616513099.114570:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513099.114616:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.114635:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513099.114640:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513099.114643:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513099.114650:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513099.114657:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdeb8da00 x1693067725929840/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513099.114669:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc088 00000800:00000200:27.0:1616513099.114673:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513099.114676:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513099.114783:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513099.114793:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.114799:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513099.114802:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513099.114813:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513099.114833:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627d70 00000400:00000200:27.0:1616513099.114839:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371676ed [1] + 192 00000400:00000200:27.0:1616513099.114843:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513099.114848:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdeb8da00 x1693067725929840/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513099.114861:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.114863:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513099.114865:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:28.0:1616513099.114871:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdeb8da00 x1693067725929840/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513099.114882:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdeb8da00 x1693067725929840/t0(0) o13->boslfs02-OST000e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513099.114892:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc2a8 00000800:00000200:7.2:1616513099.127917:0:51496:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.127972:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.127982:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.127991:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.252@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.128002:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.252@tcp1 of length 224 into portal 26 MB=0x5ffe5df0700c0 00000400:00000200:14.0:1616513099.128008:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.252@tcp1 of length 224/224 into md 0xf97d0c89 [1] + 0 00000400:00000200:14.0:1616513099.128013:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf8f8 00000400:00000200:14.0:1616513099.128016:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.128019:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513099.128023:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf8f8 00000800:00000200:2.0:1616513099.128027:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.128029:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.128032:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513099.128034:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.128037:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.128037:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513099.128043:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737637925056 00000400:00000200:2.0:1616513099.128046:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.252@tcp1 : PUT - for me 00010000:00000200:8.0:1616513099.128052:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1688737637925056/t0(0) o400->86ae17a5-c688-8ee2-3a61-4842e9b54a69@10.31.142.252@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:2.0:1616513099.128056:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.252@tcp1 of length 224 into portal 12 MB=0x5ffe5df0703c0 00000400:00000200:2.0:1616513099.128062:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.252@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 49168 00000100:00000200:8.0:1616513099.128067:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688737637925056, offset 192 00000400:00000200:2.0:1616513099.128068:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.128070:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513099.128073:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.252@tcp1 00000800:00000200:2.0:1616513099.128077:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513099.128080:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.252@tcp1 routed destination 00000800:00000200:2.0:1616513099.128080:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.128082:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513099.128095:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.252@tcp1(10.31.142.252@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.128101:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.128107:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.128110:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.128112:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.128116:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513099.128122:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.128123:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513099.128145:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688737637925824 00010000:00000200:22.0:1616513099.128158:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dc800 x1688737637925824/t0(0) o400->e3ca7a1a-65c0-c3d8-c0d3-13871fb8c6be@10.31.142.252@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.128172:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688737637925824, offset 192 00000400:00000200:22.0:1616513099.128178:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.252@tcp1 00000800:00000200:24.0:1616513099.128179:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.128185:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513099.128186:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.252@tcp1 routed destination 00000400:00000200:24.0:1616513099.128190:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513099.128197:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513099.128201:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.252@tcp1(10.31.142.252@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513099.128202:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf27f8 00000800:00000200:24.0:1616513099.128206:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.128209:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513099.128209:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.128215:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.128218:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.128220:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.128225:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.128230:0:51496:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.128285:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.128292:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.128300:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513099.128302:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.128307:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.128311:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b430 00000800:00000200:26.0:1616513099.128315:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.129140:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.129153:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.129163:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513099.129165:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.129172:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.54@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.129181:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.54@tcp1 of length 224 into portal 12 MB=0x5ffe490f27540 00000400:00000200:18.0:1616513099.129187:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.54@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 49392 00000400:00000200:18.0:1616513099.129193:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.129195:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.129202:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.129205:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.129207:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.129213:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688732032988480 00010000:00000200:22.0:1616513099.129219:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bde80 x1688732032988480/t0(0) o400->4df7d946-307b-8187-3749-e664cd13ecda@10.31.140.54@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.129231:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688732032988480, offset 192 00000400:00000200:22.0:1616513099.129236:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.54@tcp1 00000400:00000200:22.0:1616513099.129242:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.54@tcp1 routed destination 00000400:00000200:22.0:1616513099.129255:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.54@tcp1(10.31.140.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.129261:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513099.129266:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.129268:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513099.129270:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513099.129273:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.129279:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.129331:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.129338:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.129340:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.129346:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.129350:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a770 00000800:00000200:0.0:1616513099.129354:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.129357:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.129418:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.129472:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.129481:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513099.129484:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.129490:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.54@tcp1 : PUT - for me 00000400:00000200:24.0:1616513099.129499:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.54@tcp1 of length 224 into portal 26 MB=0x5ffe490f27280 00000400:00000200:24.0:1616513099.129505:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.54@tcp1 of length 224/224 into md 0xf97d0c99 [1] + 0 00000400:00000200:24.0:1616513099.129510:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfdc0 00000400:00000200:24.0:1616513099.129513:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513099.129516:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513099.129521:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfdc0 00000800:00000200:24.0:1616513099.129525:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.129527:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.129529:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513099.129585:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688732032987776 00010000:00000200:12.0:1616513099.129595:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1688732032987776/t0(0) o400->ce31160e-a90d-5e9b-8ee6-7d64444c3c4c@10.31.140.54@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.129610:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688732032987776, offset 192 00000400:00000200:12.0:1616513099.129616:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.54@tcp1 00000400:00000200:12.0:1616513099.129623:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.54@tcp1 routed destination 00000400:00000200:12.0:1616513099.129638:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.54@tcp1(10.31.140.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.129645:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513099.129651:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.129654:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513099.129656:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513099.129660:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.129665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513099.129667:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513099.129718:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.129725:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.129728:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513099.129734:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.129740:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf298 00000800:00000200:14.0:1616513099.129743:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.129746:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:26.0:1616513099.140521:0:6460:0:(o2iblnd_cb.c:3431:kiblnd_check_conns()) conn[ffff8d8cab7e8c00] (20)++ 00000800:00000200:26.0:1616513099.140528:0:6460:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cab7e8c00] (21)++ 00000800:00000200:26.0:1616513099.140533:0:6460:0:(o2iblnd_cb.c:3466:kiblnd_check_conns()) conn[ffff8d8cab7e8c00] (22)-- 00000800:00000200:29.2:1616513099.140581:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cab7e8c00] (21)++ 00000800:00000200:29.0:1616513099.140603:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cab7e8c00] (22)++ 00000800:00000200:29.0:1616513099.140608:0:6465:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cab7e8c00] (23)-- 00000800:00000200:29.0:1616513099.140611:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cab7e8c00] (22)-- 00000800:00000200:29.0:1616513099.140613:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cab7e8c00] (21)-- 00000800:00000200:11.2:1616513099.146997:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.147052:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.147061:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.147070:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.105@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.147080:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.105@tcp1 of length 224 into portal 26 MB=0x5ffdb9e1d6fc0 00000400:00000200:0.0:1616513099.147087:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.105@tcp1 of length 224/224 into md 0xf97d0cb9 [1] + 0 00000400:00000200:0.0:1616513099.147092:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf430 00000400:00000200:0.0:1616513099.147095:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.147098:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.147102:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf430 00000800:00000200:0.0:1616513099.147105:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.147106:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.147108:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.147112:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513099.147168:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693599203264 00010000:00000200:12.0:1616513099.147180:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688693599203264/t0(0) o400->b42741b1-d59f-bc83-5037-53ef758fcc94@10.31.136.105@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.147194:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688693599203264, offset 192 00000400:00000200:12.0:1616513099.147200:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.105@tcp1 00000400:00000200:12.0:1616513099.147208:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.105@tcp1 routed destination 00000400:00000200:12.0:1616513099.147223:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.105@tcp1(10.31.136.105@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.147230:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513099.147236:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.147238:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513099.147241:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513099.147246:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.147250:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513099.147252:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513099.147304:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.147311:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.147314:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.147320:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.147326:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bec38 00000800:00000200:4.0:1616513099.147329:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.147332:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.147400:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513099.147412:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.147420:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513099.147422:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.147429:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.105@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.147452:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.105@tcp1 of length 224 into portal 12 MB=0x5ffdb9e1d7080 00000400:00000200:20.0:1616513099.147458:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.105@tcp1 of length 224/224 into md 0x865be291 [1] + 81608 00000400:00000200:20.0:1616513099.147464:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.147466:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513099.147473:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.147476:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.147478:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.147537:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688693599203456 00010000:00000200:28.0:1616513099.147550:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663180 x1688693599203456/t0(0) o400->bcaa42ac-df65-ef6c-0df3-14040ebf495e@10.31.136.105@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.147564:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688693599203456, offset 192 00000400:00000200:28.0:1616513099.147569:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.105@tcp1 00000400:00000200:28.0:1616513099.147576:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.105@tcp1 routed destination 00000400:00000200:28.0:1616513099.147590:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.105@tcp1(10.31.136.105@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.147596:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.147601:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.147604:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.147606:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.147610:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.147617:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.147674:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.147681:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513099.147684:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.147690:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513099.147690:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.147694:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc2a8 00000800:00000200:2.0:1616513099.147698:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.152772:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513099.152826:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.152835:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.152844:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.170@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.152852:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.170@tcp1 of length 224 into portal 26 MB=0x5ffde3356cb40 00000400:00000200:4.0:1616513099.152862:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.170@tcp1 of length 224/224 into md 0xf97d0cc9 [1] + 0 00000400:00000200:4.0:1616513099.152866:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf4b8 00000400:00000200:4.0:1616513099.152869:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.152872:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513099.152876:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf4b8 00000800:00000200:24.0:1616513099.152879:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.152880:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.152883:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.152884:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513099.152938:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704692702016 00010000:00000200:12.0:1616513099.152950:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1688704692702016/t0(0) o400->4b257fad-2303-6285-7566-7f7b1ec8d13a@10.31.132.170@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.152964:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704692702016, offset 192 00000400:00000200:12.0:1616513099.152972:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.170@tcp1 00000400:00000200:12.0:1616513099.152979:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.170@tcp1 routed destination 00000400:00000200:12.0:1616513099.152994:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.170@tcp1(10.31.132.170@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.153001:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513099.153006:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.153009:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513099.153011:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:11.2:1616513099.153012:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513099.153016:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:12.0:1616513099.153023:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513099.153066:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.153076:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.153084:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.170@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.153092:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.170@tcp1 of length 224 into portal 12 MB=0x5ffde3356cc40 00000400:00000200:0.0:1616513099.153099:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.170@tcp1 of length 224/224 into md 0x865be291 [1] + 81832 00000400:00000200:0.0:1616513099.153104:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.153106:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.153113:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513099.153116:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513099.153118:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513099.153120:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.153129:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.153131:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513099.153132:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.153138:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.153144:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf650 00000800:00000200:20.0:1616513099.153147:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.153176:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704692702272 00010000:00000200:28.0:1616513099.153188:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9b600 x1688704692702272/t0(0) o400->af1480a9-a2af-17f8-b9d9-3d3ac769b41a@10.31.132.170@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.153203:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704692702272, offset 192 00000400:00000200:28.0:1616513099.153209:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.170@tcp1 00000400:00000200:28.0:1616513099.153216:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.170@tcp1 routed destination 00000400:00000200:28.0:1616513099.153231:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.170@tcp1(10.31.132.170@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.153238:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.153243:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.153246:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.153248:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.153253:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.153258:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.153312:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.153318:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.153321:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513099.153327:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.153331:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc3b8 00000800:00000200:14.0:1616513099.153334:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.153337:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.154857:0:51499:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.154913:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.154922:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.154931:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.101@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.154941:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.101@tcp1 of length 224 into portal 26 MB=0x6018321eab140 00000400:00000200:18.0:1616513099.154948:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.101@tcp1 of length 224/224 into md 0xf97d0ce9 [1] + 0 00000400:00000200:18.0:1616513099.154952:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf320 00000400:00000200:18.0:1616513099.154955:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.154958:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.154962:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf320 00000800:00000200:18.0:1616513099.154965:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.154968:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.154970:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.154973:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513099.155025:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512581636416 00010000:00000200:12.0:1616513099.155036:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1690512581636416/t0(0) o400->adb27f66-7339-4867-8775-37d28e80b640@10.31.136.101@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.155050:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512581636416, offset 192 00000400:00000200:12.0:1616513099.155056:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.101@tcp1 00000400:00000200:12.0:1616513099.155064:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.101@tcp1 routed destination 00000400:00000200:12.0:1616513099.155079:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.101@tcp1(10.31.136.101@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.155085:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513099.155091:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.155093:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513099.155095:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513099.155100:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.155105:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513099.155107:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513099.155158:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.155164:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.155167:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.155174:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.155179:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be6e8 00000800:00000200:4.0:1616513099.155183:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.155186:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.155311:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.155366:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.155375:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513099.155384:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.101@tcp1 : PUT - for me 00000400:00000200:24.0:1616513099.155393:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.101@tcp1 of length 224 into portal 12 MB=0x6018321eab300 00000400:00000200:24.0:1616513099.155399:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.101@tcp1 of length 224/224 into md 0x865be291 [1] + 82056 00000400:00000200:24.0:1616513099.155405:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513099.155407:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513099.155415:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.155418:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513099.155419:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513099.155423:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.155479:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512581636864 00010000:00000200:28.0:1616513099.155492:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d85193ed580 x1690512581636864/t0(0) o400->493a1a22-b11f-afb5-b537-4c08c733f43f@10.31.136.101@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.155506:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512581636864, offset 192 00000400:00000200:28.0:1616513099.155512:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.101@tcp1 00000400:00000200:28.0:1616513099.155519:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.101@tcp1 routed destination 00000400:00000200:28.0:1616513099.155534:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.101@tcp1(10.31.136.101@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.155542:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.155548:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.155551:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.155553:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.155558:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.155564:0:51499:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.155619:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.155626:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.155629:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513099.155635:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.155639:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc7f8 00000800:00000200:14.0:1616513099.155643:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.155646:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.162144:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513099.162199:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.162209:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.162217:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.238@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.162226:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.238@tcp1 of length 224 into portal 26 MB=0x5ffd440bc02c0 00000400:00000200:4.0:1616513099.162233:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.238@tcp1 of length 224/224 into md 0xf97d0cf9 [1] + 0 00000400:00000200:4.0:1616513099.162238:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be4c8 00000400:00000200:4.0:1616513099.162240:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.162243:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513099.162247:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be4c8 00000800:00000200:4.0:1616513099.162250:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.162253:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.162254:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.162258:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513099.162310:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661967766208 00010000:00000200:12.0:1616513099.162321:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1688661967766208/t0(0) o400->2e942272-7d96-4510-f646-f43ab7369199@10.31.132.238@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.162335:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661967766208, offset 192 00000400:00000200:12.0:1616513099.162341:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.238@tcp1 00000800:00000200:7.2:1616513099.162348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:12.0:1616513099.162349:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.238@tcp1 routed destination 00000400:00000200:12.0:1616513099.162364:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.238@tcp1(10.31.132.238@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.162375:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513099.162380:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.162383:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513099.162385:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513099.162390:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.162395:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513099.162396:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513099.162405:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.162414:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513099.162423:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.238@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.162431:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.238@tcp1 of length 224 into portal 12 MB=0x5ffd440bc03c0 00000800:00000200:20.0:1616513099.162450:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513099.162453:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.238@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 49616 00000800:00000200:20.0:1616513099.162457:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:2.0:1616513099.162459:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.162460:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.162461:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.162463:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513099.162467:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:2.0:1616513099.162468:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.162471:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513099.162472:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf980 00000800:00000200:2.0:1616513099.162473:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513099.162476:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.162479:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:22.0:1616513099.162542:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661967766464 00010000:00000200:22.0:1616513099.162555:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dbf00 x1688661967766464/t0(0) o400->d8f2335c-59ae-d41b-37a6-2ad596fe1343@10.31.132.238@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.162569:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661967766464, offset 192 00000400:00000200:22.0:1616513099.162575:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.238@tcp1 00000400:00000200:22.0:1616513099.162582:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.238@tcp1 routed destination 00000400:00000200:22.0:1616513099.162603:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.238@tcp1(10.31.132.238@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.162609:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.162615:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.162618:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.162620:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.162625:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.162629:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.162690:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.162696:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.162699:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513099.162706:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.162710:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b8f8 00000800:00000200:14.0:1616513099.162714:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.162717:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.168665:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.168717:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.168726:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.168734:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.4@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.168742:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.4@tcp1 of length 224 into portal 26 MB=0x5ffe77f5c0b00 00000400:00000200:0.0:1616513099.168748:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.4@tcp1 of length 224/224 into md 0xf97d0d19 [1] + 0 00000400:00000200:0.0:1616513099.168753:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf100 00000400:00000200:0.0:1616513099.168755:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.168758:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.168762:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf100 00000800:00000200:0.0:1616513099.168764:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.168767:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.168769:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.168771:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.168781:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.168789:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.4@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.168797:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.4@tcp1 of length 224 into portal 12 MB=0x5ffe77f5c0c00 00000400:00000200:4.0:1616513099.168803:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.4@tcp1 of length 224/224 into md 0x865be291 [1] + 82280 00000400:00000200:4.0:1616513099.168808:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.168813:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513099.168819:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:12.0:1616513099.168822:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688744622820096 00000800:00000200:4.0:1616513099.168822:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.168824:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.168827:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:12.0:1616513099.168832:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1688744622820096/t0(0) o400->dc93886c-707c-e5cb-d2d5-d68dd54273ea@10.31.142.4@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.168845:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688744622820096, offset 192 00000400:00000200:12.0:1616513099.168851:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.4@tcp1 00000400:00000200:12.0:1616513099.168858:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.4@tcp1 routed destination 00000400:00000200:12.0:1616513099.168872:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.4@tcp1(10.31.142.4@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.168878:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513099.168883:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000100:00000200:28.0:1616513099.168884:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688744622820352 00000800:00000200:12.0:1616513099.168886:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513099.168888:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513099.168892:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:28.0:1616513099.168896:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb238000 x1688744622820352/t0(0) o400->f7c4dba8-9e74-a3d3-0991-a2c1683d6c9d@10.31.142.4@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513099.168896:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513099.168898:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:28.0:1616513099.168911:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688744622820352, offset 192 00000400:00000200:28.0:1616513099.168917:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.4@tcp1 00000400:00000200:28.0:1616513099.168924:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.4@tcp1 routed destination 00000400:00000200:28.0:1616513099.168939:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.4@tcp1(10.31.142.4@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.168946:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.168949:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:28.0:1616513099.168952:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.168954:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.168956:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.168956:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.168959:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.168960:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513099.168961:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:24.0:1616513099.168966:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513099.168968:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513099.168973:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beee0 00000800:00000200:24.0:1616513099.168976:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513099.169020:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.169027:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.169029:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.169035:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.169039:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc908 00000800:00000200:18.0:1616513099.169045:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.169047:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.173504:0:51501:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513099.173515:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513099.173559:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.173568:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:4.0:1616513099.173576:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513099.173577:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.182@tcp1 : PUT - for me 00000800:00000200:4.0:1616513099.173586:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:14.0:1616513099.173588:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.182@tcp1 of length 224 into portal 26 MB=0x60182cbd05700 00000400:00000200:14.0:1616513099.173595:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.182@tcp1 of length 224/224 into md 0xf97d0d39 [1] + 0 00000400:00000200:4.0:1616513099.173595:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.182@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.173600:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be908 00000400:00000200:14.0:1616513099.173602:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.173604:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.182@tcp1 of length 224 into portal 12 MB=0x60182cbd05800 00000100:00000200:14.0:1616513099.173605:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513099.173609:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be908 00000400:00000200:4.0:1616513099.173610:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.182@tcp1 of length 224/224 into md 0x865be291 [1] + 82504 00000800:00000200:14.0:1616513099.173612:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.173613:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.173615:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000400:00000200:4.0:1616513099.173615:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:14.0:1616513099.173618:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:4.0:1616513099.173618:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513099.173624:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.173627:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.173629:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.173632:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513099.173691:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511137068800 00000100:00000200:28.0:1616513099.173693:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690511137069056 00010000:00000200:12.0:1616513099.173703:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1690511137068800/t0(0) o400->6f45ea69-91c1-46c4-8f79-2a1ee50b9ffb@10.31.132.182@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:28.0:1616513099.173705:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb238900 x1690511137069056/t0(0) o400->86e386e5-191b-6d80-42af-cff6ca42415e@10.31.132.182@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:12.0:1616513099.173717:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690511137068800, offset 192 00000100:00000200:28.0:1616513099.173720:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690511137069056, offset 192 00000400:00000200:12.0:1616513099.173723:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.182@tcp1 00000400:00000200:28.0:1616513099.173729:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.182@tcp1 00000400:00000200:12.0:1616513099.173731:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.182@tcp1 routed destination 00000400:00000200:12.0:1616513099.173745:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.182@tcp1(10.31.132.182@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.173752:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513099.173754:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.182@tcp1 routed destination 00000800:00000200:12.0:1616513099.173762:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.173764:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513099.173768:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513099.173769:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.182@tcp1(10.31.132.182@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.173773:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513099.173776:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513099.173778:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513099.173780:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513099.173782:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.173785:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.173787:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.173792:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.173796:0:51501:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513099.173833:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.173840:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.173852:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.173853:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513099.173859:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513099.173860:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.173862:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.173864:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfcb0 00000800:00000200:0.0:1616513099.173868:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513099.173869:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513099.173870:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513099.173873:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc4c8 00000800:00000200:18.0:1616513099.173877:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.173880:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.178395:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513099.178453:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.178472:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.178480:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.207@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.178489:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.207@tcp1 of length 224 into portal 12 MB=0x5ffe487b20fc0 00000400:00000200:4.0:1616513099.178494:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.207@tcp1 of length 224/224 into md 0x865be291 [1] + 82728 00000400:00000200:4.0:1616513099.178499:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.178502:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513099.178508:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.178510:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.178512:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.178515:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.178569:0:188409:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731877773248 00010000:00000200:28.0:1616513099.178580:0:188409:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23e300 x1688731877773248/t0(0) o400->16c45e79-e258-c76c-ec59-03e4c1ea3f7a@10.31.142.207@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513099.178587:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:28.0:1616513099.178593:0:188409:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688731877773248, offset 192 00000400:00000200:28.0:1616513099.178599:0:188409:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.207@tcp1 00000400:00000200:28.0:1616513099.178606:0:188409:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.207@tcp1 routed destination 00000400:00000200:28.0:1616513099.178619:0:188409:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.207@tcp1(10.31.142.207@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.178626:0:188409:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513099.178631:0:188409:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.178633:0:188409:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513099.178635:0:188409:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513099.178640:0:188409:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513099.178643:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513099.178645:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513099.178653:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513099.178661:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.207@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.178670:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.207@tcp1 of length 224 into portal 26 MB=0x5ffe487b20f40 00000400:00000200:26.0:1616513099.178677:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.207@tcp1 of length 224/224 into md 0xf97d0d59 [1] + 0 00000400:00000200:26.0:1616513099.178682:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf5c8 00000400:00000200:26.0:1616513099.178684:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.178687:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513099.178691:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf5c8 00000800:00000200:26.0:1616513099.178694:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.178697:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.178697:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513099.178699:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513099.178701:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513099.178705:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.178708:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.178714:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.178718:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcca18 00000800:00000200:0.0:1616513099.178721:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.178724:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513099.178753:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688731877773120 00010000:00000200:12.0:1616513099.178763:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688731877773120/t0(0) o400->cd1073ef-189e-0123-bf5f-c2fec4b2a0b2@10.31.142.207@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.178777:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688731877773120, offset 192 00000400:00000200:12.0:1616513099.178782:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.207@tcp1 00000400:00000200:12.0:1616513099.178789:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.207@tcp1 routed destination 00000400:00000200:12.0:1616513099.178803:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.207@tcp1(10.31.142.207@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.178809:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513099.178815:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.178817:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513099.178819:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513099.178824:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.178828:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:12.0:1616513099.178830:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513099.178883:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.178889:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.178892:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.178899:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.178904:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfa90 00000800:00000200:18.0:1616513099.178908:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.178911:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.187457:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.187511:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.187520:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.187528:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.218@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.187538:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.218@tcp1 of length 224 into portal 26 MB=0x5fffcfdf21f80 00000400:00000200:14.0:1616513099.187544:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.218@tcp1 of length 224/224 into md 0xf97d0d79 [1] + 0 00000400:00000200:14.0:1616513099.187548:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf188 00000400:00000200:14.0:1616513099.187550:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.187553:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513099.187557:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf188 00000800:00000200:14.0:1616513099.187560:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.187562:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.187564:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.187568:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513099.187618:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688836940898176 00010000:00000200:12.0:1616513099.187628:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688836940898176/t0(0) o400->76293125-6011-6399-c38c-77149b866ca0@10.31.136.218@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.187641:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688836940898176, offset 192 00000400:00000200:12.0:1616513099.187647:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.218@tcp1 00000400:00000200:12.0:1616513099.187654:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.218@tcp1 routed destination 00000400:00000200:12.0:1616513099.187668:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.218@tcp1(10.31.136.218@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.187674:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513099.187679:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.187682:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513099.187684:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513099.187688:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.187693:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513099.187694:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513099.187763:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.187769:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.187772:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.187779:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.187784:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bff58 00000800:00000200:20.0:1616513099.187788:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.187791:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.187922:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.187980:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.187990:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.187998:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.218@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.188007:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.218@tcp1 of length 224 into portal 12 MB=0x5fffcfdf22140 00000400:00000200:18.0:1616513099.188014:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.218@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 49840 00000400:00000200:18.0:1616513099.188019:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.188022:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.188028:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.188031:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.188033:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.188036:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.188100:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688836940898624 00010000:00000200:22.0:1616513099.188112:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bf500 x1688836940898624/t0(0) o400->6ef5b0ae-c1b9-82fb-289e-e459ca0ff0a6@10.31.136.218@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.188127:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688836940898624, offset 192 00000400:00000200:22.0:1616513099.188133:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.218@tcp1 00000400:00000200:22.0:1616513099.188140:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.218@tcp1 routed destination 00000400:00000200:22.0:1616513099.188155:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.218@tcp1(10.31.136.218@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.188162:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.188168:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.188170:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.188172:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.188177:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.188182:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.188193:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.188198:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.188202:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513099.188202:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.188209:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.188213:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a7f8 00000800:00000200:26.0:1616513099.188217:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.188251:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.188260:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.188265:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513099.188268:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.188272:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.200@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.188279:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.200@tcp1 of length 224 into portal 26 MB=0x5ffd4a9e8c8c0 00000400:00000200:18.0:1616513099.188284:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.200@tcp1 of length 224/224 into md 0xf97d0d89 [1] + 0 00000400:00000200:18.0:1616513099.188288:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfb18 00000400:00000200:18.0:1616513099.188290:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.188294:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.188298:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfb18 00000800:00000200:18.0:1616513099.188300:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.188301:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.188303:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:12.0:1616513099.188309:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663732308160 00010000:00000200:12.0:1616513099.188314:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1688663732308160/t0(0) o400->bda26bab-021f-b72d-95e3-e212a1458d4b@10.31.136.200@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.188326:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663732308160, offset 192 00000400:00000200:12.0:1616513099.188330:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.200@tcp1 00000400:00000200:12.0:1616513099.188335:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.200@tcp1 routed destination 00000400:00000200:12.0:1616513099.188346:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.200@tcp1(10.31.136.200@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.188351:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:12.0:1616513099.188355:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.188357:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:12.0:1616513099.188359:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:12.0:1616513099.188361:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513099.188365:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.188366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.188421:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.188428:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.188430:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.188449:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.188455:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be880 00000800:00000200:0.0:1616513099.188458:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.188461:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.188649:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513099.188659:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.188665:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513099.188668:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.188673:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.224@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.188682:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.224@tcp1 of length 224 into portal 26 MB=0x5ffd4e8f408c0 00000400:00000200:20.0:1616513099.188688:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.224@tcp1 of length 224/224 into md 0xf97d0da9 [1] + 0 00000400:00000200:20.0:1616513099.188693:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bee58 00000400:00000200:20.0:1616513099.188695:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.188698:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513099.188702:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bee58 00000800:00000200:20.0:1616513099.188705:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.188707:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.188709:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513099.188712:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664790010048 00010000:00000200:12.0:1616513099.188716:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688664790010048/t0(0) o400->4b375354-1b75-1ba4-76f3-27a95bbc0b9c@10.31.132.224@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.188725:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664790010048, offset 192 00000400:00000200:12.0:1616513099.188729:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.224@tcp1 00000400:00000200:12.0:1616513099.188734:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.224@tcp1 routed destination 00000400:00000200:12.0:1616513099.188745:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.224@tcp1(10.31.132.224@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.188750:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513099.188754:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.188756:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513099.188757:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513099.188760:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513099.188763:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513099.188765:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.188773:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.188777:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.188780:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513099.188781:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513099.188785:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.188790:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf870 00000800:00000200:14.0:1616513099.188792:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.188815:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513099.188827:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.188834:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513099.188835:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513099.188843:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.224@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.188852:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.224@tcp1 of length 224 into portal 12 MB=0x5ffd4e8f40940 00000400:00000200:2.0:1616513099.188858:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.224@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 50064 00000400:00000200:2.0:1616513099.188863:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.188866:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513099.188873:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.188876:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.188878:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.188884:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664790010176 00010000:00000200:22.0:1616513099.188891:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8900 x1688664790010176/t0(0) o400->38b739d1-d3e0-25dd-9f2f-df3274750291@10.31.132.224@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.188902:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664790010176, offset 192 00000400:00000200:22.0:1616513099.188907:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.224@tcp1 00000400:00000200:22.0:1616513099.188913:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.224@tcp1 routed destination 00000400:00000200:22.0:1616513099.188926:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.224@tcp1(10.31.132.224@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.188932:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513099.188937:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.188939:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513099.188941:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513099.188945:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.188950:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.188959:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.188963:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.188965:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513099.188969:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513099.188970:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.188973:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b210 00000800:00000200:0.0:1616513099.188976:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.189028:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.189037:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.189043:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513099.189045:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.189051:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.200@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.189059:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.200@tcp1 of length 224 into portal 12 MB=0x5ffd4a9e8c940 00000400:00000200:26.0:1616513099.189065:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.200@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 50288 00000400:00000200:26.0:1616513099.189070:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.189073:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513099.189079:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.189081:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.189083:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.189089:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663732308288 00010000:00000200:22.0:1616513099.189095:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21de80 x1688663732308288/t0(0) o400->3532d29d-a24d-c7f1-c0c5-76391112a138@10.31.136.200@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.189105:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663732308288, offset 192 00000400:00000200:22.0:1616513099.189110:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.200@tcp1 00000400:00000200:22.0:1616513099.189115:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.200@tcp1 routed destination 00000400:00000200:22.0:1616513099.189130:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.200@tcp1(10.31.136.200@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.189135:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.189140:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.189142:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.189143:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.189146:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.189151:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.189159:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.189162:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.189164:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513099.189167:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.189169:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.189172:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a220 00000800:00000200:18.0:1616513099.189175:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.192758:0:51503:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.192812:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.192821:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.192830:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.56@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.192843:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.56@tcp1 of length 224 into portal 12 MB=0x5ffdaf65f3a80 00000400:00000200:14.0:1616513099.192849:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.56@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 50512 00000400:00000200:14.0:1616513099.192854:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.192856:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513099.192862:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.192865:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.192865:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.192869:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.192924:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688690784942720 00010000:00000200:22.0:1616513099.192937:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebf500 x1688690784942720/t0(0) o400->10468d6f-7217-2404-2f97-cf126b151746@10.31.140.56@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.192951:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688690784942720, offset 192 00000400:00000200:22.0:1616513099.192957:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.56@tcp1 00000400:00000200:22.0:1616513099.192965:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.56@tcp1 routed destination 00000400:00000200:22.0:1616513099.192980:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.56@tcp1(10.31.140.56@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.192987:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:22.0:1616513099.192992:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.192995:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513099.192997:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513099.193002:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.193006:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.193058:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.193065:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.193068:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.193075:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.193078:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b100 00000800:00000200:4.0:1616513099.193083:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.193085:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.193092:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.193100:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.56@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.193109:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.56@tcp1 of length 224 into portal 26 MB=0x5ffdaf65f3880 00000800:00000200:20.0:1616513099.193112:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513099.193115:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.56@tcp1 of length 224/224 into md 0xf97d0dc9 [1] + 0 00000400:00000200:4.0:1616513099.193120:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be990 00000400:00000200:4.0:1616513099.193122:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.193124:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513099.193128:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be990 00000800:00000200:4.0:1616513099.193131:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.193134:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.193135:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:12.0:1616513099.193202:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688690784942208 00010000:00000200:12.0:1616513099.193211:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688690784942208/t0(0) o400->7732c16b-a0bc-2ed1-4e5b-118e6944244c@10.31.140.56@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:12.0:1616513099.193223:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688690784942208, offset 192 00000400:00000200:12.0:1616513099.193228:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.56@tcp1 00000400:00000200:12.0:1616513099.193235:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.56@tcp1 routed destination 00000400:00000200:12.0:1616513099.193249:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.56@tcp1(10.31.140.56@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:12.0:1616513099.193255:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:12.0:1616513099.193260:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.193262:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513099.193264:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513099.193269:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:12.0:1616513099.193274:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513099.193274:0:51503:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.193329:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.193336:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513099.193339:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.193341:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513099.193346:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.193351:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be110 00000800:00000200:2.0:1616513099.193355:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.203159:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.203215:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.203224:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.203234:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.100@tcp1 : PUT - for me 00000800:00000200:7.2:1616513099.203250:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513099.203250:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.100@tcp1 of length 224 into portal 26 MB=0x5ffdbe37a6a80 00000400:00000200:0.0:1616513099.203256:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.100@tcp1 of length 224/224 into md 0xf97d0dd9 [1] + 0 00000400:00000200:0.0:1616513099.203261:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be7f8 00000400:00000200:0.0:1616513099.203263:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.203266:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.203271:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be7f8 00000800:00000200:0.0:1616513099.203274:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.203277:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.203279:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.203281:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.203286:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513099.203287:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513099.203294:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.174@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.203301:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.174@tcp1 of length 224 into portal 26 MB=0x5ffd48ea6f1c0 00000400:00000200:0.0:1616513099.203306:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.174@tcp1 of length 224/224 into md 0xf97d0de9 [1] + 0 00000400:00000200:0.0:1616513099.203309:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfed0 00000400:00000200:0.0:1616513099.203311:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.203313:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.203321:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfed0 00000800:00000200:0.0:1616513099.203322:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.203324:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.203325:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.203330:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513099.203331:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513099.203332:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.203338:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513099.203340:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:4.0:1616513099.203340:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513099.203344:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694762924672 00000400:00000200:0.0:1616513099.203345:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.100@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.203349:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.174@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.203351:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.100@tcp1 of length 224 into portal 12 MB=0x5ffdbe37a6c40 00010000:00000200:12.0:1616513099.203354:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688694762924672/t0(0) o400->d6f437c7-1958-f14d-0d8e-174519f8cd5c@10.31.136.100@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:0.0:1616513099.203356:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.100@tcp1 of length 224/224 into md 0x865be291 [1] + 82952 00000400:00000200:26.0:1616513099.203358:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.174@tcp1 of length 224 into portal 12 MB=0x5ffd48ea6f500 00000400:00000200:0.0:1616513099.203359:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.203361:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:26.0:1616513099.203365:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.174@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 50736 00000800:00000200:0.0:1616513099.203367:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:12.0:1616513099.203368:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694762924672, offset 192 00000400:00000200:26.0:1616513099.203370:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.203371:0:6464:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:26.0:1616513099.203373:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:12.0:1616513099.203373:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.100@tcp1 00000100:00000200:16.0:1616513099.203374:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663275008448 00000800:00000200:0.0:1616513099.203374:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.203376:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513099.203379:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513099.203379:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513099.203380:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.100@tcp1 routed destination 00000800:00000200:26.0:1616513099.203382:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.203384:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00010000:00000200:16.0:1616513099.203386:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688663275008448/t0(0) o400->dc15b231-f190-c8d0-0ae8-1a7e37ff6d16@10.31.129.174@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513099.203387:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:12.0:1616513099.203393:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.100@tcp1(10.31.136.100@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:8.0:1616513099.203394:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694762925120 00000800:00000200:12.0:1616513099.203399:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513099.203399:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:16.0:1616513099.203401:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663275008448, offset 192 00000800:00000200:12.0:1616513099.203404:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:8.0:1616513099.203405:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9cc80 x1688694762925120/t0(0) o400->775ffaff-4345-93a2-5784-6c96430a772c@10.31.136.100@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:4.0:1616513099.203405:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:12.0:1616513099.203406:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:16.0:1616513099.203407:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.174@tcp1 00000800:00000200:24.0:1616513099.203408:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513099.203408:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:12.0:1616513099.203412:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.203412:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:16.0:1616513099.203414:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.174@tcp1 routed destination 00000800:00000200:11.2:1616513099.203417:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:8.0:1616513099.203417:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694762925120, offset 192 00000100:00000200:12.0:1616513099.203419:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:8.0:1616513099.203423:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.100@tcp1 00000800:00000200:0.0:1616513099.203427:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513099.203430:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.174@tcp1(10.31.129.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:8.0:1616513099.203430:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.100@tcp1 routed destination 00000800:00000200:0.0:1616513099.203431:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.203442:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513099.203445:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513099.203448:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513099.203449:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.100@tcp1(10.31.136.100@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:0.0:1616513099.203452:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfd38 00000800:00000200:16.0:1616513099.203453:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.203455:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:0.0:1616513099.203455:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513099.203459:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000100:00000200:22.0:1616513099.203460:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663275009280 00000800:00000200:8.0:1616513099.203460:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.203462:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.203462:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.203464:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.203465:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.203469:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513099.203469:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513099.203472:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21c800 x1688663275009280/t0(0) o400->12b574a5-ab14-551d-8636-5db4d7fab519@10.31.129.174@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:11.2:1616513099.203474:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513099.203476:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513099.203478:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513099.203484:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:22.0:1616513099.203486:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663275009280, offset 192 00000800:00000200:24.0:1616513099.203490:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.203492:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513099.203493:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.174@tcp1 00000800:00000200:4.0:1616513099.203494:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.203499:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:22.0:1616513099.203500:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.174@tcp1 routed destination 00000400:00000200:24.0:1616513099.203503:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3430 00000800:00000200:24.0:1616513099.203507:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513099.203510:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:22.0:1616513099.203515:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.174@tcp1(10.31.129.174@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:18.0:1616513099.203517:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.203519:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513099.203521:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.203525:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:22.0:1616513099.203527:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.203530:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:18.0:1616513099.203530:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788e58 00000800:00000200:22.0:1616513099.203534:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513099.203534:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.203537:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513099.203539:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.203544:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.203555:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.203559:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.203561:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513099.203565:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.203567:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.203570:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a440 00000800:00000200:26.0:1616513099.203574:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.208867:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.208918:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.208927:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.208935:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.119@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.208943:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.119@tcp1 of length 224 into portal 26 MB=0x5ffddac38de80 00000400:00000200:18.0:1616513099.208949:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.119@tcp1 of length 224/224 into md 0xf97d0e09 [1] + 0 00000400:00000200:18.0:1616513099.208954:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfba0 00000400:00000200:18.0:1616513099.208957:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.208959:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.208963:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfba0 00000800:00000200:18.0:1616513099.208966:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.208968:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.208970:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:18.0:1616513099.208976:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.208979:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.208980:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.208988:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.119@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.208996:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.119@tcp1 of length 224 into portal 12 MB=0x5ffddac38e180 00000400:00000200:14.0:1616513099.209002:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.119@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 50960 00000400:00000200:14.0:1616513099.209007:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.209009:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513099.209015:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.209018:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.209020:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.209026:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702425816704 00010000:00000200:16.0:1616513099.209037:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688702425816704/t0(0) o400->8477f066-53a5-6082-5d79-832969509c1c@10.31.142.119@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.209051:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702425816704, offset 192 00000400:00000200:16.0:1616513099.209058:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.119@tcp1 00000400:00000200:16.0:1616513099.209065:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.119@tcp1 routed destination 00000100:00000200:22.0:1616513099.209075:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702425817472 00000400:00000200:16.0:1616513099.209080:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.119@tcp1(10.31.142.119@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:22.0:1616513099.209087:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebe300 x1688702425817472/t0(0) o400->4fc03b4b-e354-75c3-77bd-0833306055ea@10.31.142.119@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513099.209087:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.209092:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.209095:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.209097:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513099.209100:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702425817472, offset 192 00000800:00000200:16.0:1616513099.209104:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513099.209106:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.119@tcp1 00000100:00000200:16.0:1616513099.209111:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513099.209113:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.119@tcp1 routed destination 00000400:00000200:22.0:1616513099.209127:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.119@tcp1(10.31.142.119@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.209133:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.209138:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.209140:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.209142:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.209147:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.209152:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513099.209154:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513099.209203:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.209210:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.209212:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.209213:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.209219:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.209222:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a088 00000800:00000200:20.0:1616513099.209224:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513099.209226:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513099.209231:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.209234:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.209241:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.209246:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789320 00000800:00000200:20.0:1616513099.209250:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.209253:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:25.0:1616513099.210764:0:6494:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929856, portal 4 00000100:00000200:25.0:1616513099.210770:0:6494:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929856, offset 0 00000400:00000200:25.0:1616513099.210777:0:6494:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:25.0:1616513099.210783:0:6494:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:25.0:1616513099.210797:0:6494:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:25.0:1616513099.210803:0:6494:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:25.0:1616513099.210810:0:6494:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:25.0:1616513099.210813:0:6494:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:25.0:1616513099.210816:0:6494:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:25.0:1616513099.210821:0:6494:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513099.210868:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.210886:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513099.210892:0:6474:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513099.210894:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513099.210901:0:6474:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513099.210907:0:6474:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cab102400 x1693067725929856/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513099.210931:0:6474:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84926726e8 00000800:00000200:27.0:1616513099.210936:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513099.210938:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513099.211040:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513099.211050:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.211056:0:6473:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513099.211060:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513099.211064:0:6473:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513099.211072:0:6473:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627d80 00000400:00000200:27.0:1616513099.211078:0:6473:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371676fd [1] + 192 00000400:00000200:27.0:1616513099.211083:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513099.211088:0:6473:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cab102400 x1693067725929856/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513099.211101:0:6473:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.211103:0:6473:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513099.211105:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:25.0:1616513099.211109:0:6494:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cab102400 x1693067725929856/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:25.0:1616513099.211122:0:6494:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cab102400 x1693067725929856/t0(0) o13->boslfs02-OST001e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:25.0:1616513099.211132:0:6494:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84926727f8 00000800:00000200:11.2:1616513099.253817:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513099.253874:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.253884:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.253894:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.54@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.253904:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.54@tcp1 of length 224 into portal 26 MB=0x601968ccd0600 00000400:00000200:4.0:1616513099.253911:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.54@tcp1 of length 224/224 into md 0xf97d0e19 [1] + 0 00000400:00000200:4.0:1616513099.253915:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922c110 00000400:00000200:4.0:1616513099.253918:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.253921:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513099.253926:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922c110 00000800:00000200:24.0:1616513099.253929:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.253929:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.253932:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.253934:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.253993:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690595979232768 00010000:00000200:16.0:1616513099.254005:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1690595979232768/t0(0) o400->5c97b998-3e7c-f74c-cdbe-f3f3fdcac75e@10.31.141.54@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.254020:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690595979232768, offset 192 00000400:00000200:16.0:1616513099.254026:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.54@tcp1 00000400:00000200:16.0:1616513099.254034:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.54@tcp1 routed destination 00000400:00000200:16.0:1616513099.254049:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.54@tcp1(10.31.141.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.254056:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.254063:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.254066:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.254068:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.254074:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.254079:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.254081:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513099.254133:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.254141:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.254144:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.254150:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.254156:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788550 00000800:00000200:20.0:1616513099.254160:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.254163:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.254271:0:51509:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.254326:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.254335:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.254343:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.54@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.254352:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.54@tcp1 of length 224 into portal 12 MB=0x601968ccd0bc0 00000400:00000200:14.0:1616513099.254358:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.54@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 51184 00000400:00000200:14.0:1616513099.254363:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.254365:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513099.254372:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.254375:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.254377:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.254379:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.254450:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690595979234240 00010000:00000200:22.0:1616513099.254462:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9f80 x1690595979234240/t0(0) o400->ac72237a-5417-11df-6bc4-ec48b932deaf@10.31.141.54@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.254476:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690595979234240, offset 192 00000400:00000200:22.0:1616513099.254481:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.54@tcp1 00000400:00000200:22.0:1616513099.254488:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.54@tcp1 routed destination 00000400:00000200:22.0:1616513099.254502:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.54@tcp1(10.31.141.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.254508:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.254513:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.254516:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.254518:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.254523:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.254527:0:51509:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.254540:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.254547:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.254549:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.254556:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.254559:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4ab28 00000800:00000200:18.0:1616513099.254563:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.254566:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.265489:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513099.265548:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.265558:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513099.265564:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:2.0:1616513099.265567:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.92@tcp1 : PUT - for me 00000800:00000200:14.0:1616513099.265573:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513099.265577:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.92@tcp1 of length 224 into portal 26 MB=0x5ffe2d4fd9ac0 00000400:00000200:14.0:1616513099.265581:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.92@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.265583:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.92@tcp1 of length 224/224 into md 0xf97d0e39 [1] + 0 00000400:00000200:2.0:1616513099.265589:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d832922cd48 00000400:00000200:14.0:1616513099.265590:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.92@tcp1 of length 224 into portal 12 MB=0x5ffe2d4fd9d40 00000400:00000200:2.0:1616513099.265591:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.265594:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513099.265598:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d832922cd48 00000800:00000200:2.0:1616513099.265601:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513099.265603:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.92@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 51408 00000800:00000200:2.0:1616513099.265604:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513099.265606:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.265608:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:2.0:1616513099.265609:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:14.0:1616513099.265610:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513099.265616:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.265619:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.265621:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.265669:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724584635072 00010000:00000200:16.0:1616513099.265680:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1688724584635072/t0(0) o400->8a900185-ced5-149f-8d8b-7d8549a86284@10.31.141.92@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513099.265686:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513099.265691:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688724584635712 00000100:00000200:16.0:1616513099.265694:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688724584635072, offset 192 00000400:00000200:16.0:1616513099.265700:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.92@tcp1 00010000:00000200:22.0:1616513099.265703:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4eb9680 x1688724584635712/t0(0) o400->e7efad9e-ca5c-333f-9277-06585a85c5b4@10.31.141.92@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513099.265708:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.92@tcp1 routed destination 00000100:00000200:22.0:1616513099.265718:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688724584635712, offset 192 00000400:00000200:16.0:1616513099.265722:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.92@tcp1(10.31.141.92@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:22.0:1616513099.265726:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.92@tcp1 00000800:00000200:16.0:1616513099.265729:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513099.265734:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.92@tcp1 routed destination 00000800:00000200:16.0:1616513099.265735:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.265738:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.265740:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513099.265745:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513099.265748:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.92@tcp1(10.31.141.92@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000100:00000200:16.0:1616513099.265752:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513099.265755:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.265756:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513099.265761:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.265764:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513099.265766:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513099.265766:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.265770:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513099.265771:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:24.0:1616513099.265775:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.62@tcp1 : PUT - for me 00000800:00000200:7.2:1616513099.265776:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513099.265778:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:4.0:1616513099.265780:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513099.265784:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.62@tcp1 of length 224 into portal 26 MB=0x5ffde4ec7ad40 00000400:00000200:4.0:1616513099.265787:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513099.265791:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.62@tcp1 of length 224/224 into md 0xf97d0e59 [1] + 0 00000400:00000200:4.0:1616513099.265791:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788880 00000400:00000200:24.0:1616513099.265795:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba3b8 00000800:00000200:4.0:1616513099.265796:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.265798:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:4.0:1616513099.265799:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513099.265801:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513099.265806:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba3b8 00000800:00000200:24.0:1616513099.265809:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.265812:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.265814:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.265818:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705153084736 00000800:00000200:11.2:1616513099.265822:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:16.0:1616513099.265824:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1688705153084736/t0(0) o400->f1b1ddb6-229d-a5c9-f1aa-96f695f1ba46@10.31.141.62@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:20.0:1616513099.265829:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.265840:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513099.265841:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513099.265846:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705153084736, offset 192 00000800:00000200:26.0:1616513099.265847:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513099.265849:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.62@tcp1 : PUT - for me 00000400:00000200:16.0:1616513099.265851:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.62@tcp1 00000800:00000200:26.0:1616513099.265854:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.265857:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:16.0:1616513099.265859:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.62@tcp1 routed destination 00000800:00000200:2.0:1616513099.265859:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.265863:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.265867:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b3a8 00000800:00000200:26.0:1616513099.265871:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:16.0:1616513099.265872:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.62@tcp1(10.31.141.62@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.265878:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513099.265879:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.62@tcp1 of length 224 into portal 12 MB=0x5ffde4ec7aec0 00000800:00000200:16.0:1616513099.265883:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513099.265885:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.62@tcp1 of length 224/224 into md 0x865be291 [1] + 83176 00000800:00000200:16.0:1616513099.265885:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.265886:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513099.265890:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.265891:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.265894:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000100:00000200:16.0:1616513099.265894:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.265895:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.265901:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513099.265904:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:20.0:1616513099.265904:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513099.265908:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:20.0:1616513099.265908:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:24.0:1616513099.265910:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513099.265911:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.265916:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513099.265920:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7882a8 00000800:00000200:24.0:1616513099.265923:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.265925:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705153085120 00010000:00000200:8.0:1616513099.265937:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c663a80 x1688705153085120/t0(0) o400->0bb054ff-0ef7-3936-0413-2ca2461b4e96@10.31.141.62@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513099.265951:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705153085120, offset 192 00000400:00000200:8.0:1616513099.265958:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.62@tcp1 00000400:00000200:8.0:1616513099.265965:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.62@tcp1 routed destination 00000400:00000200:8.0:1616513099.265980:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.62@tcp1(10.31.141.62@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.265987:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.265993:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.265995:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.265998:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.266002:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.266007:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.266026:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.266033:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.266036:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513099.266038:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.266043:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.266047:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf22a8 00000800:00000200:18.0:1616513099.266051:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.275491:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513099.275499:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:2.0:1616513099.275549:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.275558:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:0.0:1616513099.275566:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:2.0:1616513099.275567:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.46@tcp1 : PUT - for me 00000800:00000200:0.0:1616513099.275575:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:2.0:1616513099.275577:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.46@tcp1 of length 224 into portal 26 MB=0x5ffde6d4d7300 00000400:00000200:2.0:1616513099.275584:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.46@tcp1 of length 224/224 into md 0xf97d0e79 [1] + 0 00000400:00000200:0.0:1616513099.275585:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.46@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.275589:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba220 00000400:00000200:2.0:1616513099.275592:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.275594:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.275594:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.46@tcp1 of length 224 into portal 12 MB=0x5ffde6d4d7640 00000400:00000200:2.0:1616513099.275599:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba220 00000400:00000200:0.0:1616513099.275601:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.46@tcp1 of length 224/224 into md 0x865be291 [1] + 83400 00000800:00000200:26.0:1616513099.275602:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.275602:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.275604:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513099.275606:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513099.275608:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.275611:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.275618:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.275620:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.275623:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.275627:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.275631:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705665168960 00010000:00000200:8.0:1616513099.275638:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9bf00 x1688705665168960/t0(0) o400->a4f7b493-be57-4bd3-a9c8-d9b49c60ea39@10.31.129.46@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513099.275649:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688705665168960, offset 192 00000400:00000200:8.0:1616513099.275654:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.46@tcp1 00000400:00000200:8.0:1616513099.275660:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.46@tcp1 routed destination 00000400:00000200:8.0:1616513099.275672:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.46@tcp1(10.31.129.46@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.275677:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:16.0:1616513099.275682:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688705665168128 00000800:00000200:8.0:1616513099.275682:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.275684:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.275686:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.275689:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:16.0:1616513099.275692:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1688705665168128/t0(0) o400->934a677b-6e8b-3edb-3437-80547361b300@10.31.129.46@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513099.275694:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.275705:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688705665168128, offset 192 00000400:00000200:16.0:1616513099.275711:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.46@tcp1 00000400:00000200:16.0:1616513099.275718:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.46@tcp1 routed destination 00000400:00000200:16.0:1616513099.275732:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.46@tcp1(10.31.129.46@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.275738:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513099.275743:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513099.275744:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.275746:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513099.275748:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513099.275750:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.275753:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:16.0:1616513099.275753:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.275757:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:20.0:1616513099.275759:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:16.0:1616513099.275759:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513099.275763:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3cb0 00000800:00000200:20.0:1616513099.275767:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.275770:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513099.275835:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.275842:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.275844:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513099.275851:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.275856:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789298 00000800:00000200:14.0:1616513099.275859:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.275862:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.278319:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.278371:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.278381:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513099.278390:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.28@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.278399:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.28@tcp1 of length 224 into portal 26 MB=0x5ffe853110840 00000400:00000200:26.0:1616513099.278406:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.28@tcp1 of length 224/224 into md 0xf97d0e89 [1] + 0 00000400:00000200:26.0:1616513099.278411:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bacc0 00000400:00000200:26.0:1616513099.278413:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.278416:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513099.278420:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bacc0 00000800:00000200:26.0:1616513099.278423:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.278425:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.278426:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.278430:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.278484:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748174673984 00010000:00000200:16.0:1616513099.278494:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1688748174673984/t0(0) o400->eebacead-2b29-5693-3042-8e5acee250c3@10.31.142.28@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.278509:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688748174673984, offset 192 00000400:00000200:16.0:1616513099.278515:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.28@tcp1 00000400:00000200:16.0:1616513099.278522:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.28@tcp1 routed destination 00000400:00000200:16.0:1616513099.278539:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.28@tcp1(10.31.142.28@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.278545:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.278551:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.278554:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.278556:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.278561:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513099.278567:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.278568:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.278623:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.278629:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.278632:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.278639:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.278644:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789188 00000800:00000200:4.0:1616513099.278648:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.278651:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.278926:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513099.278938:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.278947:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513099.278956:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.28@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.278965:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.28@tcp1 of length 224 into portal 12 MB=0x5ffe853110fc0 00000400:00000200:20.0:1616513099.278971:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.28@tcp1 of length 224/224 into md 0x865be291 [1] + 83624 00000400:00000200:20.0:1616513099.278976:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.278979:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513099.278986:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.278989:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513099.278991:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:8.0:1616513099.278994:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688748174675904 00000800:00000200:20.0:1616513099.278995:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:8.0:1616513099.278999:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661200 x1688748174675904/t0(0) o400->1ace4534-b689-e804-2f7f-b5333ea27576@10.31.142.28@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513099.279009:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688748174675904, offset 192 00000400:00000200:8.0:1616513099.279015:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.28@tcp1 00000400:00000200:8.0:1616513099.279020:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.28@tcp1 routed destination 00000400:00000200:8.0:1616513099.279032:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.28@tcp1(10.31.142.28@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.279037:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.279041:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.279043:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.279044:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.279047:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.279051:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.279104:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.279111:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.279113:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513099.279120:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.279124:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3980 00000800:00000200:14.0:1616513099.279128:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.279131:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.279225:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513099.279235:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.279241:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513099.279245:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513099.279249:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.173@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.279257:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.173@tcp1 of length 224 into portal 12 MB=0x5ffde36295880 00000800:00000200:7.2:1616513099.279262:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:4.0:1616513099.279264:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.173@tcp1 of length 224/224 into md 0x865be291 [1] + 83848 00000400:00000200:4.0:1616513099.279269:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.279271:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513099.279273:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513099.279278:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:2.0:1616513099.279280:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:4.0:1616513099.279281:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513099.279282:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513099.279282:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.279286:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704740055168 00000400:00000200:2.0:1616513099.279288:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.173@tcp1 : PUT - for me 00010000:00000200:8.0:1616513099.279291:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23d580 x1688704740055168/t0(0) o400->4928ffdb-936a-31da-4f80-d97dc942d40c@10.31.136.173@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:2.0:1616513099.279297:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.173@tcp1 of length 224 into portal 26 MB=0x5ffde36295800 00000100:00000200:8.0:1616513099.279300:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704740055168, offset 192 00000400:00000200:2.0:1616513099.279303:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.173@tcp1 of length 224/224 into md 0xf97d0e99 [1] + 0 00000400:00000200:8.0:1616513099.279305:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.173@tcp1 00000400:00000200:2.0:1616513099.279307:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba198 00000400:00000200:8.0:1616513099.279310:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.173@tcp1 routed destination 00000400:00000200:2.0:1616513099.279310:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.279312:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513099.279316:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba198 00000800:00000200:2.0:1616513099.279319:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513099.279322:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.173@tcp1(10.31.136.173@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513099.279322:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.279323:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513099.279327:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:16.0:1616513099.279328:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704740055040 00000800:00000200:8.0:1616513099.279331:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.279333:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00010000:00000200:16.0:1616513099.279334:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1688704740055040/t0(0) o400->0d5f9d53-bdc8-0591-9ddd-cba0150289cb@10.31.136.173@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:8.0:1616513099.279334:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.279336:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.279341:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.279344:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704740055040, offset 192 00000400:00000200:16.0:1616513099.279349:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.173@tcp1 00000800:00000200:24.0:1616513099.279352:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:16.0:1616513099.279355:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.173@tcp1 routed destination 00000800:00000200:24.0:1616513099.279357:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.279360:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513099.279361:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.279366:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:16.0:1616513099.279367:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.173@tcp1(10.31.136.173@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513099.279372:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2088 00000800:00000200:16.0:1616513099.279373:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.279376:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:16.0:1616513099.279379:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.279381:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513099.279383:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513099.279386:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.279389:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513099.279390:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513099.279398:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.279402:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.279403:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.279407:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513099.279409:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.279412:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7885d8 00000800:00000200:14.0:1616513099.279415:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.282444:0:51512:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513099.282492:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:26.0:1616513099.282499:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.282508:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513099.282517:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.101@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.282527:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.101@tcp1 of length 224 into portal 26 MB=0x60595e0e17740 00000400:00000200:26.0:1616513099.282533:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.101@tcp1 of length 224/224 into md 0xf97d0ea9 [1] + 0 00000400:00000200:26.0:1616513099.282538:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bbdc0 00000400:00000200:26.0:1616513099.282541:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.282546:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:0.0:1616513099.282547:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:26.0:1616513099.282550:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bbdc0 00000800:00000200:26.0:1616513099.282553:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.282555:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513099.282556:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:26.0:1616513099.282557:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.282560:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513099.282565:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.101@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.282574:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.101@tcp1 of length 224 into portal 12 MB=0x60595e0e17a00 00000400:00000200:0.0:1616513099.282581:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.101@tcp1 of length 224/224 into md 0x865be291 [1] + 84072 00000400:00000200:0.0:1616513099.282586:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.282589:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.282595:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.282598:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.282600:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.282603:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.282606:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694991141403136 00010000:00000200:8.0:1616513099.282612:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b98900 x1694991141403136/t0(0) o400->c65ee785-46ce-4cbe-ada2-48254bf16804@10.31.141.101@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513099.282614:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694991141402432 00000100:00000200:8.0:1616513099.282621:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694991141403136, offset 192 00010000:00000200:16.0:1616513099.282625:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1694991141402432/t0(0) o400->ee04726a-a802-4936-aaed-a633f7ecd23b@10.31.141.101@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:8.0:1616513099.282626:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.101@tcp1 00000400:00000200:8.0:1616513099.282631:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.101@tcp1 routed destination 00000100:00000200:16.0:1616513099.282640:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694991141402432, offset 192 00000400:00000200:8.0:1616513099.282644:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.101@tcp1(10.31.141.101@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:16.0:1616513099.282646:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.101@tcp1 00000800:00000200:8.0:1616513099.282649:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:16.0:1616513099.282654:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.101@tcp1 routed destination 00000800:00000200:8.0:1616513099.282661:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.282663:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.282665:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.282668:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:16.0:1616513099.282669:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.101@tcp1(10.31.141.101@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513099.282673:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.282676:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513099.282681:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513099.282684:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513099.282686:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513099.282688:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513099.282691:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:16.0:1616513099.282693:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513099.282694:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513099.282697:0:51512:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513099.282699:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:20.0:1616513099.282700:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.282704:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3c28 00000800:00000200:20.0:1616513099.282708:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.282711:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513099.282761:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.282767:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.282770:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.282777:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.282782:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7887f8 00000800:00000200:18.0:1616513099.282786:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.282789:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.282871:0:51512:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513099.282883:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.282893:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513099.282894:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513099.282914:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.19@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.282923:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.19@tcp1 of length 224 into portal 26 MB=0x5ffd43fc75740 00000400:00000200:2.0:1616513099.282929:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.19@tcp1 of length 224/224 into md 0xf97d0ec9 [1] + 0 00000400:00000200:2.0:1616513099.282934:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb870 00000400:00000200:2.0:1616513099.282938:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.282941:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513099.282945:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb870 00000800:00000200:2.0:1616513099.282948:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.282950:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.282952:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.282957:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661951731520 00010000:00000200:16.0:1616513099.282963:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688661951731520/t0(0) o400->6d885dac-6066-4115-e1f8-c3b50e532469@10.31.133.19@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.282973:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661951731520, offset 192 00000400:00000200:16.0:1616513099.282978:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.19@tcp1 00000400:00000200:16.0:1616513099.282984:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.19@tcp1 routed destination 00000400:00000200:16.0:1616513099.282996:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.19@tcp1(10.31.133.19@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.283002:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.283006:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.283009:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.283010:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.283013:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513099.283017:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.283018:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.283028:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.283032:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.283034:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513099.283037:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513099.283039:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.283044:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789c28 00000800:00000200:0.0:1616513099.283047:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.283144:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513099.283153:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.283159:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.283163:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.283167:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.19@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.283177:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.19@tcp1 of length 224 into portal 12 MB=0x5ffd43fc75900 00000400:00000200:20.0:1616513099.283183:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.19@tcp1 of length 224/224 into md 0x865be291 [1] + 84296 00000400:00000200:20.0:1616513099.283188:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.283190:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513099.283197:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.283199:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.283201:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.283205:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661951731968 00010000:00000200:8.0:1616513099.283210:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c662d00 x1688661951731968/t0(0) o400->1437af55-3b45-50b4-e276-cffce3f3c1f6@10.31.133.19@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513099.283219:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661951731968, offset 192 00000400:00000200:8.0:1616513099.283223:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.19@tcp1 00000400:00000200:8.0:1616513099.283228:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.19@tcp1 routed destination 00000400:00000200:8.0:1616513099.283239:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.19@tcp1(10.31.133.19@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.283244:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.283248:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.283250:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.283251:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.283254:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.283259:0:51512:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.283268:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.283272:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.283274:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513099.283278:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.283279:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.283282:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2000 00000800:00000200:18.0:1616513099.283285:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.284447:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513099.284499:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.284508:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.284517:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.148@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.284525:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.148@tcp1 of length 224 into portal 26 MB=0x5ffdcb0a167c0 00000400:00000200:4.0:1616513099.284532:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.148@tcp1 of length 224/224 into md 0xf97d0ee9 [1] + 0 00000400:00000200:4.0:1616513099.284537:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb188 00000400:00000200:4.0:1616513099.284540:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.284543:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513099.284547:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb188 00000800:00000200:4.0:1616513099.284550:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.284552:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.284553:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.284556:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.284607:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698204809152 00010000:00000200:16.0:1616513099.284617:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1688698204809152/t0(0) o400->edf6e12e-ef96-2feb-092f-868379edb889@10.31.131.148@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.284630:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688698204809152, offset 192 00000400:00000200:16.0:1616513099.284636:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.148@tcp1 00000400:00000200:16.0:1616513099.284643:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.148@tcp1 routed destination 00000400:00000200:16.0:1616513099.284657:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.148@tcp1(10.31.131.148@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.284663:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.284669:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.284671:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.284673:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.284678:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.284683:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.284684:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513099.284738:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.284745:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.284748:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513099.284750:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.284754:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513099.284760:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788dd0 00000800:00000200:24.0:1616513099.284763:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.284811:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.284865:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.284874:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513099.284883:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.148@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.284891:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.148@tcp1 of length 224 into portal 12 MB=0x5ffdcb0a16d00 00000400:00000200:26.0:1616513099.284898:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.148@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 51632 00000400:00000200:26.0:1616513099.284903:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.284906:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513099.284912:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.284915:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513099.284917:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.284921:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.284974:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688698204810496 00010000:00000200:22.0:1616513099.284986:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21f500 x1688698204810496/t0(0) o400->4b9f0925-4b31-0326-284d-db7817f901e1@10.31.131.148@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.284999:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688698204810496, offset 192 00000400:00000200:22.0:1616513099.285005:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.148@tcp1 00000400:00000200:22.0:1616513099.285012:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.148@tcp1 routed destination 00000400:00000200:22.0:1616513099.285025:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.148@tcp1(10.31.131.148@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.285032:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.285037:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.285040:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.285042:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.285046:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.285051:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.285104:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.285111:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.285114:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.285116:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513099.285120:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.285124:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b7e8 00000800:00000200:14.0:1616513099.285128:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.286156:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513099.286209:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.286218:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513099.286227:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.5@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.286236:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.5@tcp1 of length 224 into portal 26 MB=0x60002097d9c40 00000400:00000200:2.0:1616513099.286243:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.5@tcp1 of length 224/224 into md 0xf97d0f09 [1] + 0 00000400:00000200:2.0:1616513099.286247:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba4c8 00000400:00000200:2.0:1616513099.286250:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.286252:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513099.286256:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba4c8 00000800:00000200:2.0:1616513099.286259:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.286262:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513099.286262:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513099.286269:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.286272:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:2.0:1616513099.286272:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.286281:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.5@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.286289:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.5@tcp1 of length 224 into portal 12 MB=0x60002097d9f00 00000400:00000200:26.0:1616513099.286296:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.5@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 51856 00000400:00000200:26.0:1616513099.286301:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.286303:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513099.286310:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.286313:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.286315:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.286345:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688858609425472 00010000:00000200:16.0:1616513099.286355:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688858609425472/t0(0) o400->2175d7ff-e247-7719-6f09-debc3b1b60ab@10.31.142.5@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.286370:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688858609425472, offset 192 00000100:00000200:22.0:1616513099.286374:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688858609426176 00000400:00000200:16.0:1616513099.286376:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.5@tcp1 00000400:00000200:16.0:1616513099.286383:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.5@tcp1 routed destination 00010000:00000200:22.0:1616513099.286386:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21d100 x1688858609426176/t0(0) o400->c291b7ff-a7c9-ec48-7228-774d27d45661@10.31.142.5@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:16.0:1616513099.286398:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.5@tcp1(10.31.142.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513099.286400:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688858609426176, offset 192 00000800:00000200:16.0:1616513099.286404:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:22.0:1616513099.286406:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.5@tcp1 00000800:00000200:16.0:1616513099.286410:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:22.0:1616513099.286413:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.5@tcp1 routed destination 00000800:00000200:16.0:1616513099.286413:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.286415:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.286419:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513099.286426:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.286426:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513099.286428:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.5@tcp1(10.31.142.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.286447:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.286452:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.286457:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.286459:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.286464:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.286468:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513099.286482:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.286489:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.286492:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.286498:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.286504:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788cc0 00000800:00000200:0.0:1616513099.286507:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.286510:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:14.0:1616513099.286522:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.286542:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.286544:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:2.0:1616513099.286546:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513099.286551:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.286555:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a198 00000800:00000200:14.0:1616513099.286559:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.291057:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513099.291110:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.291119:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513099.291129:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.76@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.291137:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.76@tcp1 of length 224 into portal 26 MB=0x5ffe5836edb40 00000400:00000200:20.0:1616513099.291144:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.76@tcp1 of length 224/224 into md 0xf97d0f39 [1] + 0 00000400:00000200:20.0:1616513099.291149:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6bb650 00000400:00000200:20.0:1616513099.291152:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.291155:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513099.291159:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6bb650 00000800:00000200:20.0:1616513099.291162:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.291164:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.291165:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513099.291168:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.291222:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688736101227328 00010000:00000200:16.0:1616513099.291233:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688736101227328/t0(0) o400->f505842b-4c57-481a-86be-324a9e983b02@10.31.141.76@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.291248:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688736101227328, offset 192 00000400:00000200:16.0:1616513099.291254:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.76@tcp1 00000400:00000200:16.0:1616513099.291261:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.76@tcp1 routed destination 00000400:00000200:16.0:1616513099.291276:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.76@tcp1(10.31.141.76@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513099.291276:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.291283:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.291289:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.291292:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.291294:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513099.291299:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:16.0:1616513099.291305:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513099.291329:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.291338:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.291351:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.76@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.291361:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.76@tcp1 of length 224 into portal 12 MB=0x5ffe5836edc40 00000400:00000200:0.0:1616513099.291367:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.76@tcp1 of length 224/224 into md 0x865be291 [1] + 84520 00000400:00000200:0.0:1616513099.291372:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.291375:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.291382:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513099.291385:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513099.291386:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:0.0:1616513099.291393:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513099.291394:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513099.291396:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.291397:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.291398:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688736101227584 00000400:00000200:4.0:1616513099.291403:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00010000:00000200:8.0:1616513099.291406:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cc6b9a400 x1688736101227584/t0(0) o400->39fec0f1-6661-9814-6be8-b5578018ba4f@10.31.141.76@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:4.0:1616513099.291409:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788cc0 00000800:00000200:4.0:1616513099.291413:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.291419:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688736101227584, offset 192 00000400:00000200:8.0:1616513099.291424:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.76@tcp1 00000400:00000200:8.0:1616513099.291429:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.76@tcp1 routed destination 00000400:00000200:8.0:1616513099.291451:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.76@tcp1(10.31.141.76@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.291456:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.291461:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.291463:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.291464:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.291469:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.291475:0:51513:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.291529:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.291536:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.291539:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.291545:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.291549:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3188 00000800:00000200:26.0:1616513099.291553:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.291558:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513099.306486:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929872, portal 4 00000100:00000200:10.0:1616513099.306492:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929872, offset 0 00000400:00000200:10.0:1616513099.306498:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:10.0:1616513099.306505:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:10.0:1616513099.306518:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513099.306525:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:10.0:1616513099.306530:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513099.306533:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:10.0:1616513099.306535:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:10.0:1616513099.306540:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:7.0:1616513099.306542:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929888, portal 4 00000100:00000200:7.0:1616513099.306548:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929888, offset 0 00000400:00000200:7.0:1616513099.306555:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:7.0:1616513099.306562:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:7.0:1616513099.306575:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513099.306582:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:7.0:1616513099.306589:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513099.306592:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:7.0:1616513099.306607:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:5.2:1616513099.306609:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:7.0:1616513099.306612:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513099.306620:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.306634:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513099.306640:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513099.306642:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513099.306649:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513099.306655:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bf8000 x1693067725929888/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:20.0:1616513099.306664:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000400:00000200:27.0:1616513099.306667:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637e5d8 00000800:00000200:27.0:1616513099.306671:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:20.0:1616513099.306671:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:27.0:1616513099.306674:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:20.0:1616513099.306674:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.306680:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513099.306686:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5cdaf81b00 x1693067725929872/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513099.306697:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e834b8 00000800:00000200:20.0:1616513099.306701:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513099.306704:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513099.306768:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:27.2:1616513099.306784:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513099.306792:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.306798:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513099.306801:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513099.306806:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513099.306814:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627da0 00000400:00000200:27.0:1616513099.306820:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716770d [1] + 192 00000800:00000200:24.0:1616513099.306822:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:27.0:1616513099.306825:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513099.306829:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bf8000 x1693067725929888/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:24.0:1616513099.306830:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:24.0:1616513099.306839:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:24.0:1616513099.306846:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627d90 00000400:00000200:24.0:1616513099.306852:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866ddbe1 [1] + 192 00000800:00000200:27.0:1616513099.306854:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.306856:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:24.0:1616513099.306857:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:7.0:1616513099.306857:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bf8000 x1693067725929888/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513099.306858:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:24.0:1616513099.306862:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5cdaf81b00 x1693067725929872/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000100:00000200:7.0:1616513099.306866:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bf8000 x1693067725929888/t0(0) o13->boslfs02-OST0032-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513099.306873:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637ef68 00000800:00000200:4.0:1616513099.306873:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513099.306876:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:24.0:1616513099.306879:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513099.306880:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:10.0:1616513099.306884:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5cdaf81b00 x1693067725929872/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513099.306897:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5cdaf81b00 x1693067725929872/t0(0) o13->boslfs02-OST0005-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513099.306907:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83ba0 00000800:00000200:11.2:1616513099.312313:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513099.312370:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.312379:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513099.312388:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.136@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.312398:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.136@tcp1 of length 224 into portal 26 MB=0x5ffd7f7507440 00000400:00000200:20.0:1616513099.312407:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.136@tcp1 of length 224/224 into md 0xf97d0f59 [1] + 0 00000400:00000200:20.0:1616513099.312412:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8da53f6ba7f8 00000400:00000200:20.0:1616513099.312414:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.312417:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513099.312422:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da53f6ba7f8 00000800:00000200:20.0:1616513099.312425:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.312428:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513099.312429:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.312432:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.312457:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688677915849792 00010000:00000200:16.0:1616513099.312468:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688677915849792/t0(0) o400->4556e14a-520c-86cd-8364-7e009a0f6e76@10.31.129.136@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.312482:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688677915849792, offset 192 00000400:00000200:16.0:1616513099.312489:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.136@tcp1 00000400:00000200:16.0:1616513099.312497:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.136@tcp1 routed destination 00000400:00000200:16.0:1616513099.312511:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.136@tcp1(10.31.129.136@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.312517:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.312522:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.312525:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.312527:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.312531:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.312537:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.312538:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513099.312590:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.312597:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.312600:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.312607:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.312612:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789c28 00000800:00000200:4.0:1616513099.312616:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.312619:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.312998:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.313011:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.313021:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513099.313022:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513099.313030:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.136@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.313039:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.136@tcp1 of length 224 into portal 12 MB=0x5ffd7f75075c0 00000400:00000200:0.0:1616513099.313046:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.136@tcp1 of length 224/224 into md 0x865be291 [1] + 84744 00000400:00000200:0.0:1616513099.313051:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.313054:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.313064:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.313067:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.313069:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.313081:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688677915850176 00010000:00000200:8.0:1616513099.313091:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d569bb08000 x1688677915850176/t0(0) o400->2772fd00-1ff6-fd25-b26a-e4a3fc82c51d@10.31.129.136@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513099.313113:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688677915850176, offset 192 00000400:00000200:8.0:1616513099.313118:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.136@tcp1 00000400:00000200:8.0:1616513099.313125:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.136@tcp1 routed destination 00000400:00000200:8.0:1616513099.313138:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.136@tcp1(10.31.129.136@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.313144:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.313149:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.313151:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.313153:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.313158:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.313164:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.313218:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.313224:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513099.313227:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513099.313233:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.313237:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a90 00000800:00000200:2.0:1616513099.313241:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.313244:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:31.2:1616513099.318446:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:26.0:1616513099.318504:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:26.0:1616513099.318513:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.234@o2ib1 00000400:00000200:26.0:1616513099.318522:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.234@o2ib1 : PUT - for me 00000400:00000200:26.0:1616513099.318530:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.234@o2ib1 of length 224 into portal 24 MB=0x603db0154d140 00000400:00000200:26.0:1616513099.318537:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 18 from 12345-10.242.46.234@o2ib1 of length 224/224 into md 0xf973f0e9 [1] + 0 00000400:00000200:26.0:1616513099.318542:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82baa4b6d8 00000400:00000200:26.0:1616513099.318544:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.318548:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt_out 00000400:00000200:26.0:1616513099.318554:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b6d8 00000800:00000200:26.0:1616513099.318557:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:26.0:1616513099.318560:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:18.0:1616513099.318560:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (23)-- 00000800:00000200:26.0:1616513099.318564:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000100:00000200:30.0:1616513099.318576:0:185125:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693089015320896 00010000:00000200:30.0:1616513099.318596:0:185125:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74da74e850 x1693089015320896/t0(0) o41->boslfs02-MDT0000-mdtlov_UUID@10.242.46.234@o2ib1:725/0 lens 224/368 e 0 to 0 dl 1616513105 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513099.318609:0:185125:0:(niobuf.c:85:ptl_send_buf()) Sending 336 bytes to portal 4, xid 1693089015320896, offset 192 00000400:00000200:30.0:1616513099.318615:0:185125:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.234@o2ib1 00000400:00000200:30.0:1616513099.318622:0:185125:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.242.46.234@o2ib1 local destination 00000400:00000200:30.0:1616513099.318635:0:185125:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.242.46.234@o2ib1(10.242.46.234@o2ib1:10.242.46.234@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.318641:0:185125:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 336 bytes in 1 frags to 12345-10.242.46.234@o2ib1 00000800:00000200:30.0:1616513099.318646:0:185125:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ded3ce800] -> 10.242.46.234@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.318648:0:185125:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (20)++ 00000800:00000200:30.0:1616513099.318650:0:185125:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d7486a19a00] (21)++ 00000800:00000200:30.0:1616513099.318655:0:185125:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:31.2:1616513099.318661:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d7486a19a00] (21)++ 00000100:00000200:30.0:1616513099.318662:0:185125:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 24 00000800:00000200:2.0:1616513099.318674:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)++ 00000800:00000200:2.0:1616513099.318680:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d7486a19a00] (23)-- 00000400:00000200:2.0:1616513099.318683:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513099.318690:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.234@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.318696:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee3b8 00000800:00000200:2.0:1616513099.318699:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (22)-- 00000800:00000200:2.0:1616513099.318704:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d7486a19a00] (21)-- 00000800:00000200:11.2:1616513099.321735:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513099.321791:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.321800:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.321809:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.145@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.321818:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.145@tcp1 of length 224 into portal 26 MB=0x5ffd8cf8dae00 00000400:00000200:4.0:1616513099.321824:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.145@tcp1 of length 224/224 into md 0xf97d0f69 [1] + 0 00000400:00000200:4.0:1616513099.321829:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3210 00000400:00000200:4.0:1616513099.321832:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.321835:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513099.321839:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3210 00000800:00000200:4.0:1616513099.321842:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.321845:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.321845:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.321849:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.321902:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688681543740928 00010000:00000200:16.0:1616513099.321913:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688681543740928/t0(0) o400->2cd1c497-083e-8641-8356-8f67d090f2bf@10.31.131.145@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.321928:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688681543740928, offset 192 00000400:00000200:16.0:1616513099.321934:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.145@tcp1 00000400:00000200:16.0:1616513099.321941:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.145@tcp1 routed destination 00000400:00000200:16.0:1616513099.321956:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.145@tcp1(10.31.131.145@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.321963:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.321969:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.321971:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.321973:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.321978:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.321983:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.321985:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513099.321994:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.322001:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.322004:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.322011:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.322016:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7885d8 00000800:00000200:20.0:1616513099.322020:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.322022:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.322060:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.322072:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.322080:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:4.0:1616513099.322081:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.322089:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.145@tcp1 : PUT - for me 00000400:00000200:24.0:1616513099.322097:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.145@tcp1 of length 224 into portal 12 MB=0x5ffd8cf8db000 00000400:00000200:24.0:1616513099.322104:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.145@tcp1 of length 224/224 into md 0x865be291 [1] + 84968 00000400:00000200:24.0:1616513099.322109:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513099.322111:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513099.322119:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.322121:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.322123:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.322132:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688681543741440 00010000:00000200:8.0:1616513099.322140:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d85193ed100 x1688681543741440/t0(0) o400->4fedd9c9-01eb-0abd-1d77-8f8965adc6b3@10.31.131.145@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513099.322152:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688681543741440, offset 192 00000400:00000200:8.0:1616513099.322157:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.145@tcp1 00000400:00000200:8.0:1616513099.322163:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.145@tcp1 routed destination 00000400:00000200:8.0:1616513099.322175:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.145@tcp1(10.31.131.145@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.322181:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.322186:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.322188:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.322189:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.322193:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.322199:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.322253:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.322265:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.322267:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.322274:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.322278:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3078 00000800:00000200:18.0:1616513099.322282:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.322287:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.322293:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d0[4] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513099.322295:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.322309:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.322311:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.322313:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.325683:0:51517:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.325737:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.325747:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.325755:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.131@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.325765:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.131@tcp1 of length 224 into portal 26 MB=0x5fff3b7638b40 00000400:00000200:14.0:1616513099.325772:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.131@tcp1 of length 224/224 into md 0xf97d0f89 [1] + 0 00000400:00000200:14.0:1616513099.325776:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2f68 00000400:00000200:14.0:1616513099.325779:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.325782:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513099.325786:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2f68 00000800:00000200:14.0:1616513099.325789:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.325792:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513099.325792:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.325799:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.325802:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.325803:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.325811:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.131@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.325820:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.131@tcp1 of length 224 into portal 12 MB=0x5fff3b7638bc0 00000400:00000200:18.0:1616513099.325827:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.131@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 52080 00000400:00000200:18.0:1616513099.325832:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.325835:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.325842:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.325845:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.325847:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.325849:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688797102443328 00010000:00000200:16.0:1616513099.325860:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688797102443328/t0(0) o400->76fee377-fc0e-dc19-4177-b6d1e517365c@10.31.142.131@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.325874:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688797102443328, offset 192 00000400:00000200:16.0:1616513099.325881:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.131@tcp1 00000400:00000200:16.0:1616513099.325888:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.131@tcp1 routed destination 00000400:00000200:16.0:1616513099.325903:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.131@tcp1(10.31.142.131@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513099.325906:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688797102443456 00000800:00000200:16.0:1616513099.325913:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:22.0:1616513099.325918:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bd580 x1688797102443456/t0(0) o400->6293f39a-d7e1-9036-2574-e1d8065f28fc@10.31.142.131@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513099.325919:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.325922:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.325924:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.325929:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513099.325933:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688797102443456, offset 192 00000100:00000200:16.0:1616513099.325936:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.325936:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513099.325939:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.131@tcp1 00000400:00000200:22.0:1616513099.325947:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.131@tcp1 routed destination 00000800:00000200:20.0:1616513099.325950:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513099.325961:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.131@tcp1(10.31.142.131@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513099.325961:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.325963:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513099.325968:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:20.0:1616513099.325970:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513099.325974:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:20.0:1616513099.325975:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789298 00000800:00000200:22.0:1616513099.325976:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.325979:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513099.325979:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.325982:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513099.325983:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.325987:0:51517:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.326058:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.326065:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513099.326068:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513099.326069:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:2.0:1616513099.326075:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.326079:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b540 00000800:00000200:2.0:1616513099.326083:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.344889:0:51519:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.344945:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.344955:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.344964:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.40@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.344973:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.40@tcp1 of length 224 into portal 26 MB=0x604f6266aed80 00000400:00000200:18.0:1616513099.344980:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.40@tcp1 of length 224/224 into md 0xf97d0fa9 [1] + 0 00000400:00000200:18.0:1616513099.344984:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2660 00000400:00000200:18.0:1616513099.344987:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.344990:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.344994:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2660 00000800:00000200:18.0:1616513099.344997:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.345000:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.345001:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.345005:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.345056:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694305113271680 00010000:00000200:16.0:1616513099.345067:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1694305113271680/t0(0) o400->e4450eda-580f-fe4e-ab80-93f17d2d74ab@10.31.142.40@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.345081:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694305113271680, offset 192 00000400:00000200:16.0:1616513099.345086:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.40@tcp1 00000400:00000200:16.0:1616513099.345093:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.40@tcp1 routed destination 00000800:00000200:11.2:1616513099.345105:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:16.0:1616513099.345107:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.40@tcp1(10.31.142.40@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.345114:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.345119:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.345122:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.345124:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513099.345128:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:16.0:1616513099.345134:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513099.345158:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.345167:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.345175:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.40@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.345183:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.40@tcp1 of length 224 into portal 12 MB=0x604f6266aee00 00000400:00000200:4.0:1616513099.345189:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.40@tcp1 of length 224/224 into md 0x865be291 [1] + 85192 00000400:00000200:4.0:1616513099.345198:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.345201:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513099.345207:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513099.345210:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513099.345211:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)++ 00000800:00000200:4.0:1616513099.345218:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513099.345219:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513099.345221:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.345222:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.345224:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694305113271808 00000400:00000200:24.0:1616513099.345229:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513099.345234:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788880 00010000:00000200:8.0:1616513099.345234:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23a400 x1694305113271808/t0(0) o400->470c2403-5e0c-1660-74ba-8f4174e7b9fe@10.31.142.40@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513099.345238:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.345247:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694305113271808, offset 192 00000400:00000200:8.0:1616513099.345252:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.40@tcp1 00000400:00000200:8.0:1616513099.345259:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.40@tcp1 routed destination 00000400:00000200:8.0:1616513099.345272:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.40@tcp1(10.31.142.40@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.345279:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.345284:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.345287:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.345289:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.345293:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.345298:0:51519:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.345351:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.345357:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.345360:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513099.345366:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.345370:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000800:00000200:14.0:1616513099.345374:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.345377:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:11.0:1616513099.354645:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929904, portal 4 00000100:00000200:11.0:1616513099.354649:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929904, offset 0 00000400:00000200:11.0:1616513099.354655:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:11.0:1616513099.354660:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:11.0:1616513099.354671:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513099.354676:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:11.0:1616513099.354682:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513099.354684:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:11.0:1616513099.354686:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:11.0:1616513099.354691:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513099.354738:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513099.354795:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513099.354802:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513099.354805:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.354812:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513099.354818:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cca3eb600 x1693067725929904/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513099.354830:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3abb0 00000800:00000200:26.0:1616513099.354834:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513099.354837:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513099.354949:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:14.0:1616513099.355001:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513099.355010:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:14.0:1616513099.355018:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:14.0:1616513099.355026:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627db0 00000400:00000200:14.0:1616513099.355032:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d1659 [1] + 192 00000400:00000200:14.0:1616513099.355037:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.355044:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cca3eb600 x1693067725929904/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:2.0:1616513099.355056:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513099.355059:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513099.355062:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513099.355063:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:11.0:1616513099.355065:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cca3eb600 x1693067725929904/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:11.0:1616513099.355075:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cca3eb600 x1693067725929904/t0(0) o13->boslfs02-OST0020-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513099.355084:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a990 00000800:00000200:11.2:1616513099.367215:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.367271:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.367280:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.367289:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.55@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.367299:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.55@tcp1 of length 224 into portal 26 MB=0x5ffd4ccf635c0 00000400:00000200:0.0:1616513099.367306:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.55@tcp1 of length 224/224 into md 0xf97d0fc9 [1] + 0 00000400:00000200:0.0:1616513099.367311:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2220 00000400:00000200:0.0:1616513099.367314:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.367316:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.367321:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:4.0:1616513099.367324:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.367324:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.367327:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.367329:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.367385:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664320390592 00010000:00000200:16.0:1616513099.367397:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1688664320390592/t0(0) o400->cde7966b-8814-287b-9086-bd7740c82573@10.31.129.55@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.367411:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664320390592, offset 192 00000400:00000200:16.0:1616513099.367418:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.55@tcp1 00000400:00000200:16.0:1616513099.367426:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.55@tcp1 routed destination 00000400:00000200:16.0:1616513099.367454:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.55@tcp1(10.31.129.55@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.367461:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.367467:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.367470:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.367473:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.367478:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513099.367478:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513099.367482:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.367486:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513099.367494:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.367501:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.367504:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.367510:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.367516:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789320 00000800:00000200:20.0:1616513099.367519:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.367522:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513099.367533:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.367543:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513099.367552:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.55@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.367562:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.55@tcp1 of length 224 into portal 12 MB=0x5ffd4ccf63ac0 00000400:00000200:26.0:1616513099.367568:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.55@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 52304 00000400:00000200:26.0:1616513099.367573:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.367576:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513099.367583:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.367586:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.367590:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.367596:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.367648:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664320391872 00010000:00000200:22.0:1616513099.367660:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21ad00 x1688664320391872/t0(0) o400->524eb99e-768e-f190-41e4-0cd7f3cfb0f7@10.31.129.55@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.367674:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664320391872, offset 192 00000400:00000200:22.0:1616513099.367680:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.55@tcp1 00000400:00000200:22.0:1616513099.367688:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.55@tcp1 routed destination 00000400:00000200:22.0:1616513099.367703:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.55@tcp1(10.31.129.55@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.367710:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.367715:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.367718:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.367720:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.367725:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.367729:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.367785:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.367792:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513099.367795:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513099.367801:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.367805:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a330 00000800:00000200:2.0:1616513099.367809:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.367812:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.379104:0:51523:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.379161:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513099.379164:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513099.379179:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.379188:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.254@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.379197:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.254@tcp1 of length 224 into portal 26 MB=0x5fff98487fcc0 00000400:00000200:18.0:1616513099.379206:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.254@tcp1 of length 224/224 into md 0xf97d0fe9 [1] + 0 00000400:00000200:18.0:1616513099.379211:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3e48 00000400:00000200:18.0:1616513099.379213:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.379216:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.379220:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000800:00000200:18.0:1616513099.379224:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.379226:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.379226:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:4.0:1616513099.379228:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513099.379230:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513099.379238:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.379247:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.254@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.379256:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.254@tcp1 of length 224 into portal 12 MB=0x5fff98487fec0 00000400:00000200:4.0:1616513099.379262:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.254@tcp1 of length 224/224 into md 0x865be291 [1] + 85416 00000400:00000200:4.0:1616513099.379268:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.379270:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513099.379277:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.379280:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.379282:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.379284:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.379286:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688822018997440 00000100:00000200:8.0:1616513099.379295:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688822018997952 00010000:00000200:16.0:1616513099.379297:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b8850 x1688822018997440/t0(0) o400->79314cab-cfb6-2e96-19b9-b440c4ae8c24@10.31.141.254@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:8.0:1616513099.379306:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23f980 x1688822018997952/t0(0) o400->738167e3-7f79-b8d1-4bf1-a2c3d310052f@10.31.141.254@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513099.379311:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688822018997440, offset 192 00000400:00000200:16.0:1616513099.379317:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.254@tcp1 00000100:00000200:8.0:1616513099.379321:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688822018997952, offset 192 00000400:00000200:16.0:1616513099.379325:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.254@tcp1 routed destination 00000400:00000200:8.0:1616513099.379326:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.254@tcp1 00000400:00000200:16.0:1616513099.379340:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.254@tcp1(10.31.141.254@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.379347:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.379348:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.254@tcp1 routed destination 00000800:00000200:16.0:1616513099.379355:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.379358:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.379360:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:8.0:1616513099.379362:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.254@tcp1(10.31.141.254@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.379365:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.379368:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513099.379370:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.379372:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513099.379373:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.379376:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.379378:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513099.379382:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.379382:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.379387:0:51523:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513099.379389:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.379392:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.379399:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.379406:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788660 00000800:00000200:20.0:1616513099.379409:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.379412:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513099.379457:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.379464:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513099.379467:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513099.379474:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.379478:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3e48 00000800:00000200:2.0:1616513099.379482:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.379485:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.380314:0:51523:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.380326:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.380333:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:18.0:1616513099.380336:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.380342:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.90@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.380352:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.90@tcp1 of length 224 into portal 26 MB=0x60051a7598100 00000400:00000200:26.0:1616513099.380358:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.90@tcp1 of length 224/224 into md 0xf97d1019 [1] + 0 00000400:00000200:26.0:1616513099.380363:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2770 00000400:00000200:26.0:1616513099.380366:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.380368:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513099.380373:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2770 00000800:00000200:26.0:1616513099.380376:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.380379:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.380380:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.380385:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689200560275712 00010000:00000200:16.0:1616513099.380391:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff050 x1689200560275712/t0(0) o400->8be1b25c-c93f-cc59-0e64-a68c0df52f36@10.31.136.90@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.380414:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1689200560275712, offset 192 00000400:00000200:16.0:1616513099.380419:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.90@tcp1 00000400:00000200:16.0:1616513099.380425:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.90@tcp1 routed destination 00000400:00000200:16.0:1616513099.380448:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.90@tcp1(10.31.136.90@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.380454:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.380458:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.380460:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.380462:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.380465:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513099.380469:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.380470:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.380522:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.380529:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.380531:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.380538:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.380543:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788ee0 00000800:00000200:4.0:1616513099.380547:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.380550:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.380688:0:51523:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:2.0:1616513099.380745:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.380754:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513099.380763:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.90@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.380771:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.90@tcp1 of length 224 into portal 12 MB=0x60051a75982c0 00000400:00000200:2.0:1616513099.380778:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.90@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 52528 00000400:00000200:2.0:1616513099.380784:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.380786:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513099.380793:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.380796:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513099.380798:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.380801:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.380857:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1689200560276160 00010000:00000200:22.0:1616513099.380869:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647d8d80 x1689200560276160/t0(0) o400->a1521856-b976-15b2-f5fd-7e8d8e3be3c3@10.31.136.90@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.380895:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1689200560276160, offset 192 00000400:00000200:22.0:1616513099.380901:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.90@tcp1 00000400:00000200:22.0:1616513099.380908:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.90@tcp1 routed destination 00000400:00000200:22.0:1616513099.380923:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.90@tcp1(10.31.136.90@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.380929:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.380935:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.380938:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.380940:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.380945:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.380949:0:51523:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.380959:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.380963:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.380965:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513099.380967:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.380970:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.380973:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a990 00000800:00000200:18.0:1616513099.380976:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.383762:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513099.383816:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.383825:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513099.383834:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.151@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.383843:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.151@tcp1 of length 224 into portal 26 MB=0x603e13bde3ac0 00000400:00000200:20.0:1616513099.383850:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.151@tcp1 of length 224/224 into md 0xf97d1049 [1] + 0 00000400:00000200:20.0:1616513099.383855:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf38f8 00000400:00000200:20.0:1616513099.383857:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.383860:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513099.383864:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf38f8 00000800:00000200:20.0:1616513099.383867:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.383869:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.383870:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513099.383874:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.383927:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115767208640 00010000:00000200:16.0:1616513099.383938:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e050 x1693115767208640/t0(0) o400->c7724465-873a-0b89-032b-1f8ffcc755e3@10.31.136.151@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.383954:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693115767208640, offset 192 00000400:00000200:16.0:1616513099.383960:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.151@tcp1 00000400:00000200:16.0:1616513099.383968:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.151@tcp1 routed destination 00000400:00000200:16.0:1616513099.383982:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.151@tcp1(10.31.136.151@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.383989:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.383995:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.383998:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.384000:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513099.384000:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513099.384005:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.384010:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.384011:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:2.0:1616513099.384057:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.384068:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:4.0:1616513099.384076:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:2.0:1616513099.384077:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.151@tcp1 : PUT - for me 00000800:00000200:4.0:1616513099.384082:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.384085:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513099.384085:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.151@tcp1 of length 224 into portal 12 MB=0x603e13bde3c00 00000400:00000200:2.0:1616513099.384091:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.151@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 52752 00000400:00000200:4.0:1616513099.384092:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.384096:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.384097:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788d48 00000100:00000200:2.0:1616513099.384099:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513099.384101:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.384104:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:2.0:1616513099.384105:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.384108:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513099.384110:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.384112:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.384184:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693115767208960 00010000:00000200:22.0:1616513099.384196:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647dec00 x1693115767208960/t0(0) o400->990c94b6-1ab2-a258-15e0-035dfcb7a042@10.31.136.151@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.384209:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693115767208960, offset 192 00000400:00000200:22.0:1616513099.384215:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.151@tcp1 00000400:00000200:22.0:1616513099.384222:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.151@tcp1 routed destination 00000400:00000200:22.0:1616513099.384235:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.151@tcp1(10.31.136.151@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.384242:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.384247:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.384249:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.384251:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.384256:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.384261:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.384315:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.384322:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.384325:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.384328:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.384331:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.384335:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b188 00000800:00000200:26.0:1616513099.384339:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.401838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.401895:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.401904:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513099.401913:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.18@tcp1 : PUT - for me 00000400:00000200:24.0:1616513099.401922:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.18@tcp1 of length 224 into portal 26 MB=0x5ffd35e237200 00000400:00000200:24.0:1616513099.401928:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.18@tcp1 of length 224/224 into md 0xf97d1059 [1] + 0 00000400:00000200:24.0:1616513099.401933:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3b18 00000400:00000200:24.0:1616513099.401936:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513099.401938:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:24.0:1616513099.401943:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3b18 00000800:00000200:24.0:1616513099.401946:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.401949:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513099.401949:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513099.401953:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.402008:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658166116864 00010000:00000200:16.0:1616513099.402019:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66e850 x1688658166116864/t0(0) o400->ecac4034-9862-b757-476f-9d1b94203985@10.31.133.18@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.402033:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688658166116864, offset 192 00000400:00000200:16.0:1616513099.402040:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.18@tcp1 00000400:00000200:16.0:1616513099.402047:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.18@tcp1 routed destination 00000400:00000200:16.0:1616513099.402062:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.18@tcp1(10.31.133.18@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.402069:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.402075:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.402077:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.402080:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.402084:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.402089:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.402091:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513099.402163:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.402170:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.402175:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.402181:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.402187:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789078 00000800:00000200:4.0:1616513099.402190:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.402193:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.402210:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.402218:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.402227:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.402229:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.402240:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.18@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.402249:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.18@tcp1 of length 224 into portal 12 MB=0x5ffd35e237300 00000400:00000200:0.0:1616513099.402255:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.18@tcp1 of length 224/224 into md 0x865be291 [1] + 85640 00000400:00000200:0.0:1616513099.402261:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.402263:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.402270:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.402273:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.402275:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.402287:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688658166117120 00010000:00000200:8.0:1616513099.402297:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d569bb08480 x1688658166117120/t0(0) o400->b0487875-2027-ab85-8594-6dc919876239@10.31.133.18@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513099.402310:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688658166117120, offset 192 00000400:00000200:8.0:1616513099.402315:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.18@tcp1 00000400:00000200:8.0:1616513099.402322:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.18@tcp1 routed destination 00000400:00000200:8.0:1616513099.402335:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.18@tcp1(10.31.133.18@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.402341:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.402346:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.402349:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.402350:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.402355:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.402361:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.402417:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.402424:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513099.402427:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513099.402448:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.402452:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2220 00000800:00000200:2.0:1616513099.402456:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.402459:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:3.0:1616513099.402732:0:6506:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929920, portal 4 00000100:00000200:3.0:1616513099.402737:0:6506:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929920, offset 0 00000400:00000200:3.0:1616513099.402744:0:6506:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:3.0:1616513099.402750:0:6506:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:3.0:1616513099.402763:0:6506:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:3.0:1616513099.402770:0:6506:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513099.402777:0:6506:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513099.402780:0:6506:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:3.0:1616513099.402782:0:6506:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:3.0:1616513099.402787:0:6506:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513099.402833:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513099.402845:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:4.0:1616513099.402850:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:4.0:1616513099.402852:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513099.402853:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:4.0:1616513099.402858:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:4.0:1616513099.402863:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456eec00 x1693067725929920/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:4.0:1616513099.402875:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2d48 00000800:00000200:4.0:1616513099.402878:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513099.403020:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:24.0:1616513099.403030:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513099.403036:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:0.0:1616513099.403039:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:24.0:1616513099.403043:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:24.0:1616513099.403049:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627dc0 00000400:00000200:24.0:1616513099.403055:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866ddcd1 [1] + 192 00000400:00000200:24.0:1616513099.403059:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513099.403064:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456eec00 x1693067725929920/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:24.0:1616513099.403078:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:24.0:1616513099.403081:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:24.0:1616513099.403082:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:3.0:1616513099.403089:0:6506:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456eec00 x1693067725929920/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513099.403102:0:6506:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456eec00 x1693067725929920/t0(0) o13->boslfs02-OST0009-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513099.403113:0:6506:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2a18 00000800:00000200:7.2:1616513099.403998:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.404052:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.404062:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.404071:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.147@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.404080:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.147@tcp1 of length 224 into portal 26 MB=0x5ffd78e391040 00000400:00000200:18.0:1616513099.404087:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.147@tcp1 of length 224/224 into md 0xf97d1069 [1] + 0 00000400:00000200:18.0:1616513099.404092:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3320 00000400:00000200:18.0:1616513099.404094:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.404097:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.404101:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3320 00000800:00000200:18.0:1616513099.404104:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.404107:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.404109:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.404111:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.404164:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676152709184 00010000:00000200:16.0:1616513099.404175:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5050 x1688676152709184/t0(0) o400->dea41fd8-e60b-ef82-4706-40ae9da394da@10.31.129.147@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.404190:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688676152709184, offset 192 00000400:00000200:16.0:1616513099.404196:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.147@tcp1 00000400:00000200:16.0:1616513099.404204:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.147@tcp1 routed destination 00000400:00000200:16.0:1616513099.404226:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.147@tcp1(10.31.129.147@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.404233:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.404239:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.404241:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.404244:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.404249:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513099.404255:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.404255:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.404268:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.404275:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.404277:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.404284:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.404290:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788000 00000800:00000200:20.0:1616513099.404293:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.404296:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.404329:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.404382:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.404391:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.404400:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.147@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.404409:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.147@tcp1 of length 224 into portal 12 MB=0x5ffd78e391480 00000400:00000200:0.0:1616513099.404416:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.147@tcp1 of length 224/224 into md 0x865be291 [1] + 85864 00000400:00000200:0.0:1616513099.404421:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.404423:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.404430:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:8.0:1616513099.404443:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688676152710272 00000800:00000200:0.0:1616513099.404445:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.404448:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:8.0:1616513099.404449:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d569bb08900 x1688676152710272/t0(0) o400->ec90872c-7948-3431-54dd-5302cac9c9a6@10.31.129.147@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513099.404450:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.404459:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688676152710272, offset 192 00000400:00000200:8.0:1616513099.404464:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.147@tcp1 00000400:00000200:8.0:1616513099.404469:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.147@tcp1 routed destination 00000400:00000200:8.0:1616513099.404481:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.147@tcp1(10.31.129.147@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.404486:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.404490:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.404493:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.404494:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.404497:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.404502:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.404554:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.404562:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:2.0:1616513099.404564:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1616513099.404571:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:2.0:1616513099.404575:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf25d8 00000800:00000200:2.0:1616513099.404579:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.404582:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.407418:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513099.407471:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.407481:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.407482:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.407490:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.2@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.407499:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.2@tcp1 of length 224 into portal 26 MB=0x604a468815440 00000400:00000200:20.0:1616513099.407505:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.2@tcp1 of length 224/224 into md 0xf97d1089 [1] + 0 00000400:00000200:20.0:1616513099.407510:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2330 00000400:00000200:20.0:1616513099.407513:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.407516:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513099.407522:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2330 00000800:00000200:20.0:1616513099.407525:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.407528:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.407530:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.407552:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:16.0:1616513099.407584:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954034717760 00010000:00000200:16.0:1616513099.407595:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a5850 x1693954034717760/t0(0) o400->cb43a1f7-2b03-fd61-7824-0945869e7aae@10.31.141.2@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:18.0:1616513099.407607:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513099.407610:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693954034717760, offset 192 00000800:00000200:18.0:1616513099.407616:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:16.0:1616513099.407616:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.2@tcp1 00000400:00000200:16.0:1616513099.407624:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.2@tcp1 routed destination 00000400:00000200:18.0:1616513099.407626:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.2@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.407635:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.2@tcp1 of length 224 into portal 12 MB=0x604a4688156c0 00000400:00000200:16.0:1616513099.407639:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.2@tcp1(10.31.141.2@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:18.0:1616513099.407641:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.2@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 52976 00000800:00000200:16.0:1616513099.407646:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:18.0:1616513099.407649:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.407651:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:16.0:1616513099.407652:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.407654:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.407656:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513099.407658:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.407660:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.407661:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:16.0:1616513099.407661:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:18.0:1616513099.407664:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.407667:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.407668:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513099.407721:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693954034718400 00000800:00000200:0.0:1616513099.407725:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00010000:00000200:22.0:1616513099.407732:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b8d80 x1693954034718400/t0(0) o400->06e75f19-fe2f-5530-8182-3f87356ddd0e@10.31.141.2@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513099.407732:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.407735:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.407741:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:22.0:1616513099.407745:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693954034718400, offset 192 00000400:00000200:0.0:1616513099.407747:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789760 00000800:00000200:0.0:1616513099.407750:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513099.407751:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.2@tcp1 00000800:00000200:0.0:1616513099.407753:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513099.407758:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.2@tcp1 routed destination 00000800:00000200:7.2:1616513099.407768:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:22.0:1616513099.407772:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.2@tcp1(10.31.141.2@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.407779:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.407785:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.407788:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.407790:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513099.407794:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513099.407822:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:2.0:1616513099.407832:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:2.0:1616513099.407840:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.152@tcp1 : PUT - for me 00000400:00000200:2.0:1616513099.407850:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.152@tcp1 of length 224 into portal 26 MB=0x5ffd53446ff40 00000400:00000200:2.0:1616513099.407856:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.152@tcp1 of length 224/224 into md 0xf97d10a9 [1] + 0 00000400:00000200:2.0:1616513099.407861:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3760 00000400:00000200:2.0:1616513099.407864:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.407866:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:2.0:1616513099.407871:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3760 00000800:00000200:2.0:1616513099.407874:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (23)++ 00000800:00000200:14.0:1616513099.407876:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:2.0:1616513099.407879:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000100:00000200:16.0:1616513099.407883:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666053738304 00000800:00000200:2.0:1616513099.407883:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (24)-- 00000800:00000200:14.0:1616513099.407884:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (24)-- 00000800:00000200:2.0:1616513099.407886:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000400:00000200:14.0:1616513099.407887:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000200:16.0:1616513099.407889:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd050 x1688666053738304/t0(0) o400->99e720d7-0aa0-969d-95d4-9dcd27a3911d@10.31.132.152@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:26.0:1616513099.407890:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:2.0:1616513099.407891:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.407894:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.407898:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a3b8 00000400:00000200:2.0:1616513099.407898:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.152@tcp1 : PUT - for me 00000100:00000200:16.0:1616513099.407900:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666053738304, offset 192 00000800:00000200:14.0:1616513099.407902:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513099.407905:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.152@tcp1 00000400:00000200:2.0:1616513099.407905:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.152@tcp1 of length 224 into portal 12 MB=0x5ffd534470040 00000400:00000200:2.0:1616513099.407910:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.152@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 53200 00000400:00000200:16.0:1616513099.407912:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.152@tcp1 routed destination 00000400:00000200:2.0:1616513099.407914:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:2.0:1616513099.407917:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:2.0:1616513099.407922:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:2.0:1616513099.407924:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:16.0:1616513099.407925:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.152@tcp1(10.31.132.152@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:2.0:1616513099.407925:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:16.0:1616513099.407930:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:22.0:1616513099.407933:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666053738560 00000800:00000200:16.0:1616513099.407935:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00010000:00000200:22.0:1616513099.407939:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74647df500 x1688666053738560/t0(0) o400->e2071adb-3246-a2e2-ba36-6dc28fc51552@10.31.132.152@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513099.407939:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.407941:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.407944:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513099.407948:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513099.407949:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666053738560, offset 192 00000800:00000200:11.2:1616513099.407949:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513099.407955:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.152@tcp1 00000800:00000200:8.0:1616513099.407957:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513099.407960:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.152@tcp1 routed destination 00000800:00000200:8.0:1616513099.407961:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513099.407963:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513099.407964:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513099.407968:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513099.407972:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789ba0 00000400:00000200:22.0:1616513099.407973:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.152@tcp1(10.31.132.152@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.407974:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513099.407978:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.407982:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.407984:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.407986:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.407989:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.407994:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.408004:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.408009:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.408012:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.408013:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.408018:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.408022:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4add0 00000800:00000200:26.0:1616513099.408026:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.412793:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.412846:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.412855:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.412864:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.254@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.412873:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.254@tcp1 of length 224 into portal 12 MB=0x601aa11a30380 00000400:00000200:14.0:1616513099.412879:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.254@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 53424 00000400:00000200:14.0:1616513099.412884:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.412887:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513099.412893:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.412896:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.412898:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:2.0:1616513099.412901:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.412949:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513099.412956:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690679812227968 00010000:00000200:22.0:1616513099.412969:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebd100 x1690679812227968/t0(0) o400->0bca165b-bcf5-d5aa-c7c1-442b7ec13697@10.31.140.254@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.412983:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690679812227968, offset 192 00000400:00000200:22.0:1616513099.412989:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.254@tcp1 00000400:00000200:22.0:1616513099.412996:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.254@tcp1 routed destination 00000800:00000200:0.0:1616513099.413004:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513099.413011:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.254@tcp1(10.31.140.254@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:0.0:1616513099.413014:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513099.413018:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.413023:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.254@tcp1 : PUT - for me 00000800:00000200:22.0:1616513099.413024:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.413027:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513099.413029:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:0.0:1616513099.413032:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.254@tcp1 of length 224 into portal 26 MB=0x601aa11a30300 00000800:00000200:22.0:1616513099.413033:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:0.0:1616513099.413038:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.254@tcp1 of length 224/224 into md 0xf97d10b9 [1] + 0 00000400:00000200:0.0:1616513099.413043:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2880 00000400:00000200:0.0:1616513099.413047:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.413050:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.413054:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2880 00000800:00000200:0.0:1616513099.413057:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513099.413058:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513099.413060:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513099.413065:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513099.413066:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513099.413068:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.413070:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513099.413077:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513099.413081:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4acc0 00000800:00000200:24.0:1616513099.413085:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.413126:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690679812227840 00010000:00000200:16.0:1616513099.413137:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1050 x1690679812227840/t0(0) o400->2cc4444a-3bdf-0f3e-f3d9-a8f86d2bb9ed@10.31.140.254@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.413151:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690679812227840, offset 192 00000400:00000200:16.0:1616513099.413157:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.254@tcp1 00000400:00000200:16.0:1616513099.413165:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.254@tcp1 routed destination 00000400:00000200:16.0:1616513099.413180:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.254@tcp1(10.31.140.254@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.413186:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:16.0:1616513099.413192:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.413195:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:16.0:1616513099.413197:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:16.0:1616513099.413202:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.413207:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:16.0:1616513099.413208:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513099.413219:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.413226:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.413228:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.413235:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.413241:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788198 00000800:00000200:18.0:1616513099.413244:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.413247:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.423234:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.423247:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.423254:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.423261:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.63@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.423270:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.63@tcp1 of length 224 into portal 26 MB=0x5ffdc938476c0 00000800:00000200:7.2:1616513099.423273:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513099.423275:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.63@tcp1 of length 224/224 into md 0xf97d10d9 [1] + 0 00000400:00000200:8.0:1616513099.423279:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2e58 00000400:00000200:8.0:1616513099.423282:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.423284:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:14.0:1616513099.423288:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513099.423288:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2e58 00000800:00000200:8.0:1616513099.423290:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.423293:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.423294:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:14.0:1616513099.423297:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.423297:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:6.0:1616513099.423301:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513099.423306:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.63@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.423314:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.63@tcp1 of length 224 into portal 12 MB=0x5ffdc93847a40 00000400:00000200:14.0:1616513099.423320:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.63@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 53648 00000400:00000200:14.0:1616513099.423325:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.423327:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513099.423333:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.423336:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.423338:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.423349:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697716373184 00010000:00000200:16.0:1616513099.423359:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b3850 x1688697716373184/t0(0) o400->637f0d4a-779f-b529-7bc0-458412abce15@10.31.141.63@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.423372:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688697716373184, offset 192 00000400:00000200:16.0:1616513099.423378:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.63@tcp1 00000400:00000200:16.0:1616513099.423384:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.63@tcp1 routed destination 00000100:00000200:22.0:1616513099.423397:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688697716374080 00000400:00000200:16.0:1616513099.423398:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.63@tcp1(10.31.141.63@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.423404:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:22.0:1616513099.423409:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebc380 x1688697716374080/t0(0) o400->9b736eef-ce59-c2f6-3c9b-2d95ad6e4a3e@10.31.141.63@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513099.423409:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.423412:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.423414:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.423419:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:22.0:1616513099.423423:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688697716374080, offset 192 00000800:00000200:11.2:1616513099.423424:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.423425:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:22.0:1616513099.423430:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.63@tcp1 00000800:00000200:24.0:1616513099.423447:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513099.423450:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.63@tcp1 routed destination 00000800:00000200:24.0:1616513099.423453:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.423456:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513099.423462:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:20.0:1616513099.423462:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:22.0:1616513099.423466:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.63@tcp1(10.31.141.63@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513099.423468:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788f68 00000800:00000200:24.0:1616513099.423471:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513099.423472:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.423478:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.423481:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.423494:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.423498:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.423503:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.423513:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.423520:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.423522:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.423528:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.423532:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bd38 00000800:00000200:18.0:1616513099.423536:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.423539:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.430903:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.430960:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.430970:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.430970:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.430979:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.141@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.430988:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.141@tcp1 of length 224 into portal 26 MB=0x603fcd6dba7c0 00000800:00000200:7.2:1616513099.430995:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513099.430995:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.141@tcp1 of length 224/224 into md 0xf97d10e9 [1] + 0 00000400:00000200:0.0:1616513099.431000:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf24c8 00000400:00000200:0.0:1616513099.431003:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.431005:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.431010:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf24c8 00000800:00000200:0.0:1616513099.431013:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.431015:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.431017:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:6.0:1616513099.431054:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513099.431063:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513099.431066:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:6.0:1616513099.431072:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.141@tcp1 : PUT - for me 00000400:00000200:6.0:1616513099.431080:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.141@tcp1 of length 224 into portal 12 MB=0x603fcd6dba840 00000100:00000200:16.0:1616513099.431084:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234331625408 00000400:00000200:6.0:1616513099.431086:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.141@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 53872 00000400:00000200:6.0:1616513099.431091:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:6.0:1616513099.431093:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:16.0:1616513099.431094:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3850 x1693234331625408/t0(0) o400->47d376d6-81af-db3e-02e5-afb3b9d94c10@10.31.138.141@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:6.0:1616513099.431100:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513099.431102:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513099.431104:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.431107:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693234331625408, offset 192 00000400:00000200:16.0:1616513099.431113:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.141@tcp1 00000400:00000200:16.0:1616513099.431120:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.141@tcp1 routed destination 00000400:00000200:16.0:1616513099.431133:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.141@tcp1(10.31.138.141@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.431140:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.431145:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.431147:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.431149:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.431154:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.431159:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.431160:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:22.0:1616513099.431161:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693234331625536 00010000:00000200:22.0:1616513099.431173:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74d9647080 x1693234331625536/t0(0) o400->cf85d758-d77e-f012-9b7b-7b5825cb3828@10.31.138.141@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.431187:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693234331625536, offset 192 00000400:00000200:22.0:1616513099.431193:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.141@tcp1 00000400:00000200:22.0:1616513099.431200:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.141@tcp1 routed destination 00000400:00000200:22.0:1616513099.431213:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.141@tcp1(10.31.138.141@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.431220:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.431225:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.431227:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513099.431227:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513099.431229:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.431234:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513099.431234:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.431236:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513099.431238:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.431239:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.431243:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513099.431248:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513099.431249:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788b28 00000800:00000200:20.0:1616513099.431252:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513099.431255:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.431258:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.431264:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.431268:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bba0 00000800:00000200:18.0:1616513099.431272:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.431274:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.438308:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.438322:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.438327:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.438333:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.187@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.438341:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.187@tcp1 of length 224 into portal 26 MB=0x5ffd487f2ef00 00000400:00000200:8.0:1616513099.438346:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.187@tcp1 of length 224/224 into md 0xf97d1109 [1] + 0 00000400:00000200:8.0:1616513099.438349:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2550 00000400:00000200:8.0:1616513099.438351:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.438352:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.438356:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2550 00000800:00000200:8.0:1616513099.438358:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.438359:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.438361:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.438363:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.438416:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663162547968 00010000:00000200:16.0:1616513099.438426:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6050 x1688663162547968/t0(0) o400->c3ff8d28-3b62-ed04-be23-e5fdcc630261@10.31.136.187@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.438465:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663162547968, offset 192 00000400:00000200:16.0:1616513099.438471:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.187@tcp1 00000400:00000200:16.0:1616513099.438479:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.187@tcp1 routed destination 00000400:00000200:16.0:1616513099.438493:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.187@tcp1(10.31.136.187@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.438499:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.438504:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.438507:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.438509:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.438513:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.438519:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.438520:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:24.0:1616513099.438531:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.438537:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.438540:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513099.438547:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:24.0:1616513099.438552:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7883b8 00000800:00000200:24.0:1616513099.438555:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.438558:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.438672:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.438723:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.438736:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.438744:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.187@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.438752:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.187@tcp1 of length 224 into portal 12 MB=0x5ffd487f2f040 00000400:00000200:14.0:1616513099.438757:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.187@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 54096 00000400:00000200:14.0:1616513099.438762:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.438764:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513099.438769:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.438772:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.438773:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.438776:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.438830:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663162548288 00010000:00000200:22.0:1616513099.438842:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebb600 x1688663162548288/t0(0) o400->b9762d49-4c14-6be0-aa43-65a3903561b6@10.31.136.187@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.438855:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663162548288, offset 192 00000400:00000200:22.0:1616513099.438861:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.187@tcp1 00000400:00000200:22.0:1616513099.438868:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.187@tcp1 routed destination 00000400:00000200:22.0:1616513099.438884:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.187@tcp1(10.31.136.187@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.438890:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.438896:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.438898:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.438900:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.438905:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.438922:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.438933:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.438939:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.438942:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.438948:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.438951:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b5c8 00000800:00000200:18.0:1616513099.438955:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.438958:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.439915:0:51529:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.439990:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.439997:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513099.440004:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.123@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.440013:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.123@tcp1 of length 224 into portal 26 MB=0x60182b3ea1ec0 00000400:00000200:14.0:1616513099.440018:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.123@tcp1 of length 224/224 into md 0xf97d1139 [1] + 0 00000400:00000200:14.0:1616513099.440022:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2ff0 00000400:00000200:14.0:1616513099.440024:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.440026:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513099.440029:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2ff0 00000800:00000200:14.0:1616513099.440032:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.440034:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513099.440036:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.440038:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.440089:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510736105152 00010000:00000200:16.0:1616513099.440098:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be050 x1690510736105152/t0(0) o400->0a13c080-c62b-918e-1dc3-ca3657e0e279@10.31.132.123@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.440110:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510736105152, offset 192 00000400:00000200:16.0:1616513099.440117:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.123@tcp1 00000400:00000200:16.0:1616513099.440124:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.123@tcp1 routed destination 00000400:00000200:16.0:1616513099.440137:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.123@tcp1(10.31.132.123@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.440143:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.440148:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.440150:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.440152:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.440156:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513099.440161:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.440203:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.440214:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.440217:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513099.440218:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.440223:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513099.440227:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788220 00000800:00000200:8.0:1616513099.440229:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.440231:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.440451:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.440463:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.440472:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513099.440482:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.123@tcp1 : PUT - for me 00000400:00000200:24.0:1616513099.440490:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.123@tcp1 of length 224 into portal 12 MB=0x60182b3ea2000 00000400:00000200:24.0:1616513099.440496:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.123@tcp1 of length 224/224 into md 0x865be291 [1] + 86088 00000400:00000200:24.0:1616513099.440502:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513099.440504:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513099.440511:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.440514:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513099.440516:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.440519:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.440526:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510736105472 00010000:00000200:8.0:1616513099.440534:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d85193ede80 x1690510736105472/t0(0) o400->3b7fa5d4-2431-a3b0-e3a8-58918066dc43@10.31.132.123@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513099.440545:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510736105472, offset 192 00000400:00000200:8.0:1616513099.440550:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.123@tcp1 00000400:00000200:8.0:1616513099.440555:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.123@tcp1 routed destination 00000400:00000200:8.0:1616513099.440567:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.123@tcp1(10.31.132.123@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.440572:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.440577:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.440579:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.440580:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.440588:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.440592:0:51529:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.440644:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.440652:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.440654:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.440661:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.440664:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2550 00000800:00000200:18.0:1616513099.440668:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.440671:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.446190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513099.446240:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.446249:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:6.0:1616513099.446252:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513099.446256:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.190@tcp1 : PUT - for me 00000400:00000200:14.0:1616513099.446264:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.190@tcp1 of length 224 into portal 26 MB=0x5ffd4b7e38d80 00000400:00000200:14.0:1616513099.446269:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.190@tcp1 of length 224/224 into md 0xf97d1159 [1] + 0 00000400:00000200:14.0:1616513099.446273:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3ba0 00000400:00000200:14.0:1616513099.446275:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.446278:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513099.446282:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ba0 00000800:00000200:14.0:1616513099.446284:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.446286:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.446288:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.446340:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663966846336 00010000:00000200:16.0:1616513099.446350:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2050 x1688663966846336/t0(0) o400->b1c30e64-1ded-8b5d-210e-387b421bb2de@10.31.136.190@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.446362:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663966846336, offset 192 00000400:00000200:16.0:1616513099.446368:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.190@tcp1 00000400:00000200:16.0:1616513099.446374:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.190@tcp1 routed destination 00000400:00000200:16.0:1616513099.446387:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.190@tcp1(10.31.136.190@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.446393:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.446398:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.446400:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.446402:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.446406:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513099.446412:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.446414:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.446424:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.446428:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513099.446429:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.446434:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513099.446446:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788a18 00000800:00000200:8.0:1616513099.446448:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.446450:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.446531:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.446544:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.446553:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:24.0:1616513099.446562:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.190@tcp1 : PUT - for me 00000400:00000200:24.0:1616513099.446570:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.190@tcp1 of length 224 into portal 12 MB=0x5ffd4b7e38e40 00000400:00000200:24.0:1616513099.446577:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.190@tcp1 of length 224/224 into md 0x865be291 [1] + 86312 00000400:00000200:24.0:1616513099.446584:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513099.446586:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513099.446594:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.446596:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513099.446598:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513099.446602:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:8.0:1616513099.446602:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663966846528 00010000:00000200:8.0:1616513099.446607:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d85193e8480 x1688663966846528/t0(0) o400->a16dc293-2df5-62e1-9a68-170205a27e56@10.31.136.190@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:8.0:1616513099.446616:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663966846528, offset 192 00000400:00000200:8.0:1616513099.446620:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.190@tcp1 00000400:00000200:8.0:1616513099.446625:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.190@tcp1 routed destination 00000400:00000200:8.0:1616513099.446636:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.190@tcp1(10.31.136.190@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.446641:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.446645:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:8.0:1616513099.446647:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.446649:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.446651:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.446656:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.446708:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.446714:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.446717:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513099.446723:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.446726:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2e58 00000800:00000200:18.0:1616513099.446730:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.446733:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513099.466562:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929936, portal 4 00000100:00000200:26.0:1616513099.466568:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929936, offset 0 00000400:00000200:26.0:1616513099.466574:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:26.0:1616513099.466580:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:26.0:1616513099.466594:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513099.466601:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:26.0:1616513099.466607:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513099.466610:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513099.466612:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513099.466617:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513099.466665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:6.0:1616513099.466679:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:6.0:1616513099.466684:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:6.0:1616513099.466686:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513099.466691:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:6.0:1616513099.466696:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74645b9f80 x1693067725929936/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:6.0:1616513099.466705:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aea18 00000800:00000200:6.0:1616513099.466709:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:6.0:1616513099.466711:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513099.466833:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:18.0:1616513099.466845:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513099.466854:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:26.0:1616513099.466855:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:18.0:1616513099.466862:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:18.0:1616513099.466870:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627dd0 00000400:00000200:18.0:1616513099.466876:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d17d9 [1] + 192 00000400:00000200:18.0:1616513099.466882:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.466887:0:6471:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74645b9f80 x1693067725929936/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:18.0:1616513099.466901:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:18.0:1616513099.466904:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:18.0:1616513099.466906:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:26.0:1616513099.466907:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74645b9f80 x1693067725929936/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513099.466920:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74645b9f80 x1693067725929936/t0(0) o13->boslfs02-OST0024-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513099.466930:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa90 00000800:00000200:11.2:1616513099.475006:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.475018:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.475025:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.475032:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.114@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.475039:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.114@tcp1 of length 224 into portal 26 MB=0x6001dc28d3e80 00000400:00000200:8.0:1616513099.475044:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.114@tcp1 of length 224/224 into md 0xf97d1169 [1] + 0 00000400:00000200:8.0:1616513099.475047:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2d48 00000400:00000200:8.0:1616513099.475050:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.475052:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.475055:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2d48 00000800:00000200:8.0:1616513099.475057:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.475060:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.475061:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.475063:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.475067:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513099.475070:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513099.475072:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.114@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.475076:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.114@tcp1 of length 224 into portal 12 MB=0x6001dc28d3f00 00000400:00000200:8.0:1616513099.475080:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.114@tcp1 of length 224/224 into md 0x865be291 [1] + 86536 00000400:00000200:8.0:1616513099.475083:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.475085:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513099.475090:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.475092:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.475093:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.475115:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688977678352000 00010000:00000200:16.0:1616513099.475125:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf050 x1688977678352000/t0(0) o400->343ff8ba-2f36-bb1f-6904-4ea1723e2309@10.31.142.114@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.475137:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688977678352000, offset 192 00000400:00000200:16.0:1616513099.475142:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.114@tcp1 00000400:00000200:16.0:1616513099.475149:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.114@tcp1 routed destination 00000100:00000200:12.0:1616513099.475161:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688977678352128 00000400:00000200:16.0:1616513099.475162:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.114@tcp1(10.31.142.114@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.475171:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:12.0:1616513099.475173:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8bf00 x1688977678352128/t0(0) o400->34c579d2-8968-92c5-dce4-d725ac2fcd17@10.31.142.114@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513099.475176:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.475178:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.475180:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.475184:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:12.0:1616513099.475188:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688977678352128, offset 192 00000800:00000200:11.2:1616513099.475189:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.475190:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:12.0:1616513099.475194:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.114@tcp1 00000800:00000200:24.0:1616513099.475200:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:12.0:1616513099.475201:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.114@tcp1 routed destination 00000800:00000200:24.0:1616513099.475207:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:24.0:1616513099.475209:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.475211:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.475216:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:12.0:1616513099.475216:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.114@tcp1(10.31.142.114@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513099.475221:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789540 00000800:00000200:12.0:1616513099.475223:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.475224:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:12.0:1616513099.475229:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.475233:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513099.475235:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513099.475240:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.475246:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513099.475257:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513099.475262:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513099.475264:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:6.0:1616513099.475269:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513099.475272:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be550 00000800:00000200:6.0:1616513099.475276:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513099.475279:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.483394:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.483449:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.483458:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:11.2:1616513099.483458:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:26.0:1616513099.483466:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.224@tcp1 : PUT - for me 00000800:00000200:8.0:1616513099.483468:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.483473:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:26.0:1616513099.483475:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.224@tcp1 of length 224 into portal 26 MB=0x5fffd5808b9c0 00000400:00000200:8.0:1616513099.483479:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.224@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.483481:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.224@tcp1 of length 224/224 into md 0xf97d1189 [1] + 0 00000400:00000200:8.0:1616513099.483484:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.224@tcp1 of length 224 into portal 12 MB=0x5fffd5808ba80 00000400:00000200:26.0:1616513099.483485:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2bb0 00000400:00000200:26.0:1616513099.483488:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.483488:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.224@tcp1 of length 224/224 into md 0x865be291 [1] + 86760 00000100:00000200:26.0:1616513099.483491:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.483491:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.483492:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:26.0:1616513099.483494:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2bb0 00000800:00000200:8.0:1616513099.483496:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513099.483497:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513099.483498:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.483499:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513099.483500:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.483501:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513099.483501:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513099.483503:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.483576:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688838452328896 00000100:00000200:12.0:1616513099.483577:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688838452329088 00010000:00000200:16.0:1616513099.483587:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668050 x1688838452328896/t0(0) o400->f112f123-998b-b65a-4277-38662078d66b@10.31.136.224@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:12.0:1616513099.483589:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8da00 x1688838452329088/t0(0) o400->c43c42f0-6db6-7699-82e9-8be017a42b37@10.31.136.224@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:16.0:1616513099.483602:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688838452328896, offset 192 00000100:00000200:12.0:1616513099.483604:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688838452329088, offset 192 00000400:00000200:16.0:1616513099.483608:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.224@tcp1 00000400:00000200:12.0:1616513099.483614:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.224@tcp1 00000400:00000200:16.0:1616513099.483615:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.224@tcp1 routed destination 00000400:00000200:16.0:1616513099.483630:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.224@tcp1(10.31.136.224@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.483637:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:12.0:1616513099.483639:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.224@tcp1 routed destination 00000800:00000200:16.0:1616513099.483647:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.483650:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.483652:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:12.0:1616513099.483655:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.224@tcp1(10.31.136.224@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.483657:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:12.0:1616513099.483661:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513099.483661:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.483664:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:12.0:1616513099.483667:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:12.0:1616513099.483670:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:12.0:1616513099.483672:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:12.0:1616513099.483677:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.483681:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513099.483715:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.483722:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.483724:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513099.483727:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:6.0:1616513099.483730:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:0.0:1616513099.483731:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:6.0:1616513099.483735:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513099.483737:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.483737:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788110 00000800:00000200:0.0:1616513099.483740:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:6.0:1616513099.483742:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513099.483745:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf210 00000800:00000200:6.0:1616513099.483748:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:6.0:1616513099.483750:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.490617:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.490625:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.490630:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.490635:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.48@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.490641:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.48@tcp1 of length 224 into portal 26 MB=0x5fff348c7b240 00000400:00000200:8.0:1616513099.490645:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.48@tcp1 of length 224/224 into md 0xf97d1199 [1] + 0 00000400:00000200:8.0:1616513099.490647:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2198 00000400:00000200:8.0:1616513099.490649:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.490650:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.490653:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2198 00000800:00000200:8.0:1616513099.490654:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.490656:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.490658:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.490659:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.490723:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795246735936 00010000:00000200:16.0:1616513099.490734:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669050 x1688795246735936/t0(0) o400->6b1bca5a-1ae6-e972-fd65-371ce2d2bb7f@10.31.136.48@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.490748:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688795246735936, offset 192 00000800:00000200:7.2:1616513099.490754:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:16.0:1616513099.490755:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.48@tcp1 00000400:00000200:16.0:1616513099.490762:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.48@tcp1 routed destination 00000400:00000200:16.0:1616513099.490777:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.48@tcp1(10.31.136.48@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.490784:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.490789:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.490792:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.490796:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.490801:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.490806:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.490807:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513099.490809:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.490818:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.490818:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.490825:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:26.0:1616513099.490827:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.48@tcp1 : PUT - for me 00000400:00000200:24.0:1616513099.490827:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.490833:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.490834:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.490836:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.48@tcp1 of length 224 into portal 12 MB=0x5fff348c7b300 00000400:00000200:24.0:1616513099.490840:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789430 00000400:00000200:26.0:1616513099.490842:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.48@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 54320 00000800:00000200:24.0:1616513099.490843:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513099.490847:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.490850:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513099.490858:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.490861:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.490863:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.490866:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.490926:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688795246736128 00010000:00000200:22.0:1616513099.490938:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21cc80 x1688795246736128/t0(0) o400->3c4ba62a-54bb-34cd-1953-851b7d717e1d@10.31.136.48@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.490952:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688795246736128, offset 192 00000400:00000200:22.0:1616513099.490959:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.48@tcp1 00000400:00000200:22.0:1616513099.490966:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.48@tcp1 routed destination 00000400:00000200:22.0:1616513099.490981:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.48@tcp1(10.31.136.48@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.490987:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.490993:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.490996:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.490998:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.491002:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.491008:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:6.0:1616513099.491056:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:6.0:1616513099.491061:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:6.0:1616513099.491063:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513099.491067:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:6.0:1616513099.491067:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:6.0:1616513099.491070:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b760 00000800:00000200:6.0:1616513099.491073:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:27.0:1616513099.514582:0:6507:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929952, portal 4 00000100:00000200:27.0:1616513099.514588:0:6507:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929952, offset 0 00000400:00000200:27.0:1616513099.514595:0:6507:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:27.0:1616513099.514602:0:6507:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:27.0:1616513099.514616:0:6507:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:27.0:1616513099.514623:0:6507:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:27.0:1616513099.514630:0:6507:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:27.0:1616513099.514633:0:6507:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:27.0:1616513099.514636:0:6507:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:27.0:1616513099.514642:0:6507:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000100:00000200:14.0:1616513099.514685:0:6502:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929968, portal 4 00000800:00000200:25.2:1616513099.514689:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000100:00000200:14.0:1616513099.514691:0:6502:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929968, offset 0 00000400:00000200:14.0:1616513099.514698:0:6502:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:14.0:1616513099.514704:0:6502:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:14.0:1616513099.514717:0:6502:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:14.0:1616513099.514724:0:6502:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000100:00000200:19.0:1616513099.514727:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725929984, portal 4 00000800:00000200:14.0:1616513099.514730:0:6502:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000100:00000200:19.0:1616513099.514733:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725929984, offset 0 00000800:00000200:14.0:1616513099.514733:0:6502:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:14.0:1616513099.514735:0:6502:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000400:00000200:19.0:1616513099.514740:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000800:00000200:14.0:1616513099.514741:0:6502:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:26.0:1616513099.514745:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000400:00000200:19.0:1616513099.514746:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000800:00000200:27.2:1616513099.514751:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:26.0:1616513099.514752:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513099.514755:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:19.0:1616513099.514760:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000400:00000200:26.0:1616513099.514763:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000800:00000200:27.0:1616513099.514765:0:6475:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:19.0:1616513099.514767:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000100:00000200:26.0:1616513099.514769:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d84456eec00 x1693067725929952/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513099.514771:0:6475:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513099.514774:0:6475:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:19.0:1616513099.514774:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:31.0:1616513099.514777:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:19.0:1616513099.514777:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513099.514780:0:6475:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:19.0:1616513099.514780:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:26.0:1616513099.514781:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249320 00000800:00000200:19.0:1616513099.514785:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (23)-- 00000100:00000200:27.0:1616513099.514786:0:6475:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bde300 x1693067725929968/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:26.0:1616513099.514786:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:27.2:1616513099.514789:0:6475:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:26.0:1616513099.514789:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:31.0:1616513099.514798:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)++ 00000400:00000200:27.0:1616513099.514805:0:6475:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd47575c8 00000800:00000200:31.0:1616513099.514807:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (24)-- 00000400:00000200:31.0:1616513099.514809:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:27.0:1616513099.514809:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:25.2:1616513099.514809:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:27.0:1616513099.514812:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:31.0:1616513099.514818:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000800:00000200:14.0:1616513099.514821:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000100:00000200:31.0:1616513099.514824:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d840be44800 x1693067725929984/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513099.514828:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:6.0:1616513099.514832:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:31.0:1616513099.514836:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707dc0 00000400:00000200:14.0:1616513099.514836:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000800:00000200:31.0:1616513099.514841:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:14.0:1616513099.514843:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627de0 00000400:00000200:14.0:1616513099.514849:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d1839 [1] + 192 00000400:00000200:14.0:1616513099.514854:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.514859:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d84456eec00 x1693067725929952/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513099.514874:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513099.514876:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513099.514878:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:27.0:1616513099.514882:0:6507:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d84456eec00 x1693067725929952/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:27.0:1616513099.514894:0:6507:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d84456eec00 x1693067725929952/t0(0) o13->boslfs02-OST0030-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513099.514903:0:6507:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8f1d249650 00000800:00000200:27.2:1616513099.514981:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513099.514991:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.514996:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.231@o2ib1 00000800:00000200:3.0:1616513099.515000:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:27.0:1616513099.515004:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000800:00000200:3.0:1616513099.515010:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513099.515011:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513099.515012:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627df0 00000400:00000200:3.0:1616513099.515019:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513099.515020:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716775d [1] + 192 00000400:00000200:27.0:1616513099.515026:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:3.0:1616513099.515027:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627e00 00000100:00000200:27.0:1616513099.515030:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bde300 x1693067725929968/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513099.515044:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)++ 00000400:00000200:3.0:1616513099.515045:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716776d [1] + 192 00000800:00000200:27.0:1616513099.515046:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (23)-- 00000800:00000200:27.0:1616513099.515048:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:3.0:1616513099.515050:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513099.515054:0:6502:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bde300 x1693067725929968/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:14.0:1616513099.515065:0:6502:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bde300 x1693067725929968/t0(0) o13->boslfs02-OST0026-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513099.515075:0:6502:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dd4757ed0 00000100:00000200:3.0:1616513099.515082:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d840be44800 x1693067725929984/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:3.0:1616513099.515097:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:3.0:1616513099.515100:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:3.0:1616513099.515102:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:19.0:1616513099.515103:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d840be44800 x1693067725929984/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513099.515128:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d840be44800 x1693067725929984/t0(0) o13->boslfs02-OST003e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513099.515137:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707980 00000800:00000200:7.2:1616513099.523492:0:51538:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.523546:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.523556:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513099.523565:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.118@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.523574:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.118@tcp1 of length 224 into portal 26 MB=0x5f90ff8cc78c0 00000400:00000200:26.0:1616513099.523581:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.118@tcp1 of length 224/224 into md 0xf97d11b9 [1] + 0 00000400:00000200:26.0:1616513099.523586:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf34b8 00000400:00000200:26.0:1616513099.523589:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.523591:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513099.523596:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf34b8 00000800:00000200:26.0:1616513099.523599:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.523600:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.523602:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.523607:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.523610:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.523610:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.523619:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.118@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.523628:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.118@tcp1 of length 224 into portal 12 MB=0x5f90ff8cc7940 00000400:00000200:18.0:1616513099.523634:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.118@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 54544 00000400:00000200:18.0:1616513099.523639:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.523642:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.523651:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.523654:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.523656:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.523658:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681221877528768 00010000:00000200:16.0:1616513099.523669:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9850 x1681221877528768/t0(0) o400->251a1a76-3509-326a-abfe-1ae67dc0bd2c@10.31.143.118@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.523682:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681221877528768, offset 192 00000400:00000200:16.0:1616513099.523688:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.118@tcp1 00000400:00000200:16.0:1616513099.523695:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.118@tcp1 routed destination 00000400:00000200:16.0:1616513099.523709:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.118@tcp1(10.31.143.118@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:22.0:1616513099.523714:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681221877528896 00000800:00000200:16.0:1616513099.523715:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.523721:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.523723:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.523725:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:22.0:1616513099.523726:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bd100 x1681221877528896/t0(0) o400->f55c6b9a-9fc1-df9d-a215-903027630866@10.31.143.118@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:16.0:1616513099.523730:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:16.0:1616513099.523736:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.523737:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513099.523741:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681221877528896, offset 192 00000400:00000200:22.0:1616513099.523747:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.118@tcp1 00000800:00000200:8.0:1616513099.523747:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.523752:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:22.0:1616513099.523754:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.118@tcp1 routed destination 00000400:00000200:8.0:1616513099.523754:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.523759:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513099.523762:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789a90 00000800:00000200:8.0:1616513099.523765:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.523767:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:22.0:1616513099.523769:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.118@tcp1(10.31.143.118@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.523776:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.523781:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.523784:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.523788:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.523793:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.523797:0:51538:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.523849:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.523856:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.523858:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513099.523859:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513099.523864:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.523868:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4bc28 00000800:00000200:14.0:1616513099.523871:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.558083:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.558141:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.558153:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513099.558156:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.558162:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.96@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.558173:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.96@tcp1 of length 224 into portal 26 MB=0x5ffdd2139dd80 00000400:00000200:18.0:1616513099.558180:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.96@tcp1 of length 224/224 into md 0xf97d11d9 [1] + 0 00000400:00000200:18.0:1616513099.558185:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3f58 00000400:00000200:18.0:1616513099.558187:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.558190:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.558195:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3f58 00000800:00000200:18.0:1616513099.558198:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.558201:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.558203:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:16.0:1616513099.558263:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700093848960 00010000:00000200:16.0:1616513099.558275:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fb850 x1688700093848960/t0(0) o400->f88a3759-d38e-99b7-bc62-d4b55a750e72@10.31.128.96@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.558290:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700093848960, offset 192 00000400:00000200:16.0:1616513099.558296:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.96@tcp1 00000400:00000200:16.0:1616513099.558304:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.96@tcp1 routed destination 00000400:00000200:16.0:1616513099.558319:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.96@tcp1(10.31.128.96@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.558326:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.558331:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.558334:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:16.0:1616513099.558336:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:16.0:1616513099.558341:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513099.558345:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513099.558348:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.558349:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513099.558360:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.558368:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.558370:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513099.558373:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513099.558377:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.558383:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789a90 00000800:00000200:0.0:1616513099.558386:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513099.558397:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.558406:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000800:00000200:14.0:1616513099.558410:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.558414:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.96@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.558423:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.96@tcp1 of length 224 into portal 12 MB=0x5ffdd2139e080 00000400:00000200:26.0:1616513099.558429:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.96@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 54768 00000400:00000200:26.0:1616513099.558446:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.558451:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513099.558458:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.558461:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.558463:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:22.0:1616513099.558482:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700093849728 00010000:00000200:22.0:1616513099.558495:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74dd21ec00 x1688700093849728/t0(0) o400->a955a133-dd2c-fd56-6399-ef4a29cce99c@10.31.128.96@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:22.0:1616513099.558509:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700093849728, offset 192 00000400:00000200:22.0:1616513099.558516:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.96@tcp1 00000400:00000200:22.0:1616513099.558523:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.96@tcp1 routed destination 00000400:00000200:22.0:1616513099.558538:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.96@tcp1(10.31.128.96@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.558545:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.558551:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:22.0:1616513099.558554:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.558556:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.558561:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.558565:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.558575:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.558582:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513099.558585:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.558586:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513099.558591:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513099.558595:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4b078 00000800:00000200:10.0:1616513099.558599:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.571008:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.571018:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.571024:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.571029:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.252@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.571036:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.252@tcp1 of length 224 into portal 26 MB=0x6048d685ac980 00000400:00000200:8.0:1616513099.571048:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.252@tcp1 of length 224/224 into md 0xf97d1209 [1] + 0 00000400:00000200:8.0:1616513099.571051:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf35c8 00000400:00000200:8.0:1616513099.571053:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.571054:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.571058:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf35c8 00000800:00000200:8.0:1616513099.571060:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.571061:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.571063:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.571064:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:16.0:1616513099.571134:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855247944064 00010000:00000200:16.0:1616513099.571149:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9850 x1693855247944064/t0(0) o400->a4e01450-2dfe-e0b5-ea92-e3a5b34cc07b@10.31.140.252@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.571165:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693855247944064, offset 192 00000400:00000200:16.0:1616513099.571171:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.252@tcp1 00000800:00000200:11.2:1616513099.571171:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:16.0:1616513099.571179:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.252@tcp1 routed destination 00000800:00000200:24.0:1616513099.571191:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:16.0:1616513099.571194:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.252@tcp1(10.31.140.252@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.571201:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.571201:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.571207:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513099.571210:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.252@tcp1 : PUT - for me 00000800:00000200:16.0:1616513099.571210:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513099.571212:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000400:00000200:24.0:1616513099.571218:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.252@tcp1 of length 224 into portal 12 MB=0x6048d685acac0 00000800:00000200:16.0:1616513099.571218:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (24)-- 00000100:00000200:16.0:1616513099.571224:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:24.0:1616513099.571225:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.252@tcp1 of length 224/224 into md 0x865be291 [1] + 86984 00000400:00000200:24.0:1616513099.571230:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:24.0:1616513099.571232:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513099.571241:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513099.571244:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513099.571246:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513099.571253:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513099.571255:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:24.0:1616513099.571257:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513099.571258:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.571264:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:20.0:1616513099.571265:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693855247944384 00000400:00000200:0.0:1616513099.571270:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788110 00000800:00000200:0.0:1616513099.571273:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:20.0:1616513099.571278:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d85193e8d80 x1693855247944384/t0(0) o400->5da67e50-8b7b-b8a5-08ef-afa4179160a7@10.31.140.252@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513099.571293:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693855247944384, offset 192 00000400:00000200:20.0:1616513099.571299:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.252@tcp1 00000400:00000200:20.0:1616513099.571307:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.252@tcp1 routed destination 00000400:00000200:20.0:1616513099.571322:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.252@tcp1(10.31.140.252@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513099.571329:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513099.571335:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513099.571337:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513099.571340:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513099.571344:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.571351:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.571404:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.571410:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.571412:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513099.571419:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.571422:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79a2a8 00000800:00000200:14.0:1616513099.571426:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.571429:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.610442:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.610520:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.610530:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513099.610533:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.610540:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.236@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.610551:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.236@tcp1 of length 224 into portal 12 MB=0x5ffd43e6eca40 00000400:00000200:18.0:1616513099.610557:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.236@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 54992 00000400:00000200:18.0:1616513099.610563:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.610566:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.610574:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.610578:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.610579:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.610585:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513099.610610:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:22.0:1616513099.610644:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661929151040 00000800:00000200:14.0:1616513099.610644:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.610654:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00010000:00000200:22.0:1616513099.610658:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bcc80 x1688661929151040/t0(0) o400->31f40405-d867-5319-fb21-3fa430c839f1@10.31.132.236@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:14.0:1616513099.610663:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.52@tcp1 : PUT - for me 00000800:00000200:20.0:1616513099.610669:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:14.0:1616513099.610672:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.52@tcp1 of length 224 into portal 26 MB=0x5ffdc65cab240 00000100:00000200:22.0:1616513099.610673:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661929151040, offset 192 00000800:00000200:20.0:1616513099.610679:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:14.0:1616513099.610679:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.52@tcp1 of length 224/224 into md 0xf97d1219 [1] + 0 00000800:00000200:8.0:1616513099.610679:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:14.0:1616513099.610684:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf26e8 00000400:00000200:14.0:1616513099.610687:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.610688:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.236@tcp1 : PUT - for me 00000400:00000200:22.0:1616513099.610689:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.236@tcp1 00000100:00000200:14.0:1616513099.610692:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513099.610697:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.236@tcp1 routed destination 00000400:00000200:14.0:1616513099.610697:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf26e8 00000800:00000200:26.0:1616513099.610698:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513099.610698:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.236@tcp1 of length 224 into portal 26 MB=0x5ffd43e6ec9c0 00000400:00000200:20.0:1616513099.610704:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.236@tcp1 of length 224/224 into md 0xf97d1229 [1] + 0 00000800:00000200:14.0:1616513099.610704:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.610707:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:20.0:1616513099.610709:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2110 00000800:00000200:14.0:1616513099.610709:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:22.0:1616513099.610713:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.236@tcp1(10.31.132.236@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:20.0:1616513099.610714:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.610716:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513099.610720:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:20.0:1616513099.610722:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2110 00000800:00000200:20.0:1616513099.610725:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513099.610726:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513099.610727:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:22.0:1616513099.610729:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.610729:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513099.610731:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513099.610736:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.610740:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:16.0:1616513099.610767:0:245737:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696949224000 00000100:00000200:24.0:1616513099.610777:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661929150912 00010000:00000200:16.0:1616513099.610777:0:245737:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72050 x1688696949224000/t0(0) o400->8777dfc2-c70b-ea12-57a0-0382c375581b@10.31.129.52@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:24.0:1616513099.610789:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc850 x1688661929150912/t0(0) o400->494f0376-9872-c634-c02c-66a3357800d1@10.31.132.236@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:16.0:1616513099.610791:0:245737:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688696949224000, offset 192 00000400:00000200:16.0:1616513099.610797:0:245737:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.52@tcp1 00000100:00000200:24.0:1616513099.610804:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661929150912, offset 192 00000400:00000200:16.0:1616513099.610805:0:245737:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.52@tcp1 routed destination 00000400:00000200:24.0:1616513099.610811:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.236@tcp1 00000400:00000200:24.0:1616513099.610819:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.236@tcp1 routed destination 00000400:00000200:16.0:1616513099.610819:0:245737:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.52@tcp1(10.31.129.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:16.0:1616513099.610826:0:245737:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.610828:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:24.0:1616513099.610834:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.236@tcp1(10.31.132.236@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.610836:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:28.0:1616513099.610839:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513099.610841:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:16.0:1616513099.610842:0:245737:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:16.0:1616513099.610844:0:245737:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:28.0:1616513099.610846:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:16.0:1616513099.610846:0:245737:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.610847:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:28.0:1616513099.610850:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a5d8 00000800:00000200:24.0:1616513099.610850:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:16.0:1616513099.610851:0:245737:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513099.610852:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:28.0:1616513099.610854:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:7.2:1616513099.610855:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.610857:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:24.0:1616513099.610857:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000100:00000200:16.0:1616513099.610857:0:245737:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:28.0:1616513099.610864:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:10.0:1616513099.610864:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513099.610864:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000100:00000200:24.0:1616513099.610866:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513099.610868:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:10.0:1616513099.610869:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:8.0:1616513099.610870:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513099.610871:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:28.0:1616513099.610872:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.52@tcp1 : PUT - for me 00000800:00000200:18.0:1616513099.610872:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513099.610873:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513099.610874:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:10.0:1616513099.610877:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:8.0:1616513099.610878:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3980 00000400:00000200:28.0:1616513099.610880:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.52@tcp1 of length 224 into portal 12 MB=0x5ffdc65cab5c0 00000800:00000200:8.0:1616513099.610881:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513099.610883:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788a18 00000800:00000200:10.0:1616513099.610886:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:28.0:1616513099.610888:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.52@tcp1 of length 224/224 into md 0x865be291 [1] + 87208 00000400:00000200:28.0:1616513099.610893:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:28.0:1616513099.610896:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:28.0:1616513099.610902:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513099.610905:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513099.610906:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:20.0:1616513099.610922:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688696949224896 00010000:00000200:20.0:1616513099.610933:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d56affc4c80 x1688696949224896/t0(0) o400->c5756d1a-9abe-f41b-9e7d-5df0ede84854@10.31.129.52@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:20.0:1616513099.610946:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688696949224896, offset 192 00000400:00000200:20.0:1616513099.610952:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.52@tcp1 00000400:00000200:20.0:1616513099.610958:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.52@tcp1 routed destination 00000400:00000200:20.0:1616513099.610973:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.52@tcp1(10.31.129.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:20.0:1616513099.610980:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:20.0:1616513099.610985:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:20.0:1616513099.610988:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513099.610990:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513099.610994:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.610998:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.611007:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.611013:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.611015:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513099.611017:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.611022:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.611026:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841e79bcb0 00000800:00000200:26.0:1616513099.611030:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.622823:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513099.622860:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:18.0:1616513099.622878:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.622888:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513099.622891:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.622897:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.77@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.622908:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.77@tcp1 of length 224 into portal 26 MB=0x5ffddb1d7b9c0 00000800:00000200:20.0:1616513099.622915:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:18.0:1616513099.622915:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.77@tcp1 of length 224/224 into md 0xf97d1239 [1] + 0 00000400:00000200:18.0:1616513099.622922:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3a08 00000400:00000200:18.0:1616513099.622924:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:20.0:1616513099.622925:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000100:00000200:18.0:1616513099.622927:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.622932:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3a08 00000400:00000200:20.0:1616513099.622934:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.77@tcp1 : PUT - for me 00000800:00000200:18.0:1616513099.622935:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.622938:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.622940:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513099.622944:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.77@tcp1 of length 224 into portal 12 MB=0x5ffddb1d7ba40 00000400:00000200:20.0:1616513099.622951:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.77@tcp1 of length 224/224 into md 0x865be291 [1] + 87432 00000400:00000200:20.0:1616513099.622956:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.622959:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513099.622966:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.622969:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.622969:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:20.0:1616513099.622973:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513099.623005:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702520113600 00010000:00000200:24.0:1616513099.623016:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d9050 x1688702520113600/t0(0) o400->b574779a-a2fd-3c92-5ef1-e6c753a91dc6@10.31.136.77@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.623030:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688702520113600, offset 192 00000100:00000200:28.0:1616513099.623034:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688702520113728 00000400:00000200:24.0:1616513099.623036:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.77@tcp1 00000400:00000200:24.0:1616513099.623044:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.77@tcp1 routed destination 00010000:00000200:28.0:1616513099.623046:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c660480 x1688702520113728/t0(0) o400->30917b0d-49e4-1f11-9763-cd9baca72293@10.31.136.77@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513099.623060:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.77@tcp1(10.31.136.77@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513099.623061:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688702520113728, offset 192 00000800:00000200:24.0:1616513099.623066:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:28.0:1616513099.623069:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.77@tcp1 00000800:00000200:24.0:1616513099.623072:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.623075:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:28.0:1616513099.623077:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.77@tcp1 routed destination 00000800:00000200:24.0:1616513099.623077:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.623082:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.623086:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.623088:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513099.623092:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.77@tcp1(10.31.136.77@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:8.0:1616513099.623094:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.623098:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:28.0:1616513099.623099:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000400:00000200:8.0:1616513099.623100:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.623101:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:28.0:1616513099.623105:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000400:00000200:8.0:1616513099.623105:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:28.0:1616513099.623108:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513099.623108:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3d38 00000800:00000200:28.0:1616513099.623110:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.623111:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:28.0:1616513099.623115:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.623119:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513099.623133:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513099.623139:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513099.623142:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513099.623149:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513099.623153:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccd48 00000800:00000200:14.0:1616513099.623157:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513099.623160:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513099.626575:0:6483:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930000, portal 4 00000100:00000200:28.0:1616513099.626580:0:6483:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930000, offset 0 00000400:00000200:28.0:1616513099.626586:0:6483:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:28.0:1616513099.626592:0:6483:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:28.0:1616513099.626604:0:6483:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.626610:0:6483:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:28.0:1616513099.626615:0:6483:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.626618:0:6483:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:28.0:1616513099.626620:0:6483:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:28.0:1616513099.626624:0:6483:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513099.626665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513099.626721:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:0.0:1616513099.626728:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:0.0:1616513099.626731:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.626738:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:0.0:1616513099.626744:0:6464:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d85193ea880 x1693067725930000/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513099.626756:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccc38 00000800:00000200:0.0:1616513099.626760:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513099.626763:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513099.626834:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:0.0:1616513099.626844:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513099.626850:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:8.0:1616513099.626851:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:0.0:1616513099.626858:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:0.0:1616513099.626865:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627e10 00000400:00000200:0.0:1616513099.626871:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866dde81 [1] + 192 00000400:00000200:0.0:1616513099.626876:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.626880:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d85193ea880 x1693067725930000/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:0.0:1616513099.626894:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513099.626897:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513099.626898:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:28.0:1616513099.626901:0:6483:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d85193ea880 x1693067725930000/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:28.0:1616513099.626913:0:6483:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d85193ea880 x1693067725930000/t0(0) o13->boslfs02-OST0015-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:28.0:1616513099.626922:0:6483:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccb28 00000800:00000200:7.2:1616513099.636626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513099.636678:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.636687:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513099.636696:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.54@tcp1 : PUT - for me 00000400:00000200:10.0:1616513099.636705:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.54@tcp1 of length 224 into portal 26 MB=0x5ffd4fadb0f00 00000400:00000200:10.0:1616513099.636711:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.54@tcp1 of length 224/224 into md 0xf97d1249 [1] + 0 00000400:00000200:10.0:1616513099.636715:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2b28 00000400:00000200:10.0:1616513099.636718:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513099.636720:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513099.636724:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2b28 00000800:00000200:10.0:1616513099.636727:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.636730:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513099.636731:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.636734:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.636744:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.636752:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.54@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.636761:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.54@tcp1 of length 224 into portal 12 MB=0x5ffd4fadb11c0 00000400:00000200:18.0:1616513099.636767:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.54@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 55216 00000400:00000200:18.0:1616513099.636772:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.636774:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.636781:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.636783:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.636785:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513099.636786:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665090363136 00000800:00000200:18.0:1616513099.636788:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513099.636796:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a850 x1688665090363136/t0(0) o400->e26fda8b-1507-bd80-9be2-4e08a9c1997b@10.31.138.54@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.636809:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665090363136, offset 192 00000400:00000200:24.0:1616513099.636815:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.54@tcp1 00000400:00000200:24.0:1616513099.636822:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.54@tcp1 routed destination 00000400:00000200:24.0:1616513099.636836:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.54@tcp1(10.31.138.54@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.636842:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:22.0:1616513099.636845:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665090363840 00000800:00000200:24.0:1616513099.636849:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.636851:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.636853:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.636858:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:22.0:1616513099.636858:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bf980 x1688665090363840/t0(0) o400->9e15cef9-5b14-e347-ab31-735c678409eb@10.31.138.54@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513099.636864:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.636864:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:22.0:1616513099.636873:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665090363840, offset 192 00000400:00000200:22.0:1616513099.636881:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.54@tcp1 00000400:00000200:22.0:1616513099.636889:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.54@tcp1 routed destination 00000400:00000200:22.0:1616513099.636904:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.54@tcp1(10.31.138.54@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:22.0:1616513099.636910:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:22.0:1616513099.636916:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:0.0:1616513099.636917:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513099.636918:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.636920:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513099.636924:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:22.0:1616513099.636925:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000400:00000200:0.0:1616513099.636927:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513099.636929:0:51552:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513099.636933:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.636939:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2440 00000800:00000200:0.0:1616513099.636944:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.636947:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513099.636993:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.637000:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.637003:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513099.637004:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.637010:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.637014:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82baa4a2a8 00000800:00000200:26.0:1616513099.637018:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.642978:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513099.642978:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.642988:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.642995:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.643003:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.187@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.643012:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.187@tcp1 of length 224 into portal 26 MB=0x5ffd569e04340 00000400:00000200:8.0:1616513099.643018:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.187@tcp1 of length 224/224 into md 0xf97d1269 [1] + 0 00000400:00000200:8.0:1616513099.643023:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3298 00000400:00000200:8.0:1616513099.643025:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.643027:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:18.0:1616513099.643030:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513099.643031:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3298 00000800:00000200:8.0:1616513099.643034:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.643036:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.643038:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.643040:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513099.643041:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.643050:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.187@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.643059:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.187@tcp1 of length 224 into portal 12 MB=0x5ffd569e043c0 00000400:00000200:18.0:1616513099.643065:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.187@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 55440 00000400:00000200:18.0:1616513099.643070:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.643073:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.643079:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.643082:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.643084:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.643087:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513099.643092:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666952975168 00010000:00000200:24.0:1616513099.643102:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe668850 x1688666952975168/t0(0) o400->2766fb6a-34f2-bf76-8ff3-9e8bbaeed2ee@10.31.132.187@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.643115:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688666952975168, offset 192 00000400:00000200:24.0:1616513099.643121:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.187@tcp1 00000400:00000200:24.0:1616513099.643128:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.187@tcp1 routed destination 00000400:00000200:24.0:1616513099.643142:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.187@tcp1(10.31.132.187@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.643148:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.643153:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.643156:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.643158:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:26.0:1616513099.643162:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688666952975296 00000800:00000200:24.0:1616513099.643162:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.643166:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.643168:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:26.0:1616513099.643174:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b8900 x1688666952975296/t0(0) o400->77077ba3-7b17-2abc-32a2-7661f82a38e4@10.31.132.187@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:0.0:1616513099.643177:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.643183:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.643186:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.643188:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688666952975296, offset 192 00000400:00000200:0.0:1616513099.643192:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.643194:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.187@tcp1 00000400:00000200:0.0:1616513099.643197:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2880 00000800:00000200:0.0:1616513099.643200:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:26.0:1616513099.643202:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.187@tcp1 routed destination 00000800:00000200:0.0:1616513099.643203:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513099.643217:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.187@tcp1(10.31.132.187@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513099.643223:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513099.643229:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513099.643231:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.643233:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.643238:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.643242:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.643252:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.643258:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513099.643260:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513099.643263:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513099.643267:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513099.643270:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697afa90 00000800:00000200:10.0:1616513099.643274:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.654214:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.654223:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.654230:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.654238:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.11@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.654245:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.11@tcp1 of length 224 into portal 26 MB=0x5ffd3ffcc1d00 00000400:00000200:8.0:1616513099.654250:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.11@tcp1 of length 224/224 into md 0xf97d1289 [1] + 0 00000400:00000200:8.0:1616513099.654254:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3870 00000400:00000200:8.0:1616513099.654257:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.654259:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.654263:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3870 00000800:00000200:8.0:1616513099.654265:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.654267:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.654269:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.654271:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513099.654325:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660878302464 00010000:00000200:24.0:1616513099.654335:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73050 x1688660878302464/t0(0) o400->eeda493e-593f-9529-a59e-92b1f2918b12@10.31.133.11@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.654348:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688660878302464, offset 192 00000400:00000200:24.0:1616513099.654354:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.11@tcp1 00000400:00000200:24.0:1616513099.654361:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.11@tcp1 routed destination 00000800:00000200:7.2:1616513099.654366:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:24.0:1616513099.654375:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.11@tcp1(10.31.133.11@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.654381:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.654386:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.654389:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.654391:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.654395:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.654399:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513099.654407:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513099.654410:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.654417:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.654419:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.654420:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513099.654425:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:18.0:1616513099.654429:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513099.654431:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f33a8 00000800:00000200:0.0:1616513099.654444:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.654447:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513099.654453:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.11@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.654462:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.11@tcp1 of length 224 into portal 12 MB=0x5ffd3ffcc1d80 00000400:00000200:18.0:1616513099.654470:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.11@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 55664 00000400:00000200:18.0:1616513099.654475:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.654478:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513099.654486:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.654486:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.654489:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.654491:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513099.654553:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688660878302592 00010000:00000200:26.0:1616513099.654565:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645ba400 x1688660878302592/t0(0) o400->90164de7-b050-eb54-47b6-c229745e83e4@10.31.133.11@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513099.654594:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688660878302592, offset 192 00000400:00000200:26.0:1616513099.654600:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.11@tcp1 00000400:00000200:26.0:1616513099.654607:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.11@tcp1 routed destination 00000400:00000200:26.0:1616513099.654622:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.11@tcp1(10.31.133.11@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513099.654629:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513099.654635:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513099.654637:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.654639:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.654644:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.654648:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.654662:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.654666:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.654668:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513099.654672:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.654674:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.654677:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af650 00000800:00000200:26.0:1616513099.654680:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.656869:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.656876:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.656881:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.656887:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.34@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.656893:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.34@tcp1 of length 224 into portal 26 MB=0x5ffdd1432f1c0 00000400:00000200:8.0:1616513099.656898:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.143.34@tcp1 of length 224/224 into md 0xf97d12a9 [1] + 0 00000400:00000200:8.0:1616513099.656902:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2dd0 00000400:00000200:8.0:1616513099.656903:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.656905:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.656909:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2dd0 00000800:00000200:8.0:1616513099.656910:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.656912:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.656913:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.656915:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.656921:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513099.656926:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513099.656927:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.143.34@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.656931:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.143.34@tcp1 of length 224 into portal 12 MB=0x5ffdd1432f240 00000400:00000200:8.0:1616513099.656936:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.143.34@tcp1 of length 224/224 into md 0x865be291 [1] + 87656 00000400:00000200:8.0:1616513099.656939:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.656941:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513099.656945:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.656947:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.656948:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513099.656967:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699875291584 00010000:00000200:24.0:1616513099.656977:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250850 x1688699875291584/t0(0) o400->88fe00e5-2f2b-734c-b786-00df6186c441@10.31.143.34@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.656989:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699875291584, offset 192 00000400:00000200:24.0:1616513099.656995:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.34@tcp1 00000400:00000200:24.0:1616513099.657001:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.34@tcp1 routed destination 00000100:00000200:28.0:1616513099.657011:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699875291712 00000400:00000200:24.0:1616513099.657014:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.34@tcp1(10.31.143.34@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.657019:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:28.0:1616513099.657022:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8f980 x1688699875291712/t0(0) o400->e3930ce4-289a-3f02-cf1b-c941c536ab4b@10.31.143.34@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513099.657024:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.657027:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.657029:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.657033:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513099.657036:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699875291712, offset 192 00000800:00000200:11.2:1616513099.657037:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.657039:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513099.657041:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.143.34@tcp1 00000400:00000200:28.0:1616513099.657048:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.143.34@tcp1 routed destination 00000400:00000200:28.0:1616513099.657062:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.143.34@tcp1(10.31.143.34@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.657074:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.657079:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.657082:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.657084:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.657088:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513099.657088:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513099.657092:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513099.657095:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.657098:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.657099:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.657106:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.657111:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3870 00000800:00000200:20.0:1616513099.657115:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513099.657145:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513099.657152:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513099.657155:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513099.657161:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513099.657165:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccb28 00000800:00000200:22.0:1616513099.657169:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513099.657172:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513099.658476:0:6503:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930016, portal 4 00000100:00000200:10.0:1616513099.658482:0:6503:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930016, offset 0 00000400:00000200:10.0:1616513099.658488:0:6503:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:10.0:1616513099.658493:0:6503:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:10.0:1616513099.658504:0:6503:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513099.658510:0:6503:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:10.0:1616513099.658516:0:6503:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513099.658519:0:6503:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:10.0:1616513099.658521:0:6503:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:10.0:1616513099.658526:0:6503:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513099.658574:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513099.658594:0:6466:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513099.658601:0:6466:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513099.658604:0:6466:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:5.0:1616513099.658605:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513099.658611:0:6466:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513099.658620:0:6466:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9642400 x1693067725930016/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513099.658633:0:6466:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82ff0 00000800:00000200:1.0:1616513099.658638:0:6466:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513099.658746:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513099.658757:0:6465:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513099.658763:0:6465:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513099.658764:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513099.658772:0:6465:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513099.658781:0:6465:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627e20 00000400:00000200:1.0:1616513099.658788:0:6465:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700365 [1] + 192 00000400:00000200:1.0:1616513099.658794:0:6465:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513099.658800:0:6465:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9642400 x1693067725930016/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513099.658815:0:6465:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513099.658818:0:6465:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513099.658820:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:10.0:1616513099.658821:0:6503:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9642400 x1693067725930016/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513099.658832:0:6503:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9642400 x1693067725930016/t0(0) o13->boslfs02-OST000b-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513099.658841:0:6503:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83ba0 00000800:00000200:7.2:1616513099.664066:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513099.664077:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.664085:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513099.664094:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.66@tcp1 : PUT - for me 00000400:00000200:10.0:1616513099.664103:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.66@tcp1 of length 224 into portal 26 MB=0x5ffdd14fed2c0 00000400:00000200:10.0:1616513099.664110:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.66@tcp1 of length 224/224 into md 0xf97d12d9 [1] + 0 00000400:00000200:10.0:1616513099.664114:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2990 00000400:00000200:10.0:1616513099.664117:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513099.664119:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513099.664123:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2990 00000800:00000200:10.0:1616513099.664126:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.664129:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513099.664129:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513099.664133:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513099.664188:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699888652992 00010000:00000200:24.0:1616513099.664198:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66d850 x1688699888652992/t0(0) o400->6d88d4cd-b414-c95b-bf39-555316e73dd1@10.31.129.66@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.664221:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699888652992, offset 192 00000400:00000200:24.0:1616513099.664227:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.66@tcp1 00000400:00000200:24.0:1616513099.664233:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.66@tcp1 routed destination 00000400:00000200:24.0:1616513099.664246:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.66@tcp1(10.31.129.66@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.664251:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.664256:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.664258:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.664260:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.664264:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.664268:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.664270:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513099.664277:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.664282:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513099.664284:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.664289:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513099.664294:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2770 00000800:00000200:8.0:1616513099.664297:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.664299:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.664553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.664608:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.664617:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513099.664626:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.66@tcp1 : PUT - for me 00000400:00000200:22.0:1616513099.664634:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.66@tcp1 of length 224 into portal 12 MB=0x5ffdd14fed680 00000400:00000200:22.0:1616513099.664641:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.66@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 55888 00000400:00000200:22.0:1616513099.664646:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513099.664648:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513099.664655:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513099.664658:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.664659:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513099.664664:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513099.664675:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699888653952 00010000:00000200:26.0:1616513099.664686:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2e780 x1688699888653952/t0(0) o400->2721c4a9-bd89-7cf5-260c-19b1bc035eda@10.31.129.66@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513099.664699:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699888653952, offset 192 00000400:00000200:26.0:1616513099.664705:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.66@tcp1 00000400:00000200:26.0:1616513099.664712:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.66@tcp1 routed destination 00000400:00000200:26.0:1616513099.664725:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.66@tcp1(10.31.129.66@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513099.664731:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513099.664736:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513099.664739:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.664741:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.664745:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.664749:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.664765:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.664770:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.664772:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513099.664774:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.664777:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.664780:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af650 00000800:00000200:26.0:1616513099.664783:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.667026:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.667078:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.667087:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.667096:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.165@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.667105:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.165@tcp1 of length 224 into portal 12 MB=0x6018272848140 00000400:00000200:0.0:1616513099.667111:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.165@tcp1 of length 224/224 into md 0x865be291 [1] + 87880 00000400:00000200:0.0:1616513099.667116:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.667119:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.667125:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.667128:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.667130:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.667132:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.667143:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.667143:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.667152:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.165@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.667160:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.165@tcp1 of length 224 into portal 26 MB=0x60182728480c0 00000400:00000200:20.0:1616513099.667167:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.165@tcp1 of length 224/224 into md 0xf97d12f9 [1] + 0 00000400:00000200:20.0:1616513099.667172:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2440 00000400:00000200:20.0:1616513099.667174:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.667177:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513099.667181:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2440 00000800:00000200:20.0:1616513099.667184:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.667187:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513099.667189:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509638926656 00000800:00000200:20.0:1616513099.667189:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:28.0:1616513099.667202:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d569bb08d80 x1690509638926656/t0(0) o400->9791340c-c8d8-055a-a502-ea08adf28da5@10.31.138.165@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.667216:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690509638926656, offset 192 00000400:00000200:28.0:1616513099.667222:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.165@tcp1 00000400:00000200:28.0:1616513099.667229:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.165@tcp1 routed destination 00000100:00000200:24.0:1616513099.667240:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690509638926528 00000400:00000200:28.0:1616513099.667244:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.165@tcp1(10.31.138.165@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:24.0:1616513099.667249:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe669850 x1690509638926528/t0(0) o400->4c47c298-2a83-622f-a5e9-000de2df312a@10.31.138.165@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:28.0:1616513099.667250:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513099.667256:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.667258:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513099.667261:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.667261:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690509638926528, offset 192 00000800:00000200:28.0:1616513099.667265:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.667268:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.165@tcp1 00000800:00000200:11.2:1616513099.667269:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513099.667274:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.165@tcp1 routed destination 00000400:00000200:24.0:1616513099.667287:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.165@tcp1(10.31.138.165@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.667293:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.667298:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.667300:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513099.667302:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.667306:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.667310:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513099.667312:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513099.667370:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:18.0:1616513099.667375:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513099.667377:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.667380:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.667381:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513099.667381:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:18.0:1616513099.667383:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.667387:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.667390:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.667391:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd078 00000800:00000200:4.0:1616513099.667395:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513099.667396:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3f58 00000800:00000200:18.0:1616513099.667399:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.667402:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.674556:0:6496:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930032, portal 4 00000100:00000200:30.0:1616513099.674562:0:6496:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930032, offset 0 00000400:00000200:30.0:1616513099.674568:0:6496:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:30.0:1616513099.674574:0:6496:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:30.0:1616513099.674586:0:6496:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.674593:0:6496:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:30.0:1616513099.674598:0:6496:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.674601:0:6496:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:30.0:1616513099.674604:0:6496:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:30.0:1616513099.674609:0:6496:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513099.674655:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.674676:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513099.674682:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513099.674685:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513099.674691:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513099.674698:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74c8a2f980 x1693067725930032/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513099.674709:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef8f8 00000800:00000200:27.0:1616513099.674713:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513099.674715:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:11.2:1616513099.674814:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:27.2:1616513099.674821:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:8.0:1616513099.674823:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:27.0:1616513099.674829:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:8.0:1616513099.674829:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:27.0:1616513099.674835:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000400:00000200:8.0:1616513099.674836:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.67@tcp1 : PUT - for me 00000800:00000200:3.0:1616513099.674839:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513099.674843:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513099.674844:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.67@tcp1 of length 224 into portal 26 MB=0x5ffd47cc32000 00000400:00000200:8.0:1616513099.674849:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.67@tcp1 of length 224/224 into md 0xf97d1309 [1] + 0 00000400:00000200:27.0:1616513099.674851:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627e30 00000400:00000200:8.0:1616513099.674853:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2cc0 00000400:00000200:8.0:1616513099.674855:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:27.0:1616513099.674857:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x3716778d [1] + 192 00000100:00000200:8.0:1616513099.674858:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.674861:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2cc0 00000400:00000200:27.0:1616513099.674862:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:8.0:1616513099.674864:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:27.0:1616513099.674866:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74c8a2f980 x1693067725930032/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513099.674866:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.674867:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.674869:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:27.0:1616513099.674880:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513099.674882:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513099.674884:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:30.0:1616513099.674887:0:6496:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74c8a2f980 x1693067725930032/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513099.674899:0:6496:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74c8a2f980 x1693067725930032/t0(0) o13->boslfs02-OST000a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513099.674908:0:6496:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee088 00000100:00000200:24.0:1616513099.674921:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662974865408 00010000:00000200:24.0:1616513099.674930:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7df850 x1688662974865408/t0(0) o400->5a3e4405-aebe-5ba6-959c-66676941bb7e@10.31.138.67@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.674942:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662974865408, offset 192 00000400:00000200:24.0:1616513099.674947:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.67@tcp1 00000400:00000200:24.0:1616513099.674955:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.67@tcp1 routed destination 00000400:00000200:24.0:1616513099.674967:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.67@tcp1(10.31.138.67@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.674973:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.674978:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.674980:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.674982:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.674986:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.674990:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.674992:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513099.675016:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513099.675026:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.675033:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513099.675036:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513099.675040:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.67@tcp1 : PUT - for me 00000800:00000200:0.0:1616513099.675042:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:10.0:1616513099.675048:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.67@tcp1 of length 224 into portal 12 MB=0x5ffd47cc32080 00000800:00000200:0.0:1616513099.675048:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.675051:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513099.675053:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.67@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 56112 00000400:00000200:0.0:1616513099.675057:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:10.0:1616513099.675058:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513099.675060:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:0.0:1616513099.675062:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3e48 00000800:00000200:10.0:1616513099.675065:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513099.675067:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513099.675068:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513099.675069:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513099.675070:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:26.0:1616513099.675084:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662974865536 00010000:00000200:26.0:1616513099.675095:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdcc80 x1688662974865536/t0(0) o400->850e322d-46c8-756c-1c3c-af0c6e0f9e3c@10.31.138.67@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513099.675108:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662974865536, offset 192 00000400:00000200:26.0:1616513099.675114:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.67@tcp1 00000400:00000200:26.0:1616513099.675121:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.67@tcp1 routed destination 00000400:00000200:26.0:1616513099.675134:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.67@tcp1(10.31.138.67@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513099.675140:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513099.675145:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513099.675148:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.675150:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.675154:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.675159:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.675170:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.675177:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.675179:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513099.675182:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.675185:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.675189:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af650 00000800:00000200:18.0:1616513099.675192:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:21.0:1616513099.690695:0:6495:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930048, portal 4 00000100:00000200:21.0:1616513099.690701:0:6495:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930048, offset 0 00000400:00000200:21.0:1616513099.690708:0:6495:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:21.0:1616513099.690714:0:6495:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:21.0:1616513099.690728:0:6495:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:21.0:1616513099.690735:0:6495:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:21.0:1616513099.690742:0:6495:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:21.0:1616513099.690745:0:6495:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:21.0:1616513099.690748:0:6495:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:21.0:1616513099.690753:0:6495:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513099.690800:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513099.690815:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513099.690822:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513099.690825:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.690832:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513099.690839:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5de1390d80 x1693067725930048/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513099.690851:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf504c8 00000800:00000200:8.0:1616513099.690855:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513099.690858:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513099.690989:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:0.0:1616513099.691041:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513099.691050:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:0.0:1616513099.691059:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:0.0:1616513099.691066:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627e40 00000400:00000200:0.0:1616513099.691073:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866ddf31 [1] + 192 00000400:00000200:0.0:1616513099.691078:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.691083:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5de1390d80 x1693067725930048/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:4.0:1616513099.691095:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513099.691097:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:0.0:1616513099.691100:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513099.691102:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:21.0:1616513099.691107:0:6495:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5de1390d80 x1693067725930048/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:21.0:1616513099.691122:0:6495:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5de1390d80 x1693067725930048/t0(0) o13->boslfs02-OST0039-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:21.0:1616513099.691132:0:6495:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d840bf50c38 00000800:00000200:7.2:1616513099.694960:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.695015:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.695025:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:10.0:1616513099.695026:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.695034:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.52@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.695045:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.52@tcp1 of length 224 into portal 26 MB=0x604b7d843fa00 00000400:00000200:26.0:1616513099.695051:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.52@tcp1 of length 224/224 into md 0xf97d1319 [1] + 0 00000400:00000200:26.0:1616513099.695056:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf23b8 00000400:00000200:26.0:1616513099.695059:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.695062:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513099.695066:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf23b8 00000800:00000200:26.0:1616513099.695069:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.695072:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.695073:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513099.695126:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694037514123776 00010000:00000200:24.0:1616513099.695135:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a3050 x1694037514123776/t0(0) o400->f4c7647c-0cb1-ccea-8cc6-a7f9711dca14@10.31.140.52@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513099.695147:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513099.695148:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1694037514123776, offset 192 00000400:00000200:24.0:1616513099.695154:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.52@tcp1 00000400:00000200:24.0:1616513099.695161:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.52@tcp1 routed destination 00000400:00000200:24.0:1616513099.695174:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.52@tcp1(10.31.140.52@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.695180:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.695186:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.695188:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.695190:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.695195:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513099.695200:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:22.0:1616513099.695210:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.695220:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513099.695229:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.52@tcp1 : PUT - for me 00000400:00000200:22.0:1616513099.695237:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.52@tcp1 of length 224 into portal 12 MB=0x604b7d843fcc0 00000400:00000200:22.0:1616513099.695244:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.52@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 56336 00000800:00000200:11.2:1616513099.695245:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513099.695251:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513099.695254:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513099.695260:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513099.695264:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513099.695266:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513099.695267:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:22.0:1616513099.695269:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:8.0:1616513099.695272:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513099.695275:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.695281:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000100:00000200:26.0:1616513099.695286:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1694037514124480 00000400:00000200:8.0:1616513099.695286:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3540 00000800:00000200:8.0:1616513099.695289:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.695291:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:26.0:1616513099.695297:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2cc80 x1694037514124480/t0(0) o400->a777836d-f578-2230-6313-d94c2c9af772@10.31.140.52@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:26.0:1616513099.695312:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1694037514124480, offset 192 00000400:00000200:26.0:1616513099.695318:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.52@tcp1 00000400:00000200:26.0:1616513099.695324:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.52@tcp1 routed destination 00000400:00000200:26.0:1616513099.695339:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.52@tcp1(10.31.140.52@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513099.695345:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:26.0:1616513099.695351:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513099.695353:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.695355:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.695359:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.695363:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.695369:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.695374:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513099.695376:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513099.695379:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:10.0:1616513099.695382:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513099.695385:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af100 00000800:00000200:10.0:1616513099.695388:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.726173:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.726229:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.726239:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.726249:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.198@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.726258:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.198@tcp1 of length 224 into portal 26 MB=0x5ffdbd93afd00 00000400:00000200:18.0:1616513099.726265:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.198@tcp1 of length 224/224 into md 0xf97d1329 [1] + 0 00000400:00000200:18.0:1616513099.726269:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2a18 00000400:00000200:18.0:1616513099.726273:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.726276:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.726281:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2a18 00000800:00000200:22.0:1616513099.726284:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.726285:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.726287:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.726291:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.726294:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513099.726295:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513099.726304:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.198@tcp1 : PUT - for me 00000400:00000200:22.0:1616513099.726313:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.198@tcp1 of length 224 into portal 12 MB=0x5ffdbd93afe40 00000400:00000200:22.0:1616513099.726319:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.198@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 56560 00000400:00000200:22.0:1616513099.726325:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513099.726328:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513099.726337:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.726340:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513099.726342:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513099.726349:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694590995712 00000100:00000200:30.0:1616513099.726359:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688694590996032 00010000:00000200:24.0:1616513099.726361:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4050 x1688694590995712/t0(0) o400->ffde25b8-3db5-d2c7-ad08-5c63a0da17b3@10.31.141.198@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513099.726371:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2f980 x1688694590996032/t0(0) o400->0c9b7013-7241-1c88-8910-c227283dcc66@10.31.141.198@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513099.726376:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688694590995712, offset 192 00000400:00000200:24.0:1616513099.726382:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.198@tcp1 00000400:00000200:24.0:1616513099.726389:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.198@tcp1 routed destination 00000100:00000200:30.0:1616513099.726396:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688694590996032, offset 192 00000400:00000200:30.0:1616513099.726402:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.198@tcp1 00000400:00000200:24.0:1616513099.726404:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.198@tcp1(10.31.141.198@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513099.726409:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.198@tcp1 routed destination 00000800:00000200:24.0:1616513099.726411:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.726417:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.726420:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.726422:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513099.726423:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.198@tcp1(10.31.141.198@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.726427:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513099.726429:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513099.726434:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513099.726435:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.726438:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513099.726444:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513099.726448:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513099.726448:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513099.726453:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513099.726499:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.726505:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.726508:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513099.726509:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513099.726514:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:10.0:1616513099.726515:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513099.726518:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.726520:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513099.726520:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2000 00000800:00000200:4.0:1616513099.726523:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513099.726524:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:4.0:1616513099.726525:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513099.726527:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee088 00000800:00000200:10.0:1616513099.726530:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.726856:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:7.2:1616513099.726861:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:8.0:1616513099.726867:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:22.0:1616513099.726870:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.726874:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513099.726876:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:26.0:1616513099.726880:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:8.0:1616513099.726883:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.92@tcp1 : PUT - for me 00000400:00000200:22.0:1616513099.726884:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.92@tcp1 : PUT - for me 00000400:00000200:22.0:1616513099.726891:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.92@tcp1 of length 224 into portal 12 MB=0x5ffd615060e40 00000400:00000200:8.0:1616513099.726892:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.92@tcp1 of length 224 into portal 26 MB=0x5ffd615060c80 00000400:00000200:22.0:1616513099.726897:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.92@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 56784 00000400:00000200:22.0:1616513099.726900:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.726903:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.92@tcp1 of length 224/224 into md 0xf97d1349 [1] + 0 00000400:00000200:8.0:1616513099.726908:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf3ed0 00000100:00000200:22.0:1616513099.726910:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:8.0:1616513099.726910:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:22.0:1616513099.726914:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513099.726915:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000800:00000200:22.0:1616513099.726916:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513099.726918:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:8.0:1616513099.726919:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf3ed0 00000800:00000200:20.0:1616513099.726920:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.726922:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:30.0:1616513099.726924:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688669824355904 00000800:00000200:8.0:1616513099.726924:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.726926:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00010000:00000200:30.0:1616513099.726930:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2b600 x1688669824355904/t0(0) o400->c4f00af6-fd4c-ae33-f853-50441749c967@10.31.136.92@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513099.726932:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688669824355456 00010000:00000200:24.0:1616513099.726938:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66a050 x1688669824355456/t0(0) o400->43cde11d-2567-1968-493b-8b39570d8e34@10.31.136.92@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:30.0:1616513099.726941:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688669824355904, offset 192 00000400:00000200:30.0:1616513099.726946:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.92@tcp1 00000100:00000200:24.0:1616513099.726949:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688669824355456, offset 192 00000400:00000200:30.0:1616513099.726952:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.92@tcp1 routed destination 00000400:00000200:24.0:1616513099.726955:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.92@tcp1 00000400:00000200:30.0:1616513099.726964:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.92@tcp1(10.31.136.92@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.726970:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513099.726972:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.92@tcp1 routed destination 00000800:00000200:30.0:1616513099.726974:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.726976:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513099.726978:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513099.726981:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.726985:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.92@tcp1(10.31.136.92@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513099.726985:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.726991:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:4.0:1616513099.726995:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513099.726996:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.726998:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:4.0:1616513099.726998:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513099.727000:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513099.727000:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:24.0:1616513099.727004:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513099.727004:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513099.727005:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:7.2:1616513099.727008:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:4.0:1616513099.727008:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee440 00000100:00000200:24.0:1616513099.727010:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513099.727010:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513099.727016:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.727020:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.727022:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513099.727024:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.727028:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.727032:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2c38 00000800:00000200:18.0:1616513099.727035:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.737005:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.737059:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.737069:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513099.737078:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.27.195@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.737087:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.27.195@tcp1 of length 224 into portal 26 MB=0x5f90a481ca080 00000400:00000200:26.0:1616513099.737093:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.27.195@tcp1 of length 224/224 into md 0xf97d1359 [1] + 0 00000400:00000200:26.0:1616513099.737098:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d8dcbaf2aa0 00000400:00000200:26.0:1616513099.737101:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.737104:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513099.737108:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dcbaf2aa0 00000800:00000200:26.0:1616513099.737111:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.737114:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513099.737114:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513099.737121:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.737124:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513099.737124:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513099.737133:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.27.195@tcp1 : PUT - for me 00000400:00000200:22.0:1616513099.737142:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.27.195@tcp1 of length 224 into portal 12 MB=0x5f90a481ca100 00000400:00000200:22.0:1616513099.737148:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.27.195@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 57008 00000400:00000200:22.0:1616513099.737158:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513099.737160:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513099.737167:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.737170:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513099.737172:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681197438378112 00000800:00000200:22.0:1616513099.737172:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.737181:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681197438378240 00010000:00000200:24.0:1616513099.737183:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7de850 x1681197438378112/t0(0) o400->044a89da-07c8-7a0f-0a9b-ebb778721829@10.31.27.195@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513099.737190:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74c8a2a880 x1681197438378240/t0(0) o400->71511a59-6905-19e8-2e8f-642cadd69125@10.31.27.195@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513099.737198:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681197438378112, offset 192 00000100:00000200:30.0:1616513099.737202:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681197438378240, offset 192 00000400:00000200:24.0:1616513099.737204:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.27.195@tcp1 00000400:00000200:30.0:1616513099.737207:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.27.195@tcp1 00000400:00000200:24.0:1616513099.737211:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.27.195@tcp1 routed destination 00000400:00000200:30.0:1616513099.737213:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.27.195@tcp1 routed destination 00000400:00000200:30.0:1616513099.737227:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.27.195@tcp1(10.31.27.195@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513099.737227:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.27.195@tcp1(10.31.27.195@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.737233:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.737233:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513099.737238:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.737239:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.737240:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513099.737242:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.737242:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.737244:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513099.737246:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513099.737248:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513099.737250:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513099.737255:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.737255:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513099.737261:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.737268:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.737271:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513099.737272:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.737277:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.737282:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee000 00000800:00000200:18.0:1616513099.737285:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:20.0:1616513099.737312:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.737319:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.737322:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513099.737334:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.737335:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.737340:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f22a8 00000800:00000200:20.0:1616513099.737344:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.740258:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:22.0:1616513099.740310:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.740319:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513099.740327:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.30@tcp1 : PUT - for me 00000400:00000200:22.0:1616513099.740335:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.30@tcp1 of length 224 into portal 26 MB=0x5f9222f4ea300 00000400:00000200:22.0:1616513099.740342:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.30@tcp1 of length 224/224 into md 0xf97d1389 [1] + 0 00000400:00000200:22.0:1616513099.740346:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02beb28 00000400:00000200:22.0:1616513099.740349:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513099.740351:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513099.740355:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02beb28 00000800:00000200:22.0:1616513099.740358:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513099.740360:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513099.740362:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513099.740364:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.740375:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513099.740384:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.30@tcp1 : PUT - for me 00000400:00000200:10.0:1616513099.740392:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.30@tcp1 of length 224 into portal 12 MB=0x5f9222f4ea380 00000400:00000200:10.0:1616513099.740398:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.30@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 57232 00000400:00000200:10.0:1616513099.740404:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513099.740406:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513099.740413:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000100:00000200:24.0:1616513099.740416:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681300101440256 00000800:00000200:10.0:1616513099.740416:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513099.740418:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000100:00000200:30.0:1616513099.740421:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1681300101440384 00000800:00000200:26.0:1616513099.740421:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00010000:00000200:24.0:1616513099.740426:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fa850 x1681300101440256/t0(0) o400->79c66309-7d6f-c68b-bbbe-751309566cea@10.31.132.30@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513099.740427:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdf080 x1681300101440384/t0(0) o400->bfa24568-4a53-e2d6-4629-380f944cc8ce@10.31.132.30@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513099.740441:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1681300101440384, offset 192 00000400:00000200:30.0:1616513099.740445:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.30@tcp1 00000100:00000200:24.0:1616513099.740450:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1681300101440256, offset 192 00000400:00000200:30.0:1616513099.740451:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.30@tcp1 routed destination 00000400:00000200:24.0:1616513099.740456:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.30@tcp1 00000400:00000200:30.0:1616513099.740462:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.30@tcp1(10.31.132.30@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.740467:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513099.740471:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.740474:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:24.0:1616513099.740474:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.30@tcp1 routed destination 00000800:00000200:30.0:1616513099.740476:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513099.740478:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.740485:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513099.740488:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.30@tcp1(10.31.132.30@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.740494:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.740500:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.740502:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513099.740504:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.740509:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.740513:0:51563:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513099.740515:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513099.740523:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.740530:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.740532:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:22.0:1616513099.740534:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.740539:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513099.740541:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:18.0:1616513099.740544:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3100 00000800:00000200:18.0:1616513099.740547:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513099.740548:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.740550:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.740557:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.740561:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef430 00000800:00000200:0.0:1616513099.740565:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.740568:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.750777:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.750790:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.750798:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.750807:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.25@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.750817:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.25@tcp1 of length 224 into portal 26 MB=0x5ffd45c976b80 00000400:00000200:8.0:1616513099.750823:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.25@tcp1 of length 224/224 into md 0xf97d1399 [1] + 0 00000400:00000200:8.0:1616513099.750828:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02becc0 00000400:00000200:8.0:1616513099.750831:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.750834:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.750838:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02becc0 00000800:00000200:8.0:1616513099.750841:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.750843:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.750844:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.750857:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513099.750912:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662435130240 00010000:00000200:24.0:1616513099.750922:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba050 x1688662435130240/t0(0) o400->3e51cc37-434d-2c1d-876d-880432dcaa23@10.31.133.25@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.750935:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688662435130240, offset 192 00000400:00000200:24.0:1616513099.750943:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.25@tcp1 00000400:00000200:24.0:1616513099.750950:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.25@tcp1 routed destination 00000400:00000200:24.0:1616513099.750964:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.25@tcp1(10.31.133.25@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.750969:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.750975:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.750977:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.750979:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.750984:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.750989:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.750990:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513099.751028:0:51564:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:0.0:1616513099.751044:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.751051:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.751053:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.751060:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.751065:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f25d8 00000800:00000200:0.0:1616513099.751069:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.751072:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513099.751082:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.751092:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513099.751101:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.25@tcp1 : PUT - for me 00000400:00000200:10.0:1616513099.751110:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.25@tcp1 of length 224 into portal 12 MB=0x5ffd45c976c80 00000400:00000200:10.0:1616513099.751118:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.25@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 57456 00000400:00000200:10.0:1616513099.751123:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513099.751126:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513099.751132:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.751135:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.751136:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513099.751140:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.751143:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688662435130496 00010000:00000200:30.0:1616513099.751150:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdc380 x1688662435130496/t0(0) o400->f59e4614-b4a7-8da4-c43c-42739f053e29@10.31.133.25@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513099.751161:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688662435130496, offset 192 00000400:00000200:30.0:1616513099.751165:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.25@tcp1 00000400:00000200:30.0:1616513099.751171:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.25@tcp1 routed destination 00000400:00000200:30.0:1616513099.751182:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.25@tcp1(10.31.133.25@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.751187:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513099.751192:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.751194:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513099.751195:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513099.751198:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.751203:0:51564:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.751255:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513099.751261:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513099.751264:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.751266:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513099.751270:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513099.751273:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeff0 00000800:00000200:22.0:1616513099.751277:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.757854:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:20.0:1616513099.757912:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.757921:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.757922:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.757930:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.184@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.757940:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.184@tcp1 of length 224 into portal 26 MB=0x60182aebee140 00000400:00000200:20.0:1616513099.757947:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.184@tcp1 of length 224/224 into md 0xf97d13a9 [1] + 0 00000400:00000200:20.0:1616513099.757952:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be220 00000400:00000200:20.0:1616513099.757955:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.757957:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513099.757962:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be220 00000800:00000200:20.0:1616513099.757965:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.757969:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.757971:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513099.758023:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510649385280 00010000:00000200:24.0:1616513099.758033:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cbe66f050 x1690510649385280/t0(0) o400->cbb44cff-3eb4-74c5-0422-7cc29f1b7bbe@10.31.131.184@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.758046:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510649385280, offset 192 00000400:00000200:24.0:1616513099.758051:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.184@tcp1 00000400:00000200:24.0:1616513099.758059:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.184@tcp1 routed destination 00000400:00000200:24.0:1616513099.758073:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.184@tcp1(10.31.131.184@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.758079:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.758084:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.758086:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.758088:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.758093:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.758098:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.758099:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:0.0:1616513099.758151:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.758158:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513099.758161:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513099.758167:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.758173:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3a90 00000800:00000200:0.0:1616513099.758176:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.758181:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513099.758616:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.758624:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.758630:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:20.0:1616513099.758633:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513099.758637:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.184@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.758644:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.184@tcp1 of length 224 into portal 12 MB=0x60182aebee2c0 00000400:00000200:8.0:1616513099.758650:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.184@tcp1 of length 224/224 into md 0x865be291 [1] + 88104 00000400:00000200:8.0:1616513099.758654:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.758656:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513099.758662:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.758664:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.758666:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.758727:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510649385664 00010000:00000200:28.0:1616513099.758739:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8d580 x1690510649385664/t0(0) o400->d3fbfc92-0d72-6355-df6e-f88231c36663@10.31.131.184@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.758752:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510649385664, offset 192 00000400:00000200:28.0:1616513099.758758:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.184@tcp1 00000400:00000200:28.0:1616513099.758765:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.184@tcp1 routed destination 00000400:00000200:28.0:1616513099.758779:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.184@tcp1(10.31.131.184@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.758785:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.758790:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.758793:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.758794:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.758799:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.758804:0:51565:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.758860:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.758866:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.758869:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.758876:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.758883:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcce58 00000800:00000200:26.0:1616513099.758887:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.758889:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.760614:0:51565:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.760669:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.760679:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.760688:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.5@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.760697:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.5@tcp1 of length 224 into portal 12 MB=0x5ffd480cf4a00 00000400:00000200:18.0:1616513099.760703:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.5@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 57680 00000400:00000200:18.0:1616513099.760709:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.760711:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.760718:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.760721:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.760723:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513099.760725:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.760734:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663042771456 00010000:00000200:30.0:1616513099.760744:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bba80 x1688663042771456/t0(0) o400->c84454f2-c675-3de5-556e-3a0677341032@10.31.138.5@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513099.760756:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663042771456, offset 192 00000400:00000200:30.0:1616513099.760762:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.5@tcp1 00000400:00000200:30.0:1616513099.760768:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.5@tcp1 routed destination 00000400:00000200:30.0:1616513099.760782:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.5@tcp1(10.31.138.5@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.760788:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:11.2:1616513099.760792:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513099.760793:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.760796:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513099.760797:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513099.760802:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.760848:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.760857:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.760865:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.5@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.760874:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.5@tcp1 of length 224 into portal 26 MB=0x5ffd480cf48c0 00000400:00000200:0.0:1616513099.760881:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.5@tcp1 of length 224/224 into md 0xf97d13b9 [1] + 0 00000400:00000200:0.0:1616513099.760886:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bfe48 00000400:00000200:0.0:1616513099.760889:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.760891:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.760896:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bfe48 00000800:00000200:0.0:1616513099.760899:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513099.760900:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:0.0:1616513099.760902:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513099.760908:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:4.0:1616513099.760909:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000800:00000200:0.0:1616513099.760911:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.760913:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.760920:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.760924:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee220 00000800:00000200:4.0:1616513099.760928:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513099.760957:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663042771136 00010000:00000200:24.0:1616513099.760967:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0050 x1688663042771136/t0(0) o400->5ace6d3f-debe-e5b4-afe7-e061e0f08e47@10.31.138.5@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.760980:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663042771136, offset 192 00000400:00000200:24.0:1616513099.760986:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.5@tcp1 00000400:00000200:24.0:1616513099.760993:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.5@tcp1 routed destination 00000400:00000200:24.0:1616513099.761006:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.5@tcp1(10.31.138.5@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.761012:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.761019:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.761022:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513099.761024:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.761028:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513099.761034:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513099.761036:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.761099:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.761105:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.761108:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513099.761113:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.761115:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.761120:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f27f8 00000800:00000200:26.0:1616513099.761124:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.764638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513099.764639:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.764651:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.764658:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.764666:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.69@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.764675:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.69@tcp1 of length 224 into portal 26 MB=0x6036602d2b380 00000400:00000200:8.0:1616513099.764681:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.69@tcp1 of length 224/224 into md 0xf97d13c9 [1] + 0 00000400:00000200:8.0:1616513099.764685:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bebb0 00000400:00000200:8.0:1616513099.764687:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.764690:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:8.0:1616513099.764690:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.764694:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bebb0 00000800:00000200:8.0:1616513099.764697:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.764699:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:18.0:1616513099.764700:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.764700:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.764703:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513099.764709:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.69@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.764717:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.69@tcp1 of length 224 into portal 12 MB=0x6036602d2b780 00000400:00000200:18.0:1616513099.764726:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.69@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 57904 00000400:00000200:18.0:1616513099.764731:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.764734:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.764740:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.764743:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.764745:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513099.764748:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.764755:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692586529175424 00000100:00000200:24.0:1616513099.764758:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692586529174400 00010000:00000200:30.0:1616513099.764764:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bb180 x1692586529175424/t0(0) o400->b4436b00-f27c-46d9-4184-16f04742f9b0@10.31.129.69@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00010000:00000200:24.0:1616513099.764769:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bd850 x1692586529174400/t0(0) o400->0ccbe3c9-8754-fc3a-470d-b202f669e270@10.31.129.69@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.764783:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692586529174400, offset 192 00000100:00000200:30.0:1616513099.764786:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692586529175424, offset 192 00000400:00000200:24.0:1616513099.764789:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.69@tcp1 00000400:00000200:30.0:1616513099.764790:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.69@tcp1 00000400:00000200:30.0:1616513099.764797:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.69@tcp1 routed destination 00000400:00000200:24.0:1616513099.764797:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.69@tcp1 routed destination 00000400:00000200:30.0:1616513099.764812:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.69@tcp1(10.31.129.69@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513099.764812:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.69@tcp1(10.31.129.69@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.764818:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.764819:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513099.764823:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.764825:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.764825:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.764827:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.764827:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513099.764830:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513099.764831:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513099.764834:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:11.2:1616513099.764836:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513099.764838:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513099.764841:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513099.764890:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.764897:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.764899:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.764906:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:10.0:1616513099.764909:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513099.764910:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee110 00000800:00000200:4.0:1616513099.764914:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513099.764916:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:4.0:1616513099.764917:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513099.764935:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513099.764935:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513099.764941:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513099.764946:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f23b8 00000800:00000200:10.0:1616513099.764950:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.770407:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513099.770417:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.770428:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.770434:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.770449:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.24@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.770457:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.24@tcp1 of length 224 into portal 12 MB=0x60182c15d8500 00000400:00000200:8.0:1616513099.770462:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.133.24@tcp1 of length 224/224 into md 0x865be291 [1] + 88328 00000800:00000200:18.0:1616513099.770463:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513099.770466:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.770469:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.770473:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.770474:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.770476:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.770478:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.770481:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:18.0:1616513099.770482:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.133.24@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.770491:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.133.24@tcp1 of length 224 into portal 26 MB=0x60182c15d83c0 00000400:00000200:18.0:1616513099.770497:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.133.24@tcp1 of length 224/224 into md 0xf97d13e9 [1] + 0 00000400:00000200:18.0:1616513099.770502:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf078 00000400:00000200:18.0:1616513099.770505:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.770507:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513099.770511:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf078 00000800:00000200:18.0:1616513099.770514:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513099.770517:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.770517:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.770521:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513099.770555:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510961771776 00010000:00000200:28.0:1616513099.770568:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8f080 x1690510961771776/t0(0) o400->cb7c3ab1-0eb9-f528-eff4-72a08299a157@10.31.133.24@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513099.770580:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510961771456 00000100:00000200:28.0:1616513099.770582:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510961771776, offset 192 00000400:00000200:28.0:1616513099.770588:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.24@tcp1 00010000:00000200:24.0:1616513099.770590:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a4850 x1690510961771456/t0(0) o400->e30bdbe9-0868-93ea-f39e-cabd2505ed09@10.31.133.24@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513099.770595:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.24@tcp1 routed destination 00000400:00000200:28.0:1616513099.770614:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.24@tcp1(10.31.133.24@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:24.0:1616513099.770616:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510961771456, offset 192 00000800:00000200:28.0:1616513099.770620:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513099.770621:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.133.24@tcp1 00000800:00000200:28.0:1616513099.770626:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000400:00000200:24.0:1616513099.770628:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.133.24@tcp1 routed destination 00000800:00000200:28.0:1616513099.770629:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513099.770631:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513099.770636:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.770641:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.133.24@tcp1(10.31.133.24@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513099.770642:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.770647:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.770651:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.770654:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513099.770656:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.770660:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.770665:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513099.770667:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513099.770696:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513099.770703:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.770705:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.770712:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.770716:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fccf68 00000800:00000200:4.0:1616513099.770720:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513099.770722:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:4.0:1616513099.770723:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513099.770729:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.770732:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.770738:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:10.0:1616513099.770738:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.770744:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f23b8 00000800:00000200:26.0:1616513099.770747:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.786570:0:6497:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930064, portal 4 00000100:00000200:30.0:1616513099.786574:0:6497:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930064, offset 0 00000400:00000200:30.0:1616513099.786580:0:6497:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513099.786584:0:6497:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513099.786596:0:6497:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.786601:0:6497:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513099.786607:0:6497:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.786609:0:6497:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513099.786611:0:6497:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513099.786615:0:6497:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513099.786623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513099.786677:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:0.0:1616513099.786683:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:0.0:1616513099.786686:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513099.786687:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:0.0:1616513099.786692:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:0.0:1616513099.786698:0:6463:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74dd218480 x1693067725930064/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:0.0:1616513099.786709:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef3a8 00000800:00000200:0.0:1616513099.786713:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513099.786860:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:4.0:1616513099.786915:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513099.786924:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000800:00000200:20.0:1616513099.786929:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000400:00000200:4.0:1616513099.786933:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:4.0:1616513099.786941:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627e50 00000400:00000200:4.0:1616513099.786948:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866de001 [1] + 192 00000400:00000200:4.0:1616513099.786953:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.786958:0:6464:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74dd218480 x1693067725930064/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:4.0:1616513099.786973:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:4.0:1616513099.786976:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000100:00000200:30.0:1616513099.786978:0:6497:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74dd218480 x1693067725930064/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:4.0:1616513099.786978:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513099.786989:0:6497:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74dd218480 x1693067725930064/t0(0) o13->boslfs02-OST0021-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513099.786999:0:6497:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee908 00000800:00000200:11.2:1616513099.814001:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.814015:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.814023:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:7.2:1616513099.814023:0:51571:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513099.814032:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.24@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.814040:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.24@tcp1 of length 224 into portal 26 MB=0x6018c3fc35040 00000400:00000200:8.0:1616513099.814047:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.24@tcp1 of length 224/224 into md 0xf97d13f9 [1] + 0 00000400:00000200:8.0:1616513099.814051:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bed48 00000400:00000200:8.0:1616513099.814053:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.814056:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.814061:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bed48 00000800:00000200:8.0:1616513099.814064:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.814066:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.814068:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.814072:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513099.814090:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513099.814100:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513099.814109:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.24@tcp1 : PUT - for me 00000400:00000200:22.0:1616513099.814118:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.24@tcp1 of length 224 into portal 12 MB=0x6018c3fc350c0 00000400:00000200:22.0:1616513099.814124:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.24@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 58128 00000100:00000200:24.0:1616513099.814130:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690551737077824 00000400:00000200:22.0:1616513099.814130:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513099.814134:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00010000:00000200:24.0:1616513099.814141:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2be850 x1690551737077824/t0(0) o400->8447777a-8b58-630d-f79f-652eb6219f4c@10.31.142.24@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:22.0:1616513099.814142:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513099.814145:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.814145:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513099.814149:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513099.814156:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690551737077824, offset 192 00000100:00000200:30.0:1616513099.814162:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690551737077952 00000400:00000200:24.0:1616513099.814162:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.24@tcp1 00000400:00000200:24.0:1616513099.814169:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.24@tcp1 routed destination 00010000:00000200:30.0:1616513099.814174:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7ad00 x1690551737077952/t0(0) o400->6f5d3274-a24b-714f-357e-c8c5ed130626@10.31.142.24@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000400:00000200:24.0:1616513099.814184:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.24@tcp1(10.31.142.24@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:30.0:1616513099.814188:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690551737077952, offset 192 00000800:00000200:24.0:1616513099.814190:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:30.0:1616513099.814194:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.24@tcp1 00000800:00000200:24.0:1616513099.814196:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.814198:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.814201:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513099.814202:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.24@tcp1 routed destination 00000800:00000200:24.0:1616513099.814205:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.814211:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.814213:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:30.0:1616513099.814228:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.24@tcp1(10.31.142.24@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.814234:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513099.814240:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.814242:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513099.814243:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513099.814247:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.814252:0:51571:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.814263:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.814268:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000800:00000200:20.0:1616513099.814269:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:10.0:1616513099.814271:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513099.814274:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513099.814276:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:10.0:1616513099.814276:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.814279:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513099.814279:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee908 00000800:00000200:10.0:1616513099.814283:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:20.0:1616513099.814285:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.814291:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3a90 00000800:00000200:20.0:1616513099.814294:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.814297:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:31.0:1616513099.871547:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.216@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000400:00000200:31.0:1616513099.871555:0:6477:0:(router.c:1092:lnet_ping_router_locked()) rtr 10.242.46.217@o2ib1 60: deadline 0 ping_notsent 0 alive 1 alive_count 1 lpni_ping_timestamp 1878376 00000800:00000200:7.2:1616513099.893268:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513099.893325:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513099.893335:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513099.893345:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.88@tcp1 : PUT - for me 00000400:00000200:18.0:1616513099.893354:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.88@tcp1 of length 224 into portal 12 MB=0x5ffdd33e37800 00000400:00000200:18.0:1616513099.893361:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.88@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 58352 00000400:00000200:18.0:1616513099.893366:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513099.893369:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513099.893376:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513099.893380:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.893380:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513099.893384:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.893396:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700406953984 00010000:00000200:30.0:1616513099.893408:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645ba880 x1688700406953984/t0(0) o400->82d9d262-9945-f208-e72e-b6c299c49c20@10.31.138.88@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513099.893421:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688700406953984, offset 192 00000400:00000200:30.0:1616513099.893428:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.88@tcp1 00000800:00000200:11.2:1616513099.893433:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:30.0:1616513099.893435:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.88@tcp1 routed destination 00000400:00000200:30.0:1616513099.893455:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.88@tcp1(10.31.138.88@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.893461:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513099.893466:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.893469:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513099.893471:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513099.893476:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513099.893491:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.893500:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.893501:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:8.0:1616513099.893507:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:0.0:1616513099.893509:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.88@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.893510:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.893516:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513099.893519:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.88@tcp1 of length 224 into portal 26 MB=0x5ffdd33e37400 00000400:00000200:8.0:1616513099.893521:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef4b8 00000800:00000200:8.0:1616513099.893524:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.893526:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:0.0:1616513099.893526:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.88@tcp1 of length 224/224 into md 0xf97d1409 [1] + 0 00000400:00000200:0.0:1616513099.893531:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf760 00000400:00000200:0.0:1616513099.893533:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.893536:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:0.0:1616513099.893541:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf760 00000800:00000200:0.0:1616513099.893545:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.893549:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.893551:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513099.893606:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688700406952960 00010000:00000200:24.0:1616513099.893617:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a0850 x1688700406952960/t0(0) o400->a5da1624-ce45-5a54-6440-1d692b9d447d@10.31.138.88@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.893632:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688700406952960, offset 192 00000400:00000200:24.0:1616513099.893638:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.88@tcp1 00000400:00000200:24.0:1616513099.893645:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.88@tcp1 routed destination 00000400:00000200:24.0:1616513099.893660:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.88@tcp1(10.31.138.88@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.893667:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.893673:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.893676:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513099.893678:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.893683:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.893688:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513099.893690:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513099.893701:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.893708:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.893711:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:10.0:1616513099.893712:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.893717:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.893723:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3100 00000800:00000200:26.0:1616513099.893727:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:19.0:1616513099.898638:0:6509:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930080, portal 4 00000100:00000200:19.0:1616513099.898644:0:6509:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930080, offset 0 00000400:00000200:19.0:1616513099.898651:0:6509:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:19.0:1616513099.898658:0:6509:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:19.0:1616513099.898672:0:6509:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:19.0:1616513099.898679:0:6509:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:19.0:1616513099.898686:0:6509:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:19.0:1616513099.898689:0:6509:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:19.0:1616513099.898692:0:6509:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:19.0:1616513099.898698:0:6509:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513099.898743:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513099.898801:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513099.898808:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:22.0:1616513099.898812:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513099.898818:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:22.0:1616513099.898825:0:6472:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cca3ebf00 x1693067725930080/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:22.0:1616513099.898837:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1706990 00000800:00000200:22.0:1616513099.898841:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513099.898844:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513099.898850:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:10.0:1616513099.898854:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513099.898858:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513099.898865:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627e60 00000400:00000200:22.0:1616513099.898872:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d1b39 [1] + 192 00000400:00000200:22.0:1616513099.898877:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513099.898881:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cca3ebf00 x1693067725930080/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513099.898894:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513099.898897:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513099.898899:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:19.0:1616513099.898906:0:6509:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cca3ebf00 x1693067725930080/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:19.0:1616513099.898919:0:6509:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cca3ebf00 x1693067725930080/t0(0) o13->boslfs02-OST0010-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:19.0:1616513099.898930:0:6509:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c1707980 00000100:00000200:7.0:1616513099.914559:0:6510:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930096, portal 4 00000100:00000200:7.0:1616513099.914562:0:6510:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930096, offset 0 00000400:00000200:7.0:1616513099.914566:0:6510:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.232@o2ib1 00000400:00000200:7.0:1616513099.914570:0:6510:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.232@o2ib1 local destination 00000400:00000200:7.0:1616513099.914578:0:6510:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.232@o2ib1(10.242.46.232@o2ib1:10.242.46.232@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513099.914581:0:6510:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.232@o2ib1 00000800:00000200:7.0:1616513099.914585:0:6510:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d8cc4318080] -> 10.242.46.232@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513099.914587:0:6510:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:7.0:1616513099.914588:0:6510:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:7.0:1616513099.914591:0:6510:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.2:1616513099.914638:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513099.914658:0:6468:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)++ 00000800:00000200:1.0:1616513099.914664:0:6468:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d8cc87efa00] (23)-- 00000400:00000200:1.0:1616513099.914667:0:6468:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1616513099.914674:0:6468:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.232@o2ib1: PUT: OK 00000100:00000200:1.0:1616513099.914681:0:6468:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cca3eba80 x1693067725930096/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:1.0:1616513099.914693:0:6468:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637eaa0 00000800:00000200:1.0:1616513099.914697:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513099.914700:0:6468:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000800:00000200:1.2:1616513099.914761:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d8cc87efa00] (20)++ 00000800:00000200:1.0:1616513099.914770:0:6467:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)++ 00000800:00000200:1.0:1616513099.914775:0:6467:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.232@o2ib1 00000800:00000200:5.0:1616513099.914779:0:6465:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (22)-- 00000400:00000200:1.0:1616513099.914784:0:6467:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.232@o2ib1 : PUT - for me 00000400:00000200:1.0:1616513099.914793:0:6467:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.232@o2ib1 of length 336 into portal 4 MB=0x603d60c627e70 00000400:00000200:1.0:1616513099.914799:0:6467:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.232@o2ib1 of length 336/336 into md 0x43700385 [1] + 192 00000400:00000200:1.0:1616513099.914804:0:6467:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:1.0:1616513099.914809:0:6467:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cca3eba80 x1693067725930096/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513099.914822:0:6467:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (21)++ 00000100:00000200:7.0:1616513099.914825:0:6510:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cca3eba80 x1693067725930096/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:1.0:1616513099.914825:0:6467:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d8cc87efa00] (22)-- 00000800:00000200:1.0:1616513099.914826:0:6467:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d8cc87efa00] (21)-- 00000100:00000200:7.0:1616513099.914831:0:6510:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cca3eba80 x1693067725930096/t0(0) o13->boslfs02-OST0027-osc-MDT0001@10.242.46.232@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513099.914837:0:6510:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637ef68 00000800:00000200:11.2:1616513099.926390:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513099.926449:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.926458:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.926460:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513099.926468:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.111@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.926478:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.111@tcp1 of length 224 into portal 26 MB=0x6001d34b83d80 00000400:00000200:4.0:1616513099.926487:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.111@tcp1 of length 224/224 into md 0xf97d1419 [1] + 0 00000400:00000200:4.0:1616513099.926491:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be000 00000400:00000200:4.0:1616513099.926494:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.926497:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513099.926502:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be000 00000800:00000200:4.0:1616513099.926505:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.926508:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.926510:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513099.926567:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975298805120 00010000:00000200:24.0:1616513099.926578:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d746a250050 x1688975298805120/t0(0) o400->56c335d7-cc56-f221-bf38-e649066cb69f@10.31.142.111@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.926593:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688975298805120, offset 192 00000400:00000200:24.0:1616513099.926601:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.111@tcp1 00000400:00000200:24.0:1616513099.926609:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.111@tcp1 routed destination 00000400:00000200:24.0:1616513099.926624:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.111@tcp1(10.31.142.111@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.926631:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.926637:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.926639:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.926642:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.926647:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513099.926652:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.926654:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513099.926705:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.926712:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.926715:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.926722:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.926728:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2c38 00000800:00000200:20.0:1616513099.926731:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.926734:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.926740:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.926740:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.926747:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.111@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.926755:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.111@tcp1 of length 224 into portal 12 MB=0x6001d34b83f80 00000400:00000200:20.0:1616513099.926762:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.111@tcp1 of length 224/224 into md 0x865be291 [1] + 88552 00000400:00000200:20.0:1616513099.926767:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.926769:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513099.926776:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.926778:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.926780:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.926839:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688975298805632 00010000:00000200:28.0:1616513099.926851:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661b00 x1688975298805632/t0(0) o400->b1396153-fb20-b899-35fb-abcedd34153d@10.31.142.111@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.926864:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688975298805632, offset 192 00000400:00000200:28.0:1616513099.926870:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.111@tcp1 00000400:00000200:28.0:1616513099.926877:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.111@tcp1 routed destination 00000400:00000200:28.0:1616513099.926891:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.111@tcp1(10.31.142.111@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.926897:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.926902:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.926905:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.926907:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.926911:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.926920:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.926975:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.926982:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.926985:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.926991:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.926995:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc000 00000800:00000200:26.0:1616513099.926999:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.927002:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513099.928812:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513099.928822:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.928829:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513099.928836:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.71@tcp1 : PUT - for me 00000400:00000200:10.0:1616513099.928843:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.71@tcp1 of length 224 into portal 26 MB=0x601ab16d465c0 00000400:00000200:10.0:1616513099.928849:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.71@tcp1 of length 224/224 into md 0xf97d1439 [1] + 0 00000400:00000200:10.0:1616513099.928852:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf540 00000400:00000200:10.0:1616513099.928855:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513099.928857:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513099.928860:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf540 00000800:00000200:10.0:1616513099.928862:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.928865:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513099.928867:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513099.928869:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513099.928923:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690684194317760 00010000:00000200:24.0:1616513099.928934:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a2850 x1690684194317760/t0(0) o400->f57cf01f-35ad-d4e1-8cda-fc90d35412a4@10.31.141.71@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.928948:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690684194317760, offset 192 00000400:00000200:24.0:1616513099.928954:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.71@tcp1 00000400:00000200:24.0:1616513099.928962:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.71@tcp1 routed destination 00000400:00000200:24.0:1616513099.928977:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.71@tcp1(10.31.141.71@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.928983:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.928989:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.928991:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.928994:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.928998:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513099.929005:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.929015:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.929070:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.929080:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513099.929080:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000800:00000200:8.0:1616513099.929085:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (24)-- 00000400:00000200:8.0:1616513099.929087:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.929089:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.71@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.929093:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:0.0:1616513099.929093:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513099.929097:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3540 00000400:00000200:4.0:1616513099.929097:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.71@tcp1 of length 224 into portal 12 MB=0x601ab16d46640 00000800:00000200:8.0:1616513099.929100:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513099.929104:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.71@tcp1 of length 224/224 into md 0x865be291 [1] + 88776 00000400:00000200:4.0:1616513099.929109:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.929111:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:4.0:1616513099.929118:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.929120:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513099.929122:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.929182:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690684194317888 00010000:00000200:28.0:1616513099.929194:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb23e780 x1690684194317888/t0(0) o400->044c6967-09ef-ee1f-c9c1-3a766762baae@10.31.141.71@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.929208:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690684194317888, offset 192 00000400:00000200:28.0:1616513099.929214:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.71@tcp1 00000400:00000200:28.0:1616513099.929222:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.71@tcp1 routed destination 00000400:00000200:28.0:1616513099.929238:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.71@tcp1(10.31.141.71@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.929245:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.929251:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.929253:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.929256:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.929261:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.929266:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.929319:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.929326:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.929329:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513099.929335:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.929339:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd188 00000800:00000200:26.0:1616513099.929345:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.929348:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:7.0:1616513099.930568:0:6511:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930112, portal 4 00000100:00000200:7.0:1616513099.930572:0:6511:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930112, offset 0 00000400:00000200:7.0:1616513099.930580:0:6511:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:7.0:1616513099.930586:0:6511:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:7.0:1616513099.930600:0:6511:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:7.0:1616513099.930606:0:6511:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:7.0:1616513099.930612:0:6511:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:7.0:1616513099.930615:0:6511:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:7.0:1616513099.930617:0:6511:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:7.0:1616513099.930622:0:6511:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513099.930630:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:20.0:1616513099.930685:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:20.0:1616513099.930692:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:20.0:1616513099.930695:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.930702:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:20.0:1616513099.930709:0:6461:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8da4dd600900 x1693067725930112/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:20.0:1616513099.930721:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637ff58 00000800:00000200:20.0:1616513099.930727:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:20.0:1616513099.930730:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513099.930857:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:8.0:1616513099.930867:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513099.930874:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:8.0:1616513099.930881:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:8.0:1616513099.930888:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627e80 00000400:00000200:8.0:1616513099.930894:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866de071 [1] + 192 00000400:00000200:8.0:1616513099.930898:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.930903:0:6462:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8da4dd600900 x1693067725930112/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513099.930916:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513099.930918:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (23)-- 00000800:00000200:4.0:1616513099.930918:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (23)-- 00000100:00000200:7.0:1616513099.930921:0:6511:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8da4dd600900 x1693067725930112/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513099.930923:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:7.0:1616513099.930928:0:6511:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8da4dd600900 x1693067725930112/t0(0) o13->boslfs02-OST0035-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:7.0:1616513099.930934:0:6511:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841637ef68 00000800:00000200:7.2:1616513099.958781:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513099.958793:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513099.958800:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513099.958807:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.82@tcp1 : PUT - for me 00000400:00000200:10.0:1616513099.958814:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.82@tcp1 of length 224 into portal 26 MB=0x5ffd4dca5dac0 00000400:00000200:10.0:1616513099.958820:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.82@tcp1 of length 224/224 into md 0xf97d1449 [1] + 0 00000400:00000200:10.0:1616513099.958823:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bf3a8 00000400:00000200:10.0:1616513099.958825:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513099.958827:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513099.958830:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bf3a8 00000800:00000200:10.0:1616513099.958833:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.958835:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513099.958836:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513099.958838:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513099.958895:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664583559872 00010000:00000200:24.0:1616513099.958905:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc050 x1688664583559872/t0(0) o400->4dd93725-b389-9170-51ba-33e79b2c9376@10.31.136.82@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.958920:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688664583559872, offset 192 00000400:00000200:24.0:1616513099.958926:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.82@tcp1 00000400:00000200:24.0:1616513099.958934:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.82@tcp1 routed destination 00000400:00000200:24.0:1616513099.958949:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.82@tcp1(10.31.136.82@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.958956:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.958964:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.958967:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.958969:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.958974:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513099.958980:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513099.958983:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.959038:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.959045:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.959048:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.959054:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.959060:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3f58 00000800:00000200:20.0:1616513099.959063:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.959066:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.959072:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513099.959080:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.82@tcp1 : PUT - for me 00000400:00000200:20.0:1616513099.959088:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.82@tcp1 of length 224 into portal 12 MB=0x5ffd4dca5dc00 00000800:00000200:0.0:1616513099.959092:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513099.959094:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.82@tcp1 of length 224/224 into md 0x865be291 [1] + 89000 00000400:00000200:20.0:1616513099.959099:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513099.959102:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513099.959108:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513099.959110:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.959112:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513099.959172:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688664583560192 00010000:00000200:28.0:1616513099.959184:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c661680 x1688664583560192/t0(0) o400->ce57591e-5026-f4fe-2ec0-5722f78f9236@10.31.136.82@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.959199:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688664583560192, offset 192 00000400:00000200:28.0:1616513099.959205:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.82@tcp1 00000400:00000200:28.0:1616513099.959213:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.82@tcp1 routed destination 00000400:00000200:28.0:1616513099.959228:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.82@tcp1(10.31.136.82@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.959235:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.959240:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.959243:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.959245:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.959250:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.959255:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.959310:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.959317:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.959320:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.959323:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.959327:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.959331:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd298 00000800:00000200:26.0:1616513099.959335:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.963341:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.963354:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.963361:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.963369:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.209@tcp1 : PUT - for me 00000800:00000200:7.2:1616513099.963372:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:8.0:1616513099.963376:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.209@tcp1 of length 224 into portal 26 MB=0x5ffe4276d77c0 00000400:00000200:8.0:1616513099.963382:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.128.209@tcp1 of length 224/224 into md 0xf97d1489 [1] + 0 00000800:00000200:10.0:1616513099.963384:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513099.963386:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02bedd0 00000400:00000200:8.0:1616513099.963389:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000800:00000200:10.0:1616513099.963390:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000100:00000200:8.0:1616513099.963392:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.963395:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02bedd0 00000400:00000200:10.0:1616513099.963397:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.128.209@tcp1 : PUT - for me 00000800:00000200:8.0:1616513099.963398:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.963400:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.963402:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:10.0:1616513099.963403:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.128.209@tcp1 of length 224 into portal 12 MB=0x5ffe4276d7b00 00000800:00000200:8.0:1616513099.963404:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513099.963407:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.128.209@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 58576 00000400:00000200:10.0:1616513099.963411:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513099.963412:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:10.0:1616513099.963417:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.963419:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513099.963420:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513099.963422:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.963446:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688730262665984 00010000:00000200:30.0:1616513099.963458:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bdec00 x1688730262665984/t0(0) o400->bba65a1c-25cd-ee81-a956-8b5f1af2b730@10.31.128.209@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513099.963469:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688730262665152 00000100:00000200:30.0:1616513099.963472:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688730262665984, offset 192 00000400:00000200:30.0:1616513099.963478:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.209@tcp1 00010000:00000200:24.0:1616513099.963480:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2ba850 x1688730262665152/t0(0) o400->7acd755e-a0b1-0f42-672e-8eca3d2ced8a@10.31.128.209@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:30.0:1616513099.963486:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.209@tcp1 routed destination 00000100:00000200:24.0:1616513099.963495:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688730262665152, offset 192 00000400:00000200:30.0:1616513099.963501:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.209@tcp1(10.31.128.209@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513099.963501:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.128.209@tcp1 00000800:00000200:30.0:1616513099.963507:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513099.963509:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.128.209@tcp1 routed destination 00000800:00000200:30.0:1616513099.963513:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.963516:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513099.963519:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513099.963524:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.963524:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.128.209@tcp1(10.31.128.209@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513099.963529:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.963531:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.963537:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.963539:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513099.963542:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.963546:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.963552:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513099.963553:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:20.0:1616513099.963582:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.963589:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513099.963592:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513099.963598:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.963602:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eecc0 00000800:00000200:20.0:1616513099.963606:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513099.963609:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513099.963609:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.963618:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.963621:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513099.963623:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.963627:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.963633:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3870 00000800:00000200:18.0:1616513099.963636:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.966394:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.966406:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.966412:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.966419:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.59@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.966427:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.59@tcp1 of length 224 into portal 26 MB=0x5ffd49a49ac80 00000400:00000200:8.0:1616513099.966432:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.59@tcp1 of length 224/224 into md 0xf97d1499 [1] + 0 00000400:00000200:8.0:1616513099.966443:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d82d02be660 00000400:00000200:8.0:1616513099.966445:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.966447:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513099.966450:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d82d02be660 00000800:00000200:8.0:1616513099.966452:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.966454:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.966456:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.966458:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.966462:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.966469:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.59@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.966474:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.59@tcp1 of length 224 into portal 12 MB=0x5ffd49a49ad40 00000400:00000200:8.0:1616513099.966480:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.59@tcp1 of length 224/224 into md 0x865be291 [1] + 89224 00000400:00000200:8.0:1616513099.966483:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.966485:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513099.966490:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513099.966491:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.966493:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.966495:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513099.966513:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663470222464 00010000:00000200:24.0:1616513099.966524:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bf850 x1688663470222464/t0(0) o400->9d8b6e30-2898-1314-a533-00895796539e@10.31.138.59@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.966538:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688663470222464, offset 192 00000400:00000200:24.0:1616513099.966544:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.59@tcp1 00000400:00000200:24.0:1616513099.966552:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.59@tcp1 routed destination 00000100:00000200:28.0:1616513099.966557:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688663470222656 00000400:00000200:24.0:1616513099.966567:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.59@tcp1(10.31.138.59@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:28.0:1616513099.966570:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb8ba80 x1688663470222656/t0(0) o400->dede8388-3942-5d5d-ded7-a6a6b4bb9d4c@10.31.138.59@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513099.966574:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513099.966579:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.966582:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.966584:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:28.0:1616513099.966585:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688663470222656, offset 192 00000800:00000200:24.0:1616513099.966589:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:28.0:1616513099.966591:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.59@tcp1 00000800:00000200:11.2:1616513099.966594:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.966597:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513099.966598:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.59@tcp1 routed destination 00000400:00000200:28.0:1616513099.966613:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.59@tcp1(10.31.138.59@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.966620:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.966625:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.966628:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.966632:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.966637:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513099.966647:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513099.966652:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513099.966655:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513099.966658:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.966660:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513099.966663:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513099.966665:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:10.0:1616513099.966668:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513099.966670:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513099.966670:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2880 00000800:00000200:4.0:1616513099.966674:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513099.966676:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513099.966680:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd8f8 00000800:00000200:10.0:1616513099.966684:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513099.966686:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.978472:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930128, portal 4 00000100:00000200:30.0:1616513099.978476:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930128, offset 0 00000400:00000200:30.0:1616513099.978482:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:30.0:1616513099.978486:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:30.0:1616513099.978497:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.978503:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:30.0:1616513099.978507:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.978510:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:30.0:1616513099.978511:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:30.0:1616513099.978515:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:5.2:1616513099.978523:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:8.0:1616513099.978535:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513099.978539:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:8.0:1616513099.978541:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513099.978546:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513099.978551:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d5df6bd9b00 x1693067725930128/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513099.978561:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efcb0 00000800:00000200:8.0:1616513099.978564:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513099.978567:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:5.2:1616513099.978741:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:0.0:1616513099.978794:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513099.978804:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:0.0:1616513099.978813:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:0.0:1616513099.978820:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627e90 00000400:00000200:0.0:1616513099.978827:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866de0d1 [1] + 192 00000400:00000200:0.0:1616513099.978832:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.978837:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d5df6bd9b00 x1693067725930128/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:4.0:1616513099.978848:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513099.978852:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513099.978855:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513099.978856:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:30.0:1616513099.978857:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d5df6bd9b00 x1693067725930128/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:30.0:1616513099.978869:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d5df6bd9b00 x1693067725930128/t0(0) o13->boslfs02-OST0025-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513106 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513099.978878:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee990 00000800:00000200:7.2:1616513099.981365:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513099.981383:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513099.981395:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.981399:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513099.981406:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.85@tcp1 : PUT - for me 00000400:00000200:8.0:1616513099.981412:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.85@tcp1 of length 224 into portal 12 MB=0x5ffde3dc0c1c0 00000400:00000200:8.0:1616513099.981417:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.85@tcp1 of length 224/224 into md 0x865be291 [1] + 89448 00000800:00000200:26.0:1616513099.981420:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513099.981421:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513099.981423:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513099.981427:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:26.0:1616513099.981428:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000800:00000200:8.0:1616513099.981429:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513099.981431:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.981433:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:26.0:1616513099.981448:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.85@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.981457:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.85@tcp1 of length 224 into portal 26 MB=0x5ffde3dc0bf00 00000400:00000200:26.0:1616513099.981463:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.85@tcp1 of length 224/224 into md 0xf97d14a9 [1] + 0 00000400:00000200:26.0:1616513099.981468:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789e48 00000400:00000200:26.0:1616513099.981470:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.981472:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513099.981476:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789e48 00000800:00000200:26.0:1616513099.981479:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.981482:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.981484:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.981486:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:28.0:1616513099.981507:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704867418560 00010000:00000200:28.0:1616513099.981518:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89b00 x1688704867418560/t0(0) o400->f2db997c-5301-a7f4-28eb-a862e95a3dc3@10.31.142.85@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513099.981531:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688704867418560, offset 192 00000400:00000200:28.0:1616513099.981537:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.85@tcp1 00000100:00000200:24.0:1616513099.981540:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688704867417856 00000400:00000200:28.0:1616513099.981544:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.85@tcp1 routed destination 00010000:00000200:24.0:1616513099.981551:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb850 x1688704867417856/t0(0) o400->a3581b9e-d306-979a-7f6e-b860e69a36d5@10.31.142.85@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513099.981558:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.85@tcp1(10.31.142.85@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.981564:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513099.981566:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688704867417856, offset 192 00000800:00000200:28.0:1616513099.981569:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.981572:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000400:00000200:24.0:1616513099.981572:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.85@tcp1 00000800:00000200:28.0:1616513099.981574:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513099.981578:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:24.0:1616513099.981580:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.85@tcp1 routed destination 00000800:00000200:11.2:1616513099.981584:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513099.981595:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.85@tcp1(10.31.142.85@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.981601:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.981607:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.981610:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513099.981612:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.981617:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.981621:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513099.981623:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:10.0:1616513099.981629:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.981632:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:10.0:1616513099.981634:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:4.0:1616513099.981637:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:10.0:1616513099.981638:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:10.0:1616513099.981641:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3d38 00000800:00000200:10.0:1616513099.981643:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513099.981644:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:10.0:1616513099.981645:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513099.981647:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513099.981649:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513099.981653:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513099.981657:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcc550 00000800:00000200:4.0:1616513099.981661:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513099.983077:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513099.983131:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513099.983139:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513099.983148:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.175@tcp1 : PUT - for me 00000400:00000200:26.0:1616513099.983157:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.175@tcp1 of length 224 into portal 26 MB=0x5ffd41cc91180 00000400:00000200:26.0:1616513099.983163:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.175@tcp1 of length 224/224 into md 0xf97d14d9 [1] + 0 00000400:00000200:26.0:1616513099.983174:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d7884c8 00000400:00000200:26.0:1616513099.983177:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513099.983179:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513099.983183:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7884c8 00000800:00000200:26.0:1616513099.983186:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513099.983188:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513099.983191:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513099.983194:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513099.983244:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661364642176 00010000:00000200:24.0:1616513099.983254:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a6850 x1688661364642176/t0(0) o400->2e3f7af4-cc49-e285-3a65-b4c595083314@10.31.138.175@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:7.2:1616513099.983262:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513099.983267:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688661364642176, offset 192 00000800:00000200:10.0:1616513099.983270:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:24.0:1616513099.983273:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.175@tcp1 00000800:00000200:10.0:1616513099.983274:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513099.983280:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.175@tcp1 : PUT - for me 00000400:00000200:24.0:1616513099.983281:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.175@tcp1 routed destination 00000400:00000200:10.0:1616513099.983286:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.175@tcp1 of length 224 into portal 12 MB=0x5ffd41cc91240 00000400:00000200:10.0:1616513099.983291:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.175@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 58800 00000400:00000200:10.0:1616513099.983294:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000400:00000200:24.0:1616513099.983295:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.175@tcp1(10.31.138.175@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:10.0:1616513099.983296:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513099.983301:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:10.0:1616513099.983301:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.983302:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513099.983304:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513099.983305:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:24.0:1616513099.983306:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.983309:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.983311:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:30.0:1616513099.983315:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688661364642368 00000800:00000200:24.0:1616513099.983315:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513099.983321:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00010000:00000200:30.0:1616513099.983323:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bde300 x1688661364642368/t0(0) o400->c286a46e-e2d1-d90c-10c4-2ab68ad6bf81@10.31.138.175@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513099.983334:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688661364642368, offset 192 00000400:00000200:30.0:1616513099.983340:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.175@tcp1 00000400:00000200:30.0:1616513099.983346:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.175@tcp1 routed destination 00000400:00000200:30.0:1616513099.983358:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.175@tcp1(10.31.138.175@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:11.2:1616513099.983361:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513099.983364:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513099.983368:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.983371:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513099.983372:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513099.983373:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:30.0:1616513099.983375:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513099.983377:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513099.983379:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:7.2:1616513099.983381:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000400:00000200:8.0:1616513099.983384:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513099.983388:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3ed0 00000800:00000200:26.0:1616513099.983390:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:8.0:1616513099.983391:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.983393:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513099.983394:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513099.983396:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513099.983399:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:26.0:1616513099.983401:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513099.983404:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee990 00000800:00000200:26.0:1616513099.983407:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.991038:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:0.0:1616513099.991094:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.991103:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.991111:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.119@tcp1 : PUT - for me 00000800:00000200:7.2:1616513099.991113:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000400:00000200:0.0:1616513099.991120:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.119@tcp1 of length 224 into portal 26 MB=0x5ffdd7ca22900 00000800:00000200:10.0:1616513099.991124:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000400:00000200:0.0:1616513099.991126:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.119@tcp1 of length 224/224 into md 0xf97d14e9 [1] + 0 00000800:00000200:10.0:1616513099.991130:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:0.0:1616513099.991130:0:6463:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789cb0 00000400:00000200:0.0:1616513099.991133:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.991135:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513099.991137:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.119@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.991140:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789cb0 00000800:00000200:0.0:1616513099.991142:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:10.0:1616513099.991143:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.119@tcp1 of length 224 into portal 12 MB=0x5ffdd7ca22ac0 00000800:00000200:0.0:1616513099.991145:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.991147:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:10.0:1616513099.991148:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.119@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 59024 00000400:00000200:10.0:1616513099.991151:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513099.991153:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.991153:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:10.0:1616513099.991158:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513099.991160:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513099.991162:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513099.991164:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513099.991168:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701627411136 00010000:00000200:30.0:1616513099.991174:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6bd9680 x1688701627411136/t0(0) o400->cfa8be43-ad66-7cf9-8b11-153e15fe8723@10.31.132.119@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513099.991183:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688701627411136, offset 192 00000400:00000200:30.0:1616513099.991188:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.119@tcp1 00000400:00000200:30.0:1616513099.991193:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.119@tcp1 routed destination 00000400:00000200:30.0:1616513099.991205:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.119@tcp1(10.31.132.119@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513099.991210:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000100:00000200:24.0:1616513099.991212:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688701627410688 00000800:00000200:30.0:1616513099.991214:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513099.991216:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513099.991218:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513099.991220:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00010000:00000200:24.0:1616513099.991222:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460328850 x1688701627410688/t0(0) o400->881dfb22-d175-2953-4069-e0c1229f80b1@10.31.132.119@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:11.2:1616513099.991226:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513099.991234:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000100:00000200:24.0:1616513099.991236:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688701627410688, offset 192 00000800:00000200:8.0:1616513099.991237:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513099.991239:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:24.0:1616513099.991243:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.119@tcp1 00000400:00000200:8.0:1616513099.991243:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513099.991246:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee4c8 00000400:00000200:24.0:1616513099.991250:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.119@tcp1 routed destination 00000800:00000200:8.0:1616513099.991255:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513099.991257:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:24.0:1616513099.991264:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.119@tcp1(10.31.132.119@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.991271:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513099.991276:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.991279:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513099.991281:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513099.991285:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.991290:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513099.991291:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513099.991344:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513099.991351:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513099.991354:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513099.991356:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513099.991360:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513099.991366:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3a08 00000800:00000200:18.0:1616513099.991369:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513099.996224:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513099.996281:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513099.996290:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513099.996299:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.108@tcp1 : PUT - for me 00000400:00000200:4.0:1616513099.996309:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.108@tcp1 of length 224 into portal 26 MB=0x602ada2a92800 00000400:00000200:4.0:1616513099.996316:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.108@tcp1 of length 224/224 into md 0xf97d1519 [1] + 0 00000400:00000200:4.0:1616513099.996320:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d7897e8 00000400:00000200:4.0:1616513099.996323:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513099.996326:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513099.996330:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7897e8 00000800:00000200:4.0:1616513099.996333:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513099.996335:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)++ 00000800:00000200:4.0:1616513099.996336:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.996343:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513099.996346:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.996346:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513099.996355:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.108@tcp1 : PUT - for me 00000400:00000200:0.0:1616513099.996365:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.108@tcp1 of length 224 into portal 12 MB=0x602ada2a92880 00000400:00000200:0.0:1616513099.996373:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.108@tcp1 of length 224/224 into md 0x865be291 [1] + 89672 00000400:00000200:0.0:1616513099.996378:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513099.996381:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513099.996387:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513099.996391:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513099.996392:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513099.996405:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691794641856512 00010000:00000200:24.0:1616513099.996415:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b0850 x1691794641856512/t0(0) o400->17b45124-cb65-c5c7-cf73-9fbfd5fdca58@10.31.141.108@tcp1:56/0 lens 224/224 e 0 to 0 dl 1616513191 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513099.996429:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691794641856512, offset 192 00000400:00000200:24.0:1616513099.996466:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.108@tcp1 00000400:00000200:24.0:1616513099.996474:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.108@tcp1 routed destination 00000100:00000200:28.0:1616513099.996485:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691794641856640 00000400:00000200:24.0:1616513099.996487:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.108@tcp1(10.31.141.108@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513099.996493:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:28.0:1616513099.996497:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d569bb09200 x1691794641856640/t0(0) o400->5a4acf17-8332-0d3a-25a5-ecd704a7417b@10.31.141.108@tcp1:725/0 lens 224/224 e 0 to 0 dl 1616513105 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513099.996499:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513099.996501:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513099.996503:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513099.996508:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513099.996510:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691794641856640, offset 192 00000800:00000200:11.2:1616513099.996513:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513099.996514:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513099.996516:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.108@tcp1 00000400:00000200:28.0:1616513099.996523:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.108@tcp1 routed destination 00000400:00000200:28.0:1616513099.996537:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.108@tcp1(10.31.141.108@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513099.996543:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513099.996549:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513099.996551:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513099.996553:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513099.996557:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513099.996564:0:51591:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513099.996565:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513099.996572:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:10.0:1616513099.996574:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513099.996575:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513099.996575:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:10.0:1616513099.996579:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:20.0:1616513099.996581:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:10.0:1616513099.996581:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:10.0:1616513099.996585:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:20.0:1616513099.996586:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3760 00000800:00000200:20.0:1616513099.996589:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000400:00000200:10.0:1616513099.996589:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd210 00000800:00000200:10.0:1616513099.996592:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:10.0:1616513099.996594:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513100.016481:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:4.0:1616513100.016537:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513100.016546:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513100.016555:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.179@tcp1 : PUT - for me 00000400:00000200:4.0:1616513100.016565:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.179@tcp1 of length 224 into portal 26 MB=0x60182bd8bbb40 00000400:00000200:4.0:1616513100.016571:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.179@tcp1 of length 224/224 into md 0xf97d1529 [1] + 0 00000400:00000200:4.0:1616513100.016576:0:6464:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788aa0 00000400:00000200:4.0:1616513100.016578:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513100.016581:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:4.0:1616513100.016586:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788aa0 00000800:00000200:4.0:1616513100.016589:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:0.0:1616513100.016589:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513100.016592:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513100.016596:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513100.016599:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513100.016599:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:0.0:1616513100.016607:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.179@tcp1 : PUT - for me 00000400:00000200:0.0:1616513100.016616:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.179@tcp1 of length 224 into portal 12 MB=0x60182bd8bbc00 00000400:00000200:0.0:1616513100.016621:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.179@tcp1 of length 224/224 into md 0x865be291 [1] + 89896 00000400:00000200:0.0:1616513100.016627:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513100.016629:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513100.016635:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513100.016638:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513100.016640:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513100.016648:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510897691456 00010000:00000200:24.0:1616513100.016657:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e72850 x1690510897691456/t0(0) o400->5732845d-c027-747e-76da-d74c4cf29b3e@10.31.138.179@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513100.016671:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690510897691456, offset 192 00000400:00000200:24.0:1616513100.016677:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.179@tcp1 00000400:00000200:24.0:1616513100.016683:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.179@tcp1 routed destination 00000400:00000200:24.0:1616513100.016697:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.179@tcp1(10.31.138.179@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000100:00000200:28.0:1616513100.016699:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690510897691648 00000800:00000200:24.0:1616513100.016707:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:28.0:1616513100.016711:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d569bb09680 x1690510897691648/t0(0) o400->68495f8b-de12-fc50-f722-742000ae3afc@10.31.138.179@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513100.016713:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.016715:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513100.016717:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513100.016722:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513100.016726:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690510897691648, offset 192 00000100:00000200:24.0:1616513100.016728:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513100.016728:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513100.016734:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.179@tcp1 00000400:00000200:28.0:1616513100.016741:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.179@tcp1 routed destination 00000400:00000200:28.0:1616513100.016756:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.179@tcp1(10.31.138.179@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513100.016763:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513100.016768:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513100.016771:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513100.016773:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513100.016778:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:20.0:1616513100.016780:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513100.016786:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513100.016787:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513100.016790:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513100.016791:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513100.016798:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513100.016803:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2330 00000800:00000200:20.0:1616513100.016806:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513100.016841:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513100.016847:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513100.016850:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513100.016856:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513100.016860:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcde48 00000800:00000200:26.0:1616513100.016864:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513100.016866:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513100.039452:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:10.0:1616513100.039467:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:10.0:1616513100.039475:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.216@o2ib1 00000400:00000200:10.0:1616513100.039484:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.125@tcp1 : PUT - for me 00000400:00000200:10.0:1616513100.039493:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.125@tcp1 of length 224 into portal 26 MB=0x6018311aad8c0 00000400:00000200:10.0:1616513100.039500:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.125@tcp1 of length 224/224 into md 0xf97d1539 [1] + 0 00000400:00000200:10.0:1616513100.039504:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789dc0 00000400:00000200:10.0:1616513100.039507:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:10.0:1616513100.039510:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:10.0:1616513100.039514:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789dc0 00000800:00000200:10.0:1616513100.039516:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:10.0:1616513100.039519:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:10.0:1616513100.039521:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513100.039523:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513100.039534:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513100.039542:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.125@tcp1 : PUT - for me 00000400:00000200:22.0:1616513100.039551:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.125@tcp1 of length 224 into portal 12 MB=0x6018311aad9c0 00000400:00000200:22.0:1616513100.039558:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.125@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 59248 00000400:00000200:22.0:1616513100.039563:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513100.039566:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:22.0:1616513100.039573:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513100.039578:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513100.039580:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513100.039593:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513100.039598:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512309016768 00000100:00000200:30.0:1616513100.039605:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690512309017024 00010000:00000200:24.0:1616513100.039609:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdf0cf050 x1690512309016768/t0(0) o400->bf34e61d-cb14-0439-419f-e61a5e690530@10.31.136.125@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513100.039615:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5df6a7b180 x1690512309017024/t0(0) o400->007e8ba6-cf59-57ec-df48-d26eedbd4be7@10.31.136.125@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513100.039623:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690512309016768, offset 192 00000100:00000200:30.0:1616513100.039627:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690512309017024, offset 192 00000400:00000200:24.0:1616513100.039630:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.125@tcp1 00000400:00000200:30.0:1616513100.039633:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.125@tcp1 00000400:00000200:24.0:1616513100.039637:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.125@tcp1 routed destination 00000400:00000200:30.0:1616513100.039639:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.125@tcp1 routed destination 00000400:00000200:30.0:1616513100.039652:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.125@tcp1(10.31.136.125@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513100.039652:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.125@tcp1(10.31.136.125@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513100.039658:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513100.039659:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513100.039663:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.039664:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513100.039665:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513100.039667:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513100.039667:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513100.039669:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513100.039671:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513100.039674:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:7.2:1616513100.039676:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513100.039681:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:18.0:1616513100.039731:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:11.2:1616513100.039735:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:18.0:1616513100.039739:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513100.039741:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513100.039746:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513100.039748:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513100.039752:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ee660 00000800:00000200:18.0:1616513100.039756:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:4.0:1616513100.039792:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513100.039799:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513100.039802:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513100.039808:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513100.039814:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2e58 00000800:00000200:4.0:1616513100.039817:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:4.0:1616513100.039820:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513100.046094:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513100.046107:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513100.046114:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513100.046122:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.135@tcp1 : PUT - for me 00000400:00000200:8.0:1616513100.046129:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.135@tcp1 of length 224 into portal 12 MB=0x602154b1c0100 00000400:00000200:8.0:1616513100.046135:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.135@tcp1 of length 224/224 into md 0x865be291 [1] + 90120 00000400:00000200:8.0:1616513100.046140:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513100.046142:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513100.046148:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513100.046150:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513100.046152:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513100.046154:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513100.046212:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691140337959168 00010000:00000200:28.0:1616513100.046224:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89680 x1691140337959168/t0(0) o400->610a4995-dc50-2c04-4012-7091cf7de8aa@10.31.140.135@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:7.2:1616513100.046232:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:28.0:1616513100.046237:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1691140337959168, offset 192 00000400:00000200:28.0:1616513100.046243:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.135@tcp1 00000400:00000200:28.0:1616513100.046250:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.135@tcp1 routed destination 00000400:00000200:28.0:1616513100.046270:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.135@tcp1(10.31.140.135@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513100.046276:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:28.0:1616513100.046281:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513100.046284:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513100.046286:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:28.0:1616513100.046290:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513100.046296:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:26.0:1616513100.046308:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513100.046317:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513100.046326:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.135@tcp1 : PUT - for me 00000400:00000200:26.0:1616513100.046334:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.135@tcp1 of length 224 into portal 26 MB=0x602154b1bfe40 00000400:00000200:26.0:1616513100.046341:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.135@tcp1 of length 224/224 into md 0xf97d1549 [1] + 0 00000400:00000200:26.0:1616513100.046346:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789f58 00000800:00000200:4.0:1616513100.046348:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:26.0:1616513100.046349:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513100.046351:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513100.046355:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789f58 00000800:00000200:4.0:1616513100.046355:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:26.0:1616513100.046358:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000400:00000200:4.0:1616513100.046358:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513100.046361:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:0.0:1616513100.046361:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513100.046363:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:4.0:1616513100.046365:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:26.0:1616513100.046366:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:4.0:1616513100.046369:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcded0 00000800:00000200:4.0:1616513100.046373:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513100.046419:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1691140337958464 00010000:00000200:24.0:1616513100.046430:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e73850 x1691140337958464/t0(0) o400->7991c4c1-ff80-8394-c96b-24487abd10d0@10.31.140.135@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513100.046455:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1691140337958464, offset 192 00000400:00000200:24.0:1616513100.046461:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.135@tcp1 00000400:00000200:24.0:1616513100.046469:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.135@tcp1 routed destination 00000400:00000200:24.0:1616513100.046484:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.135@tcp1(10.31.140.135@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513100.046490:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513100.046496:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.046499:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513100.046503:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513100.046508:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513100.046512:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513100.046515:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:14.0:1616513100.046522:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513100.046527:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513100.046530:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513100.046536:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513100.046542:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3dc0 00000800:00000200:14.0:1616513100.046545:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513100.046548:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513100.058469:0:6499:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930144, portal 4 00000100:00000200:30.0:1616513100.058473:0:6499:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930144, offset 0 00000400:00000200:30.0:1616513100.058478:0:6499:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:30.0:1616513100.058483:0:6499:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:30.0:1616513100.058494:0:6499:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513100.058500:0:6499:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:30.0:1616513100.058506:0:6499:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513100.058509:0:6499:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:30.0:1616513100.058510:0:6499:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:30.0:1616513100.058515:0:6499:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.2:1616513100.058563:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513100.058582:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:27.0:1616513100.058589:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513100.058592:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:27.0:1616513100.058599:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513100.058606:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9643600 x1693067725930144/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:27.0:1616513100.058621:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92ef760 00000800:00000200:27.0:1616513100.058626:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513100.058629:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000800:00000200:27.2:1616513100.058732:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513100.058743:0:6476:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513100.058750:0:6476:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513100.058753:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513100.058758:0:6476:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513100.058766:0:6476:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627ea0 00000400:00000200:27.0:1616513100.058773:0:6476:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371677cd [1] + 192 00000400:00000200:27.0:1616513100.058779:0:6476:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513100.058783:0:6476:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9643600 x1693067725930144/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513100.058798:0:6476:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000100:00000200:30.0:1616513100.058810:0:6499:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9643600 x1693067725930144/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513100.058811:0:6476:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:27.0:1616513100.058813:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:30.0:1616513100.058821:0:6499:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9643600 x1693067725930144/t0(0) o13->boslfs02-OST002e-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:30.0:1616513100.058830:0:6499:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efba0 00000800:00000200:11.2:1616513100.083970:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513100.083984:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513100.083994:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513100.084003:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.14@tcp1 : PUT - for me 00000400:00000200:8.0:1616513100.084013:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.14@tcp1 of length 224 into portal 26 MB=0x603d0aa5cee80 00000400:00000200:8.0:1616513100.084020:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.14@tcp1 of length 224/224 into md 0xf97d1559 [1] + 0 00000400:00000200:8.0:1616513100.084025:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788ff0 00000400:00000200:8.0:1616513100.084028:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513100.084031:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513100.084036:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788ff0 00000800:00000200:8.0:1616513100.084039:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513100.084040:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513100.084042:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513100.084047:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513100.084050:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513100.084051:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513100.084060:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.14@tcp1 : PUT - for me 00000400:00000200:20.0:1616513100.084070:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.14@tcp1 of length 224 into portal 12 MB=0x603d0aa5cef00 00000400:00000200:20.0:1616513100.084076:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.14@tcp1 of length 224/224 into md 0x865be291 [1] + 90344 00000400:00000200:20.0:1616513100.084082:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513100.084084:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:20.0:1616513100.084091:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513100.084095:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513100.084097:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513100.084104:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693044606561920 00010000:00000200:24.0:1616513100.084117:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bb050 x1693044606561920/t0(0) o400->6f43b0eb-679e-2185-f66d-20a83e8fe2ca@10.31.142.14@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513100.084132:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1693044606561920, offset 192 00000400:00000200:24.0:1616513100.084138:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.14@tcp1 00000400:00000200:24.0:1616513100.084146:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.14@tcp1 routed destination 00000100:00000200:28.0:1616513100.084157:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1693044606562048 00000400:00000200:24.0:1616513100.084162:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.14@tcp1(10.31.142.14@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513100.084169:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00010000:00000200:28.0:1616513100.084171:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d575c667980 x1693044606562048/t0(0) o400->a01d5d97-7d0c-4d9a-a0f6-ce2db3802e62@10.31.142.14@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513100.084175:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.084177:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513100.084179:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513100.084185:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:28.0:1616513100.084186:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1693044606562048, offset 192 00000800:00000200:11.2:1616513100.084190:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513100.084192:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.14@tcp1 00000100:00000200:24.0:1616513100.084193:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513100.084200:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.14@tcp1 routed destination 00000400:00000200:28.0:1616513100.084215:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.14@tcp1(10.31.142.14@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513100.084222:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513100.084228:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513100.084233:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513100.084235:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513100.084241:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:4.0:1616513100.084243:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513100.084248:0:51600:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:4.0:1616513100.084250:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513100.084252:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513100.084253:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:4.0:1616513100.084259:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513100.084265:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3cb0 00000800:00000200:4.0:1616513100.084268:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513100.084304:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513100.084312:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513100.084315:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513100.084321:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513100.084326:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcddc0 00000800:00000200:22.0:1616513100.084329:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513100.084333:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:10.0:1616513100.090523:0:6500:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930160, portal 4 00000100:00000200:10.0:1616513100.090529:0:6500:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930160, offset 0 00000400:00000200:10.0:1616513100.090536:0:6500:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:10.0:1616513100.090541:0:6500:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:10.0:1616513100.090554:0:6500:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:10.0:1616513100.090561:0:6500:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:10.0:1616513100.090566:0:6500:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:10.0:1616513100.090569:0:6500:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:10.0:1616513100.090572:0:6500:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:10.0:1616513100.090577:0:6500:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513100.090623:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:14.0:1616513100.090681:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:14.0:1616513100.090689:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:14.0:1616513100.090691:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513100.090698:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:14.0:1616513100.090704:0:6469:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9643600 x1693067725930160/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:14.0:1616513100.090716:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e82c38 00000800:00000200:14.0:1616513100.090720:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:14.0:1616513100.090723:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513100.090797:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513100.090852:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513100.090862:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000800:00000200:26.0:1616513100.090865:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000400:00000200:22.0:1616513100.090871:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513100.090879:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627eb0 00000400:00000200:22.0:1616513100.090886:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d1c79 [1] + 192 00000400:00000200:22.0:1616513100.090891:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513100.090896:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9643600 x1693067725930160/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:22.0:1616513100.090910:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513100.090913:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513100.090915:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:10.0:1616513100.090917:0:6500:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9643600 x1693067725930160/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:10.0:1616513100.090931:0:6500:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9643600 x1693067725930160/t0(0) o13->boslfs02-OST0034-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:10.0:1616513100.090941:0:6500:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d4553e83ba0 00000800:00000200:7.2:1616513100.108669:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513100.108726:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513100.108735:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513100.108745:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.94@tcp1 : PUT - for me 00000400:00000200:14.0:1616513100.108754:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.94@tcp1 of length 224 into portal 26 MB=0x60194e4da4d00 00000400:00000200:14.0:1616513100.108761:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.94@tcp1 of length 224/224 into md 0xf97d1569 [1] + 0 00000400:00000200:14.0:1616513100.108766:0:6469:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789100 00000400:00000200:14.0:1616513100.108768:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513100.108771:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:14.0:1616513100.108776:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789100 00000800:00000200:14.0:1616513100.108779:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513100.108780:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513100.108782:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513100.108787:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513100.108790:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513100.108791:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513100.108800:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.94@tcp1 : PUT - for me 00000400:00000200:18.0:1616513100.108809:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.94@tcp1 of length 224 into portal 12 MB=0x60194e4da4f00 00000400:00000200:18.0:1616513100.108816:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.94@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 59472 00000400:00000200:18.0:1616513100.108821:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513100.108824:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513100.108831:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513100.108834:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513100.108836:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513100.108842:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588866563328 00000100:00000200:30.0:1616513100.108852:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690588866563840 00010000:00000200:24.0:1616513100.108853:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2bc850 x1690588866563328/t0(0) o400->08aeb1e7-7bea-0ccf-acc8-fbe3b4710c5c@10.31.140.94@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513100.108864:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bc380 x1690588866563840/t0(0) o400->893df218-5f71-a998-8684-819fcf103fb9@10.31.140.94@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513100.108868:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690588866563328, offset 192 00000400:00000200:24.0:1616513100.108874:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.94@tcp1 00000100:00000200:30.0:1616513100.108878:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690588866563840, offset 192 00000400:00000200:24.0:1616513100.108881:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.94@tcp1 routed destination 00000400:00000200:30.0:1616513100.108885:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.94@tcp1 00000400:00000200:30.0:1616513100.108892:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.94@tcp1 routed destination 00000400:00000200:24.0:1616513100.108897:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.94@tcp1(10.31.140.94@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513100.108904:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:30.0:1616513100.108907:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.94@tcp1(10.31.140.94@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513100.108910:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.108913:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513100.108914:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513100.108915:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513100.108919:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.108920:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513100.108922:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513100.108924:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:11.2:1616513100.108926:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513100.108927:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513100.108929:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513100.108934:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513100.108982:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513100.108989:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513100.108992:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513100.108999:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000800:00000200:22.0:1616513100.109004:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000400:00000200:20.0:1616513100.109004:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2d48 00000800:00000200:20.0:1616513100.109007:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513100.109010:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513100.109011:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513100.109014:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:14.0:1616513100.109015:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513100.109021:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513100.109025:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efba0 00000800:00000200:22.0:1616513100.109029:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513100.122297:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513100.122309:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513100.122318:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513100.122327:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.36@tcp1 : PUT - for me 00000400:00000200:8.0:1616513100.122337:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.36@tcp1 of length 224 into portal 26 MB=0x5ffdce5a1f8c0 00000400:00000200:8.0:1616513100.122344:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.36@tcp1 of length 224/224 into md 0xf97d1579 [1] + 0 00000400:00000200:8.0:1616513100.122349:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d7894b8 00000400:00000200:8.0:1616513100.122352:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513100.122355:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513100.122359:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7894b8 00000800:00000200:8.0:1616513100.122362:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513100.122365:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:4.0:1616513100.122366:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513100.122370:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513100.122424:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699094038720 00010000:00000200:24.0:1616513100.122449:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02fc050 x1688699094038720/t0(0) o400->21befdc4-b491-e9fc-9068-228e4d260e50@10.31.140.36@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513100.122465:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688699094038720, offset 192 00000400:00000200:24.0:1616513100.122472:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.36@tcp1 00000400:00000200:24.0:1616513100.122479:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.36@tcp1 routed destination 00000400:00000200:24.0:1616513100.122494:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.36@tcp1(10.31.140.36@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513100.122501:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513100.122506:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.122509:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513100.122511:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513100.122516:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513100.122521:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513100.122523:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:7.2:1616513100.122548:0:51604:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:20.0:1616513100.122575:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513100.122583:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513100.122585:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513100.122592:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513100.122598:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2220 00000800:00000200:20.0:1616513100.122601:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513100.122604:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513100.122606:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513100.122616:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513100.122634:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.36@tcp1 : PUT - for me 00000400:00000200:18.0:1616513100.122642:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.36@tcp1 of length 224 into portal 12 MB=0x5ffdce5a1f940 00000400:00000200:18.0:1616513100.122649:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.36@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 59696 00000400:00000200:18.0:1616513100.122656:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513100.122659:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:26.0:1616513100.122666:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513100.122666:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513100.122669:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513100.122671:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513100.122684:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688699094038848 00010000:00000200:30.0:1616513100.122694:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645b9680 x1688699094038848/t0(0) o400->c09e4994-4089-bc2e-c832-311584cbad69@10.31.140.36@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513100.122708:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688699094038848, offset 192 00000400:00000200:30.0:1616513100.122714:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.36@tcp1 00000400:00000200:30.0:1616513100.122721:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.36@tcp1 routed destination 00000400:00000200:30.0:1616513100.122734:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.36@tcp1(10.31.140.36@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513100.122741:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513100.122746:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513100.122750:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513100.122752:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513100.122756:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513100.122760:0:51604:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513100.122813:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513100.122820:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513100.122822:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:14.0:1616513100.122831:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513100.122834:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eebb0 00000800:00000200:14.0:1616513100.122838:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513100.122841:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513100.129554:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513100.129609:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513100.129618:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513100.129627:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.178@tcp1 : PUT - for me 00000400:00000200:26.0:1616513100.129639:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.178@tcp1 of length 224 into portal 26 MB=0x6033c0e8f77c0 00000400:00000200:26.0:1616513100.129646:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.136.178@tcp1 of length 224/224 into md 0xf97d1589 [1] + 0 00000400:00000200:26.0:1616513100.129650:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788990 00000400:00000200:26.0:1616513100.129653:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513100.129656:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513100.129660:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788990 00000800:00000200:18.0:1616513100.129662:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513100.129663:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513100.129666:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513100.129667:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513100.129723:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692406337468352 00010000:00000200:24.0:1616513100.129734:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e71850 x1692406337468352/t0(0) o400->3e6e2986-43d8-56bb-6f66-09a529392c2f@10.31.136.178@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513100.129748:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1692406337468352, offset 192 00000400:00000200:24.0:1616513100.129754:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.178@tcp1 00000400:00000200:24.0:1616513100.129762:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.178@tcp1 routed destination 00000400:00000200:24.0:1616513100.129777:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.178@tcp1(10.31.136.178@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513100.129784:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513100.129789:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.129792:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513100.129794:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513100.129799:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000100:00000200:24.0:1616513100.129805:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513100.129807:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:4.0:1616513100.129862:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513100.129869:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513100.129872:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:8.0:1616513100.129873:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513100.129879:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513100.129884:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2aa0 00000800:00000200:4.0:1616513100.129888:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513100.130300:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513100.130353:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513100.130362:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513100.130371:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.136.178@tcp1 : PUT - for me 00000400:00000200:14.0:1616513100.130380:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.136.178@tcp1 of length 224 into portal 12 MB=0x6033c0e8f7900 00000400:00000200:14.0:1616513100.130386:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.136.178@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 59920 00000400:00000200:14.0:1616513100.130392:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513100.130394:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513100.130401:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513100.130404:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513100.130406:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513100.130409:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513100.130417:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1692406337468672 00010000:00000200:30.0:1616513100.130426:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebf080 x1692406337468672/t0(0) o400->0d905ba0-d215-e6f2-b5d4-5e8140e9abd8@10.31.136.178@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513100.130444:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1692406337468672, offset 192 00000400:00000200:30.0:1616513100.130450:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.136.178@tcp1 00000400:00000200:30.0:1616513100.130456:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.136.178@tcp1 routed destination 00000400:00000200:30.0:1616513100.130469:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.136.178@tcp1(10.31.136.178@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513100.130475:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513100.130480:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513100.130482:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513100.130484:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513100.130488:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513100.130494:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513100.130505:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513100.130510:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513100.130513:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513100.130515:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:18.0:1616513100.130520:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513100.130524:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efed0 00000800:00000200:18.0:1616513100.130528:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513100.138281:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:14.0:1616513100.138340:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513100.138349:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513100.138358:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.57@tcp1 : PUT - for me 00000400:00000200:14.0:1616513100.138367:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.57@tcp1 of length 224 into portal 12 MB=0x5ffd59d0f4c40 00000400:00000200:14.0:1616513100.138374:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.129.57@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 60144 00000400:00000200:14.0:1616513100.138379:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513100.138382:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513100.138388:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513100.138391:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:22.0:1616513100.138393:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513100.138397:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:30.0:1616513100.138401:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667811695680 00010000:00000200:30.0:1616513100.138409:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebbf00 x1688667811695680/t0(0) o400->e9284cfc-b778-8512-3b05-e63f91ce44d1@10.31.129.57@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513100.138421:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688667811695680, offset 192 00000400:00000200:30.0:1616513100.138426:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.57@tcp1 00000400:00000200:30.0:1616513100.138432:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.57@tcp1 routed destination 00000400:00000200:30.0:1616513100.138451:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.57@tcp1(10.31.129.57@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513100.138457:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513100.138461:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513100.138463:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513100.138465:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513100.138469:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513100.138476:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513100.138530:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513100.138537:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513100.138540:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513100.138546:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:20.0:1616513100.138547:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513100.138551:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeaa0 00000800:00000200:11.2:1616513100.138553:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513100.138556:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513100.138559:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513100.138566:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513100.138567:0:6461:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:20.0:1616513100.138574:0:6461:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.129.57@tcp1 : PUT - for me 00000400:00000200:20.0:1616513100.138583:0:6461:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.129.57@tcp1 of length 224 into portal 26 MB=0x5ffd59d0f4ac0 00000400:00000200:20.0:1616513100.138589:0:6461:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.129.57@tcp1 of length 224/224 into md 0xf97d15b9 [1] + 0 00000400:00000200:20.0:1616513100.138594:0:6461:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d7893a8 00000400:00000200:20.0:1616513100.138597:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:20.0:1616513100.138599:0:6461:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:20.0:1616513100.138603:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7893a8 00000800:00000200:20.0:1616513100.138606:0:6461:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:20.0:1616513100.138608:0:6461:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513100.138610:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513100.138668:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688667811695296 00010000:00000200:24.0:1616513100.138679:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5ce02ff850 x1688667811695296/t0(0) o400->134dcdf4-e4ee-9423-23eb-ca3172b8cd5b@10.31.129.57@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513100.138693:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688667811695296, offset 192 00000400:00000200:24.0:1616513100.138699:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.129.57@tcp1 00000400:00000200:24.0:1616513100.138707:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.129.57@tcp1 routed destination 00000400:00000200:24.0:1616513100.138724:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.129.57@tcp1(10.31.129.57@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513100.138730:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513100.138736:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.138739:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513100.138741:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513100.138745:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513100.138750:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513100.138752:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:26.0:1616513100.138805:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513100.138812:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513100.138815:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513100.138821:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513100.138827:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2f68 00000800:00000200:26.0:1616513100.138831:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513100.138834:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513100.146626:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:11.2:1616513100.146655:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:22.0:1616513100.146682:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513100.146691:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:22.0:1616513100.146700:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.203@tcp1 : PUT - for me 00000400:00000200:22.0:1616513100.146710:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.203@tcp1 of length 224 into portal 26 MB=0x5ffe0c52f23c0 00000800:00000200:0.0:1616513100.146714:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:22.0:1616513100.146716:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.142.203@tcp1 of length 224/224 into md 0xf97d15c9 [1] + 0 00000400:00000200:22.0:1616513100.146724:0:6472:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788dd0 00000400:00000200:22.0:1616513100.146726:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513100.146729:0:6472:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:22.0:1616513100.146734:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788dd0 00000800:00000200:22.0:1616513100.146737:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513100.146739:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:8.0:1616513100.146739:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513100.146739:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:22.0:1616513100.146741:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513100.146744:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000400:00000200:0.0:1616513100.146747:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.142.203@tcp1 : PUT - for me 00000400:00000200:0.0:1616513100.146756:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.142.203@tcp1 of length 224 into portal 12 MB=0x5ffe0c52f26c0 00000400:00000200:0.0:1616513100.146762:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.142.203@tcp1 of length 224/224 into md 0x865be291 [1] + 90568 00000400:00000200:0.0:1616513100.146766:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513100.146769:0:6463:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:0.0:1616513100.146775:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:0.0:1616513100.146778:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513100.146780:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:24.0:1616513100.146798:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688715729511360 00010000:00000200:24.0:1616513100.146808:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7dc050 x1688715729511360/t0(0) o400->fabf1fe6-0d44-ff49-f669-5bd7f2ca2b33@10.31.142.203@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00000100:00000200:24.0:1616513100.146821:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688715729511360, offset 192 00000400:00000200:24.0:1616513100.146826:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.203@tcp1 00000400:00000200:24.0:1616513100.146833:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.203@tcp1 routed destination 00000100:00000200:28.0:1616513100.146841:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688715729512128 00000400:00000200:24.0:1616513100.146847:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.203@tcp1(10.31.142.203@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00010000:00000200:28.0:1616513100.146853:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d569bb09b00 x1688715729512128/t0(0) o400->43ec5197-f756-9ec8-e588-ae5b5e33a231@10.31.142.203@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000800:00000200:24.0:1616513100.146854:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513100.146859:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.146861:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513100.146863:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:28.0:1616513100.146867:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688715729512128, offset 192 00000800:00000200:24.0:1616513100.146867:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513100.146872:0:5839:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:28.0:1616513100.146873:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.142.203@tcp1 00000100:00000200:24.0:1616513100.146874:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000400:00000200:28.0:1616513100.146881:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.142.203@tcp1 routed destination 00000400:00000200:28.0:1616513100.146896:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.142.203@tcp1(10.31.142.203@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513100.146902:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513100.146908:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513100.146911:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513100.146913:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513100.146918:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513100.146922:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:20.0:1616513100.146925:0:6461:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:20.0:1616513100.146932:0:6461:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:20.0:1616513100.146934:0:6461:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:20.0:1616513100.146941:0:6461:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:20.0:1616513100.146947:0:6461:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2660 00000800:00000200:20.0:1616513100.146950:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513100.146953:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:26.0:1616513100.146975:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513100.146981:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:26.0:1616513100.146984:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513100.146991:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:26.0:1616513100.146995:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdcb0 00000800:00000200:26.0:1616513100.146999:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:26.0:1616513100.147002:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513100.148214:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513100.148223:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513100.148229:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513100.148236:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.231@tcp1 : PUT - for me 00000400:00000200:8.0:1616513100.148245:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.231@tcp1 of length 224 into portal 12 MB=0x5ffed998b4800 00000400:00000200:8.0:1616513100.148250:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.140.231@tcp1 of length 224/224 into md 0x865be291 [1] + 90792 00000400:00000200:8.0:1616513100.148255:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513100.148257:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:8.0:1616513100.148263:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513100.148265:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513100.148267:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513100.148269:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513100.148274:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000800:00000200:0.0:1616513100.148274:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513100.148280:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.140.231@tcp1 : PUT - for me 00000400:00000200:8.0:1616513100.148285:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.140.231@tcp1 of length 224 into portal 26 MB=0x5ffed998b4740 00000400:00000200:8.0:1616513100.148290:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.140.231@tcp1 of length 224/224 into md 0xf97d15e9 [1] + 0 00000400:00000200:8.0:1616513100.148293:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d7887f8 00000400:00000200:8.0:1616513100.148295:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513100.148297:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513100.148300:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7887f8 00000800:00000200:8.0:1616513100.148302:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513100.148304:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513100.148305:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513100.148337:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688770831927296 00010000:00000200:28.0:1616513100.148349:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5cdeb89f80 x1688770831927296/t0(0) o400->c1b766dd-8309-c1d8-40d1-67e1f397db77@10.31.140.231@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513100.148363:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688770831927104 00000100:00000200:28.0:1616513100.148364:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688770831927296, offset 192 00000400:00000200:28.0:1616513100.148370:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.231@tcp1 00010000:00000200:24.0:1616513100.148373:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c7d8850 x1688770831927104/t0(0) o400->a2233e78-aeee-cdfe-4591-7548970fdb8b@10.31.140.231@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00000400:00000200:28.0:1616513100.148378:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.231@tcp1 routed destination 00000100:00000200:24.0:1616513100.148387:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688770831927104, offset 192 00000400:00000200:28.0:1616513100.148392:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.231@tcp1(10.31.140.231@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:24.0:1616513100.148393:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.140.231@tcp1 00000800:00000200:28.0:1616513100.148399:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:24.0:1616513100.148401:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.140.231@tcp1 routed destination 00000800:00000200:28.0:1616513100.148408:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513100.148411:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:28.0:1616513100.148413:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:24.0:1616513100.148415:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.140.231@tcp1(10.31.140.231@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513100.148418:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:24.0:1616513100.148421:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:11.2:1616513100.148423:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513100.148426:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.148430:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:24.0:1616513100.148432:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513100.148449:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513100.148454:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513100.148456:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:4.0:1616513100.148476:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:4.0:1616513100.148483:0:6464:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:4.0:1616513100.148485:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:0.0:1616513100.148486:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:4.0:1616513100.148492:0:6464:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:4.0:1616513100.148496:0:6464:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcdba0 00000800:00000200:4.0:1616513100.148500:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513100.148509:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513100.148516:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513100.148519:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:22.0:1616513100.148525:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513100.148531:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2aa0 00000800:00000200:22.0:1616513100.148534:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:22.0:1616513100.148537:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:26.0:1616513100.154515:0:6501:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930176, portal 4 00000100:00000200:26.0:1616513100.154521:0:6501:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930176, offset 0 00000400:00000200:26.0:1616513100.154528:0:6501:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.229@o2ib1 00000400:00000200:26.0:1616513100.154533:0:6501:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.229@o2ib1 local destination 00000400:00000200:26.0:1616513100.154545:0:6501:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.229@o2ib1(10.242.46.229@o2ib1:10.242.46.229@o2ib1) : PUT try# 0 00000800:00000200:26.0:1616513100.154552:0:6501:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.229@o2ib1 00000800:00000200:26.0:1616513100.154558:0:6501:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9f80] -> 10.242.46.229@o2ib1 (2) version: 12 00000800:00000200:26.0:1616513100.154561:0:6501:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da650200] (20)++ 00000800:00000200:26.0:1616513100.154565:0:6501:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513100.154570:0:6501:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:25.2:1616513100.154615:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (21)++ 00000800:00000200:26.0:1616513100.154632:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (22)++ 00000800:00000200:26.0:1616513100.154636:0:6470:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da650200] (23)-- 00000400:00000200:26.0:1616513100.154638:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:26.0:1616513100.154644:0:6470:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.229@o2ib1: PUT: OK 00000100:00000200:26.0:1616513100.154649:0:6470:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d74d9642400 x1693067725930176/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513100.154660:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697af100 00000800:00000200:26.0:1616513100.154663:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:26.0:1616513100.154666:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000800:00000200:25.2:1616513100.154809:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da650200] (20)++ 00000800:00000200:22.0:1616513100.154864:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da650200] (21)++ 00000800:00000200:22.0:1616513100.154874:0:6472:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.229@o2ib1 00000400:00000200:22.0:1616513100.154882:0:6472:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.229@o2ib1 : PUT - for me 00000400:00000200:22.0:1616513100.154890:0:6472:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.229@o2ib1 of length 336 into portal 4 MB=0x603d60c627ec0 00000400:00000200:22.0:1616513100.154897:0:6472:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.229@o2ib1 of length 336/336 into md 0xf97d1d39 [1] + 192 00000400:00000200:22.0:1616513100.154902:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:22.0:1616513100.154907:0:6472:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d74d9642400 x1693067725930176/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:14.0:1616513100.154931:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513100.154934:0:6472:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da650200] (22)++ 00000800:00000200:22.0:1616513100.154936:0:6472:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da650200] (22)-- 00000800:00000200:22.0:1616513100.154938:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da650200] (21)-- 00000100:00000200:26.0:1616513100.154939:0:6501:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d74d9642400 x1693067725930176/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:26.0:1616513100.154950:0:6501:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d74d9642400 x1693067725930176/t0(0) o13->boslfs02-OST0008-osc-MDT0001@10.242.46.229@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:26.0:1616513100.154958:0:6501:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d84697aeff0 00000800:00000200:7.2:1616513100.163260:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:26.0:1616513100.163315:0:6470:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:26.0:1616513100.163323:0:6470:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[2] from 10.242.46.216@o2ib1 00000400:00000200:26.0:1616513100.163331:0:6470:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.39@tcp1 : PUT - for me 00000400:00000200:26.0:1616513100.163341:0:6470:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.39@tcp1 of length 224 into portal 26 MB=0x5ffd51b023f00 00000400:00000200:26.0:1616513100.163347:0:6470:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.138.39@tcp1 of length 224/224 into md 0xf97d1619 [1] + 0 00000400:00000200:26.0:1616513100.163351:0:6470:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d789188 00000400:00000200:26.0:1616513100.163354:0:6470:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:26.0:1616513100.163356:0:6470:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:26.0:1616513100.163361:0:6470:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d789188 00000800:00000200:26.0:1616513100.163363:0:6470:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:26.0:1616513100.163366:0:6470:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513100.163368:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (23)++ 00000800:00000200:26.0:1616513100.163375:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:26.0:1616513100.163377:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513100.163378:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513100.163387:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.138.39@tcp1 : PUT - for me 00000400:00000200:18.0:1616513100.163396:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.138.39@tcp1 of length 224 into portal 12 MB=0x5ffd51b023f80 00000400:00000200:18.0:1616513100.163402:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.138.39@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 60368 00000400:00000200:18.0:1616513100.163408:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513100.163410:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:18.0:1616513100.163417:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513100.163421:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513100.163422:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513100.163424:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665629802240 00000100:00000200:30.0:1616513100.163445:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665629802368 00010000:00000200:24.0:1616513100.163446:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d747c6b7850 x1688665629802240/t0(0) o400->30fd0063-d3a7-1703-6b51-6a86f8d7639b@10.31.138.39@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513100.163456:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74645bda00 x1688665629802368/t0(0) o400->e2986215-a4b8-31a4-30e5-da6902e8c7d3@10.31.138.39@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513100.163460:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665629802240, offset 192 00000400:00000200:24.0:1616513100.163465:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.39@tcp1 00000400:00000200:24.0:1616513100.163473:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.39@tcp1 routed destination 00000100:00000200:30.0:1616513100.163478:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665629802368, offset 192 00000400:00000200:30.0:1616513100.163483:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.138.39@tcp1 00000400:00000200:24.0:1616513100.163486:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.39@tcp1(10.31.138.39@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513100.163490:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.138.39@tcp1 routed destination 00000800:00000200:24.0:1616513100.163493:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:24.0:1616513100.163498:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.163501:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513100.163502:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000400:00000200:30.0:1616513100.163503:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.138.39@tcp1(10.31.138.39@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513100.163507:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513100.163509:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000100:00000200:24.0:1616513100.163513:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:30.0:1616513100.163514:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:11.2:1616513100.163515:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513100.163517:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513100.163519:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513100.163523:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:8.0:1616513100.163526:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513100.163528:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:8.0:1616513100.163530:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513100.163532:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:8.0:1616513100.163538:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513100.163542:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2d48 00000800:00000200:8.0:1616513100.163545:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513100.163547:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:22.0:1616513100.163601:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513100.163608:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513100.163610:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513100.163611:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513100.163616:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:22.0:1616513100.163620:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efa08 00000800:00000200:22.0:1616513100.163624:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:7.2:1616513100.179044:0:51610:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000800:00000200:18.0:1616513100.179101:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000800:00000200:18.0:1616513100.179111:0:6471:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:18.0:1616513100.179120:0:6471:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.88@tcp1 : PUT - for me 00000400:00000200:18.0:1616513100.179129:0:6471:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.88@tcp1 of length 224 into portal 26 MB=0x5ffdf1e7f8f00 00000400:00000200:18.0:1616513100.179136:0:6471:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.141.88@tcp1 of length 224/224 into md 0xf97d1639 [1] + 0 00000400:00000200:18.0:1616513100.179141:0:6471:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d7882a8 00000400:00000200:18.0:1616513100.179144:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:18.0:1616513100.179147:0:6471:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:18.0:1616513100.179151:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d7882a8 00000800:00000200:18.0:1616513100.179154:0:6471:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513100.179155:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513100.179159:0:6471:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (24)-- 00000800:00000200:18.0:1616513100.179163:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (23)-- 00000800:00000200:18.0:1616513100.179165:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513100.179166:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.216@o2ib1 00000400:00000200:14.0:1616513100.179175:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.141.88@tcp1 : PUT - for me 00000400:00000200:14.0:1616513100.179184:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.141.88@tcp1 of length 224 into portal 12 MB=0x5ffdf1e7f91c0 00000400:00000200:14.0:1616513100.179190:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.141.88@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 60592 00000400:00000200:14.0:1616513100.179196:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513100.179200:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:14.0:1616513100.179207:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513100.179211:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:14.0:1616513100.179213:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:24.0:1616513100.179216:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708638019328 00000100:00000200:30.0:1616513100.179225:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688708638020032 00010000:00000200:24.0:1616513100.179227:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d74603a1850 x1688708638019328/t0(0) o400->d2766df1-1e17-8d19-7974-29c836ea7bfb@10.31.141.88@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00010000:00000200:30.0:1616513100.179235:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebb180 x1688708638020032/t0(0) o400->3c996321-e97d-c96e-30a7-b108427b2bea@10.31.141.88@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:24.0:1616513100.179241:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688708638019328, offset 192 00000400:00000200:24.0:1616513100.179247:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.88@tcp1 00000100:00000200:30.0:1616513100.179249:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688708638020032, offset 192 00000400:00000200:30.0:1616513100.179255:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.141.88@tcp1 00000400:00000200:24.0:1616513100.179255:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.88@tcp1 routed destination 00000400:00000200:30.0:1616513100.179261:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.141.88@tcp1 routed destination 00000400:00000200:24.0:1616513100.179270:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.88@tcp1(10.31.141.88@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:30.0:1616513100.179276:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.141.88@tcp1(10.31.141.88@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:24.0:1616513100.179277:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:30.0:1616513100.179282:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:24.0:1616513100.179283:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.179286:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:30.0:1616513100.179287:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.179288:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:30.0:1616513100.179290:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513100.179292:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:24.0:1616513100.179293:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:30.0:1616513100.179296:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000100:00000200:24.0:1616513100.179300:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:11.2:1616513100.179301:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:7.2:1616513100.179301:0:51610:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:22.0:1616513100.179356:0:6472:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:22.0:1616513100.179365:0:6472:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:22.0:1616513100.179368:0:6472:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:18.0:1616513100.179369:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:22.0:1616513100.179374:0:6472:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000800:00000200:0.0:1616513100.179376:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000400:00000200:22.0:1616513100.179379:0:6472:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92efa08 00000800:00000200:22.0:1616513100.179383:0:6472:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:0.0:1616513100.179383:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513100.179388:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513100.179395:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513100.179413:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3dc0 00000800:00000200:0.0:1616513100.179416:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513100.179419:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513100.180886:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513100.180896:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513100.180902:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[3] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513100.180910:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.95@tcp1 : PUT - for me 00000400:00000200:8.0:1616513100.180918:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.95@tcp1 of length 224 into portal 26 MB=0x5ffd50b4e6d40 00000400:00000200:8.0:1616513100.180924:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.131.95@tcp1 of length 224/224 into md 0xf97d1649 [1] + 0 00000400:00000200:8.0:1616513100.180928:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788550 00000400:00000200:8.0:1616513100.180930:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513100.180932:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513100.180936:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788550 00000800:00000200:8.0:1616513100.180938:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513100.180941:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513100.180942:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:8.0:1616513100.180944:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:7.2:1616513100.180969:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (20)++ 00000100:00000200:24.0:1616513100.181002:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665366359360 00010000:00000200:24.0:1616513100.181013:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d465b2b9050 x1688665366359360/t0(0) o400->e20c45e7-260d-64eb-d06f-690f4726a226@10.31.131.95@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:14.0:1616513100.181025:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (21)++ 00000100:00000200:24.0:1616513100.181028:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1688665366359360, offset 192 00000400:00000200:24.0:1616513100.181034:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.95@tcp1 00000800:00000200:14.0:1616513100.181034:0:6469:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.216@o2ib1 00000400:00000200:24.0:1616513100.181042:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.95@tcp1 routed destination 00000400:00000200:14.0:1616513100.181044:0:6469:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.131.95@tcp1 : PUT - for me 00000400:00000200:14.0:1616513100.181052:0:6469:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.131.95@tcp1 of length 224 into portal 12 MB=0x5ffd50b4e6f00 00000400:00000200:24.0:1616513100.181057:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.95@tcp1(10.31.131.95@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000400:00000200:14.0:1616513100.181058:0:6469:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.131.95@tcp1 of length 224/224 into md 0xf9654ef9 [1] + 60816 00000800:00000200:24.0:1616513100.181064:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000400:00000200:14.0:1616513100.181064:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:14.0:1616513100.181066:0:6469:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000800:00000200:24.0:1616513100.181070:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:24.0:1616513100.181073:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:14.0:1616513100.181073:0:6469:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d74da651200] (22)++ 00000800:00000200:24.0:1616513100.181075:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:14.0:1616513100.181076:0:6469:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d74da651200] (23)-- 00000800:00000200:14.0:1616513100.181078:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:24.0:1616513100.181080:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:26.0:1616513100.181081:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000800:00000200:11.2:1616513100.181085:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513100.181086:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000100:00000200:30.0:1616513100.181088:0:170086:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1688665366359808 00010000:00000200:30.0:1616513100.181097:0:170086:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5dc4ebd580 x1688665366359808/t0(0) o400->2d32e117-9141-229c-801f-ac74f3a5075d@10.31.131.95@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:30.0:1616513100.181109:0:170086:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1688665366359808, offset 192 00000400:00000200:30.0:1616513100.181114:0:170086:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.131.95@tcp1 00000400:00000200:30.0:1616513100.181120:0:170086:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.131.95@tcp1 routed destination 00000400:00000200:30.0:1616513100.181133:0:170086:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.131.95@tcp1(10.31.131.95@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:30.0:1616513100.181139:0:170086:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:30.0:1616513100.181144:0:170086:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:30.0:1616513100.181146:0:170086:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:30.0:1616513100.181148:0:170086:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:30.0:1616513100.181152:0:170086:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:0.0:1616513100.181155:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:7.2:1616513100.181157:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:0.0:1616513100.181162:0:6463:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:0.0:1616513100.181164:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1616513100.181171:0:6463:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:0.0:1616513100.181176:0:6463:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f2330 00000800:00000200:0.0:1616513100.181180:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:0.0:1616513100.181183:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:18.0:1616513100.181210:0:6471:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:18.0:1616513100.181217:0:6471:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:18.0:1616513100.181220:0:6471:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:18.0:1616513100.181227:0:6471:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:18.0:1616513100.181231:0:6471:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4c92eeaa0 00000800:00000200:18.0:1616513100.181235:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000800:00000200:18.0:1616513100.181237:0:6471:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- 00000100:00000200:11.0:1616513100.202844:0:6504:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930192, portal 4 00000100:00000200:11.0:1616513100.202850:0:6504:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930192, offset 0 00000400:00000200:11.0:1616513100.202857:0:6504:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.231@o2ib1 00000400:00000200:11.0:1616513100.202864:0:6504:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.231@o2ib1 local destination 00000400:00000200:11.0:1616513100.202877:0:6504:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.231@o2ib1(10.242.46.231@o2ib1:10.242.46.231@o2ib1) : PUT try# 0 00000800:00000200:11.0:1616513100.202896:0:6504:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.231@o2ib1 00000800:00000200:11.0:1616513100.202902:0:6504:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8da4c91edf80] -> 10.242.46.231@o2ib1 (2) version: 12 00000800:00000200:11.0:1616513100.202905:0:6504:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:11.0:1616513100.202907:0:6504:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:11.0:1616513100.202912:0:6504:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:3.0:1616513100.202924:0:6505:0:(niobuf.c:884:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1693067725930208, portal 4 00000100:00000200:3.0:1616513100.202930:0:6505:0:(niobuf.c:85:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1693067725930208, offset 0 00000400:00000200:3.0:1616513100.202939:0:6505:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.242.46.230@o2ib1 00000400:00000200:3.0:1616513100.202945:0:6505:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source ANY to NMR: 10.242.46.230@o2ib1 local destination 00000400:00000200:3.0:1616513100.202958:0:6505:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:) -> 10.242.46.230@o2ib1(10.242.46.230@o2ib1:10.242.46.230@o2ib1) : PUT try# 0 00000800:00000200:27.2:1616513100.202959:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:3.0:1616513100.202965:0:6505:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 224 bytes in 1 frags to 12345-10.242.46.230@o2ib1 00000800:00000200:3.0:1616513100.202972:0:6505:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac7f80] -> 10.242.46.230@o2ib1 (2) version: 12 00000800:00000200:3.0:1616513100.202975:0:6505:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:3.0:1616513100.202977:0:6505:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:27.0:1616513100.202978:0:6473:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)++ 00000800:00000200:3.0:1616513100.202982:0:6505:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:27.0:1616513100.202983:0:6473:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8da4d5f4e200] (23)-- 00000400:00000200:27.0:1616513100.202985:0:6473:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:31.0:1616513100.202988:0:6475:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000800:00000200:5.2:1616513100.202990:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (21)++ 00000400:00000200:27.0:1616513100.202992:0:6473:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.231@o2ib1: PUT: OK 00000100:00000200:27.0:1616513100.202999:0:6473:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8cca3ef980 x1693067725930192/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:8.0:1616513100.203003:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)++ 00000800:00000200:8.0:1616513100.203010:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa0000] (23)-- 00000400:00000200:27.0:1616513100.203011:0:6473:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a3b8 00000400:00000200:8.0:1616513100.203013:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:27.0:1616513100.203015:0:6473:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000400:00000200:8.0:1616513100.203020:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.230@o2ib1: PUT: OK 00000100:00000200:8.0:1616513100.203026:0:6462:0:(events.c:57:request_out_callback()) @@@ type 5, status 0 req@ffff8d8dd8bfc800 x1693067725930208/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 2 fl Rpc:/0/ffffffff rc 0/-1 00000400:00000200:8.0:1616513100.203037:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2198 00000800:00000200:8.0:1616513100.203041:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:8.0:1616513100.203043:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000800:00000200:27.2:1616513100.203092:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8da4d5f4e200] (20)++ 00000800:00000200:27.0:1616513100.203102:0:6474:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513100.203108:0:6474:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.231@o2ib1 00000800:00000200:31.0:1616513100.203110:0:6476:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (22)-- 00000400:00000200:27.0:1616513100.203117:0:6474:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.231@o2ib1 : PUT - for me 00000400:00000200:27.0:1616513100.203124:0:6474:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.231@o2ib1 of length 336 into portal 4 MB=0x603d60c627ed0 00000400:00000200:27.0:1616513100.203131:0:6474:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.231@o2ib1 of length 336/336 into md 0x371677dd [1] + 192 00000400:00000200:27.0:1616513100.203136:0:6474:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:27.0:1616513100.203140:0:6474:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8cca3ef980 x1693067725930192/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513100.203154:0:6474:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (21)++ 00000800:00000200:27.0:1616513100.203159:0:6474:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8da4d5f4e200] (22)-- 00000100:00000200:11.0:1616513100.203159:0:6504:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8cca3ef980 x1693067725930192/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000800:00000200:27.0:1616513100.203161:0:6474:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8da4d5f4e200] (21)-- 00000100:00000200:11.0:1616513100.203169:0:6504:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8cca3ef980 x1693067725930192/t0(0) o13->boslfs02-OST003a-osc-MDT0001@10.242.46.231@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:11.0:1616513100.203177:0:6504:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d81fcb3a990 00000800:00000200:5.2:1616513100.203178:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa0000] (20)++ 00000800:00000200:0.0:1616513100.203232:0:6463:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513100.203242:0:6463:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.230@o2ib1 00000400:00000200:0.0:1616513100.203252:0:6463:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.242.46.230@o2ib1 : PUT - for me 00000400:00000200:0.0:1616513100.203260:0:6463:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.242.46.230@o2ib1 of length 336 into portal 4 MB=0x603d60c627ee0 00000400:00000200:0.0:1616513100.203267:0:6463:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 4 from 12345-10.242.46.230@o2ib1 of length 336/336 into md 0x866de261 [1] + 192 00000400:00000200:0.0:1616513100.203273:0:6463:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:0.0:1616513100.203278:0:6463:0:(events.c:93:reply_in_callback()) @@@ type 2, status 0 req@ffff8d8dd8bfc800 x1693067725930208/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:/0/ffffffff rc 0/-1 00000800:00000200:4.0:1616513100.203286:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513100.203294:0:6463:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (21)++ 00000800:00000200:0.0:1616513100.203296:0:6463:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa0000] (22)-- 00000800:00000200:0.0:1616513100.203298:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa0000] (21)-- 00000100:00000200:3.0:1616513100.203303:0:6505:0:(events.c:93:reply_in_callback()) @@@ type 6, status 0 req@ffff8d8dd8bfc800 x1693067725930208/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000100:00000200:3.0:1616513100.203315:0:6505:0:(events.c:114:reply_in_callback()) @@@ unlink req@ffff8d8dd8bfc800 x1693067725930208/t0(0) o13->boslfs02-OST0029-osc-MDT0001@10.242.46.230@o2ib1:7/4 lens 224/368 e 0 to 0 dl 1616513107 ref 1 fl Rpc:R/0/ffffffff rc 0/-1 00000400:00000200:3.0:1616513100.203326:0:6505:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8da4cbbd2a18 00000800:00000200:11.2:1616513100.210888:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:8.0:1616513100.210900:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:8.0:1616513100.210906:0:6462:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[1] from 10.242.46.217@o2ib1 00000400:00000200:8.0:1616513100.210926:0:6462:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.34@tcp1 : PUT - for me 00000400:00000200:8.0:1616513100.210934:0:6462:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.34@tcp1 of length 224 into portal 26 MB=0x60183f9c44600 00000400:00000200:8.0:1616513100.210941:0:6462:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index 1a from 12345-10.31.132.34@tcp1 of length 224/224 into md 0xf97d1669 [1] + 0 00000400:00000200:8.0:1616513100.210945:0:6462:0:(lib-md.c:65:lnet_md_unlink()) Queueing unlink of md ffff8d841d788e58 00000400:00000200:8.0:1616513100.210947:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:8.0:1616513100.210950:0:6462:0:(events.c:305:request_in_callback()) event type 2, status 0, service mgs 00000400:00000200:8.0:1616513100.210954:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d841d788e58 00000800:00000200:8.0:1616513100.210957:0:6462:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513100.210959:0:6462:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:8.0:1616513100.210961:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:20.0:1616513100.210963:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:11.2:1616513100.211014:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (20)++ 00000100:00000200:24.0:1616513100.211020:0:252364:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516202997248 00000800:00000200:4.0:1616513100.211024:0:6464:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)++ 00010000:00000200:24.0:1616513100.211031:0:252364:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d7460e74050 x1690516202997248/t0(0) o400->76297a0e-5388-0201-8871-d55a8b705b46@10.31.132.34@tcp1:57/0 lens 224/224 e 0 to 0 dl 1616513192 ref 1 fl Interpret:/0/0 rc 0/0 00000800:00000200:4.0:1616513100.211032:0:6464:0:(o2iblnd_cb.c:343:kiblnd_handle_rx()) Received d1[0] from 10.242.46.217@o2ib1 00000400:00000200:4.0:1616513100.211041:0:6464:0:(lib-move.c:4150:lnet_parse()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1) <- 10.31.132.34@tcp1 : PUT - for me 00000100:00000200:24.0:1616513100.211045:0:252364:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 25, xid 1690516202997248, offset 192 00000400:00000200:4.0:1616513100.211050:0:6464:0:(lib-ptl.c:571:lnet_ptl_match_md()) Request from 12345-10.31.132.34@tcp1 of length 224 into portal 12 MB=0x60183f9c44740 00000400:00000200:24.0:1616513100.211052:0:252364:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.34@tcp1 00000400:00000200:4.0:1616513100.211057:0:6464:0:(lib-ptl.c:200:lnet_try_match_md()) Incoming put index c from 12345-10.31.132.34@tcp1 of length 224/224 into md 0x865be291 [1] + 91016 00000400:00000200:24.0:1616513100.211059:0:252364:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.34@tcp1 routed destination 00000400:00000200:4.0:1616513100.211063:0:6464:0:(lib-msg.c:825:lnet_is_health_check()) health check = 0, status = 0, hstatus = 0 00000100:00000200:4.0:1616513100.211065:0:6464:0:(events.c:305:request_in_callback()) event type 2, status 0, service mdt 00000400:00000200:24.0:1616513100.211074:0:252364:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.34@tcp1(10.31.132.34@tcp1:10.242.46.217@o2ib1) : PUT try# 0 00000800:00000200:4.0:1616513100.211079:0:6464:0:(o2iblnd_cb.c:205:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:24.0:1616513100.211081:0:252364:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.217@o2ib1 00000800:00000200:4.0:1616513100.211084:0:6464:0:(o2iblnd_cb.c:239:kiblnd_post_rx()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:0.0:1616513100.211084:0:6463:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (23)-- 00000800:00000200:24.0:1616513100.211087:0:252364:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d5ccbac6680] -> 10.242.46.217@o2ib1 (2) version: 12 00000800:00000200:4.0:1616513100.211088:0:6464:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000800:00000200:24.0:1616513100.211089:0:252364:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (20)++ 00000800:00000200:24.0:1616513100.211091:0:252364:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d5ccbfa7400] (21)++ 00000800:00000200:24.0:1616513100.211096:0:252364:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d5ccbfa7400] (22)-- 00000800:00000200:11.2:1616513100.211101:0:0:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d5ccbfa7400] (21)++ 00000100:00000200:24.0:1616513100.211103:0:252364:0:(niobuf.c:962:ptlrpc_register_rqbd()) LNetMEAttach: portal 26 00000800:00000200:8.0:1616513100.211108:0:6462:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)++ 00000800:00000200:8.0:1616513100.211112:0:6462:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d5ccbfa7400] (23)-- 00000400:00000200:8.0:1616513100.211114:0:6462:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:20.0:1616513100.211117:0:6461:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (22)-- 00000400:00000200:8.0:1616513100.211119:0:6462:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.217@o2ib1: PUT: OK 00000400:00000200:8.0:1616513100.211123:0:6462:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d8dc32f3a08 00000800:00000200:8.0:1616513100.211125:0:6462:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d5ccbfa7400] (21)-- 00000100:00000200:28.0:1616513100.211138:0:188420:0:(service.c:2145:ptlrpc_server_handle_request()) got req 1690516202997568 00010000:00000200:28.0:1616513100.211151:0:188420:0:(ldlm_lib.c:2944:target_send_reply_msg()) @@@ sending reply req@ffff8d5afb239b00 x1690516202997568/t0(0) o400->5c0f4ddd-32ab-4449-d129-5455590cf8c9@10.31.132.34@tcp1:726/0 lens 224/224 e 0 to 0 dl 1616513106 ref 1 fl Interpret:H/0/0 rc 0/0 00000100:00000200:28.0:1616513100.211176:0:188420:0:(niobuf.c:85:ptl_send_buf()) Sending 192 bytes to portal 10, xid 1690516202997568, offset 192 00000400:00000200:28.0:1616513100.211182:0:188420:0:(lib-move.c:4644:LNetPut()) LNetPut -> 12345-10.31.132.34@tcp1 00000400:00000200:28.0:1616513100.211189:0:188420:0:(lib-move.c:2486:lnet_handle_send_case_locked()) Source Specified: 10.242.46.233@o2ib1 to NMR: 10.31.132.34@tcp1 routed destination 00000400:00000200:28.0:1616513100.211205:0:188420:0:(lib-move.c:1752:lnet_handle_send()) TRACE: 10.242.46.233@o2ib1(10.242.46.233@o2ib1:10.242.46.233@o2ib1) -> 10.31.132.34@tcp1(10.31.132.34@tcp1:10.242.46.216@o2ib1) : PUT try# 0 00000800:00000200:28.0:1616513100.211211:0:188420:0:(o2iblnd_cb.c:1662:kiblnd_send()) sending 192 bytes in 1 frags to 12345-10.242.46.216@o2ib1 00000800:00000200:28.0:1616513100.211217:0:188420:0:(o2iblnd.c:405:kiblnd_find_peer_locked()) got peer_ni [ffff8d7482fa9600] -> 10.242.46.216@o2ib1 (2) version: 12 00000800:00000200:28.0:1616513100.211220:0:188420:0:(o2iblnd_cb.c:1542:kiblnd_launch_tx()) conn[ffff8d74da651200] (20)++ 00000800:00000200:28.0:1616513100.211222:0:188420:0:(o2iblnd_cb.c:1316:kiblnd_queue_tx_locked()) conn[ffff8d74da651200] (21)++ 00000800:00000200:28.0:1616513100.211227:0:188420:0:(o2iblnd_cb.c:1548:kiblnd_launch_tx()) conn[ffff8d74da651200] (22)-- 00000800:00000200:7.2:1616513100.211233:0:51613:0:(o2iblnd_cb.c:3736:kiblnd_cq_completion()) conn[ffff8d74da651200] (21)++ 00000800:00000200:14.0:1616513100.211288:0:6469:0:(o2iblnd_cb.c:3858:kiblnd_scheduler()) conn[ffff8d74da651200] (22)++ 00000800:00000200:14.0:1616513100.211295:0:6469:0:(o2iblnd_cb.c:75:kiblnd_tx_done()) conn[ffff8d74da651200] (23)-- 00000400:00000200:14.0:1616513100.211297:0:6469:0:(lib-msg.c:825:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000200:26.0:1616513100.211301:0:6470:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (22)-- 00000400:00000200:14.0:1616513100.211303:0:6469:0:(lib-msg.c:635:lnet_health_check()) health check: 10.242.46.233@o2ib1->10.242.46.216@o2ib1: PUT: OK 00000400:00000200:14.0:1616513100.211307:0:6469:0:(lib-md.c:69:lnet_md_unlink()) Unlinking md ffff8d5cd4fcd980 00000800:00000200:14.0:1616513100.211311:0:6469:0:(o2iblnd_cb.c:3874:kiblnd_scheduler()) conn[ffff8d74da651200] (21)-- Debug log: 162881 lines, 162881 kept, 0 dropped, 0 bad.